Optionally do not slugify paths (#875)

* maybe_slugify() only does simple sanitation if config.slugify is false

* slugify is disabled by default, turn on for backwards-compatibility

* First docs changes for optional slugification

* Remove # from slugs but not &

* Add/fix tests for utf8 slugs

* Fix test sites for i18n slugs

* fix templates tests for i18n slugs

* Rename slugify setting to slugify_paths

* Default slugify_paths

* Update documentation for slugify_paths

* quasi_slugify removes ?, /, # and newlines

* Remove forbidden NTFS chars in quasi_slugify()

* Slugification forbidden chars can be configured

* Remove trailing dot/space in quasi_slugify

* Fix NTFS path sanitation

* Revert configurable slugification charset

* Remove \r for windows newlines and \t tabulations in quasi_slugify()

* Update docs for output paths

* Replace slugify with slugify_paths

* Fix test

* Default to not slugifying

* Move slugs utils to utils crate

* Use slugify_paths for anchors as well

Author: Keats

.gitignore

@@ -25,3 +25,5 @@ stage
 
 # nixos dependencies snippet
 shell.nix
+# vim temporary files
+**/.*.sw*

CHANGELOG.md

@@ -5,6 +5,8 @@
 ### Breaking
 - Remove `toc` variable in section/page context and pass it to `page.toc` and `section.toc` instead so they are
 accessible everywhere
+- [Slugification](https://en.wikipedia.org/wiki/Slug_(web_publishing)#Slug) of page paths is now optional. By default, every path will be slugified as it is happening right now.
+To keep non-ASCII characters,  set `slugify_paths = true` in your config.
 
 ### Other
 - Add zenburn syntax highlighting theme

Cargo.lock

@@ -130,6 +130,8 @@ pub struct Config {
     /// key into different language.
     translations: HashMap<String, TranslateTerm>,
 
+    /// Whether to slugify page and taxonomy URLs (disable for UTF-8 URLs)
+    pub slugify_paths: bool,
     /// Whether to highlight all code blocks found in markdown files. Defaults to false
     pub highlight_code: bool,
     /// Which themes to use for code highlighting. See Readme for supported themes
@@ -354,6 +356,7 @@ impl Default for Config {
             title: None,
             description: None,
             theme: None,
+            slugify_paths: true,
             highlight_code: false,
             highlight_theme: "base16-ocean-dark".to_string(),
             default_language: "en".to_string(),

components/config/src/config.rs

@@ -10,7 +10,6 @@ chrono = { version = "0.4", features = ["serde"] }
 tera = "1"
 serde = "1"
 serde_derive = "1"
-slug = "0.1"
 regex = "1"
 lazy_static = "1"
 

components/library/Cargo.toml

@@ -4,7 +4,6 @@ use std::path::{Path, PathBuf};
 
 use regex::Regex;
 use slotmap::DefaultKey;
-use slug::slugify;
 use tera::{Context as TeraContext, Tera};
 
 use config::Config;
@@ -19,6 +18,7 @@ use utils::templates::render_template;
 use content::file_info::FileInfo;
 use content::has_anchor;
 use content::ser::SerializingPage;
+use utils::slugs::maybe_slugify_paths;
 
 lazy_static! {
     // Based on https://regex101.com/r/H2n38Z/1/tests
@@ -160,21 +160,21 @@ impl Page {
 
         page.slug = {
             if let Some(ref slug) = page.meta.slug {
-                slugify(&slug.trim())
+                maybe_slugify_paths(&slug.trim(), config.slugify_paths)
             } else if page.file.name == "index" {
                 if let Some(parent) = page.file.path.parent() {
                     if let Some(slug) = slug_from_dated_filename {
-                        slugify(&slug)
+                        maybe_slugify_paths(&slug, config.slugify_paths)
                     } else {
-                        slugify(parent.file_name().unwrap().to_str().unwrap())
+                        maybe_slugify_paths(parent.file_name().unwrap().to_str().unwrap(), config.slugify_paths)
                     }
                 } else {
-                    slugify(&page.file.name)
+                    maybe_slugify_paths(&page.file.name, config.slugify_paths)
                 }
             } else if let Some(slug) = slug_from_dated_filename {
-                slugify(&slug)
+                maybe_slugify_paths(&slug, config.slugify_paths)
             } else {
-                slugify(&page.file.name)
+                maybe_slugify_paths(&page.file.name, config.slugify_paths)
             }
         };
 
@@ -443,7 +443,8 @@ Hello world"#;
     slug = "hello-&-world"
     +++
     Hello world"#;
-        let config = Config::default();
+        let mut config = Config::default();
+        config.slugify_paths = true;
         let res = Page::parse(Path::new("start.md"), content, &config, &PathBuf::new());
         assert!(res.is_ok());
         let page = res.unwrap();
@@ -452,6 +453,23 @@ Hello world"#;
         assert_eq!(page.permalink, config.make_permalink("hello-world"));
     }
 
+    #[test]
+    fn can_make_url_from_utf8_slug_frontmatter() {
+        let content = r#"
+    +++
+    slug = "日本"
+    +++
+    Hello world"#;
+        let mut config = Config::default();
+        config.slugify_paths = false;
+        let res = Page::parse(Path::new("start.md"), content, &config, &PathBuf::new());
+        assert!(res.is_ok());
+        let page = res.unwrap();
+        assert_eq!(page.path, "日本/");
+        assert_eq!(page.components, vec!["日本"]);
+        assert_eq!(page.permalink, config.make_permalink("日本"));
+    }
+
     #[test]
     fn can_make_url_from_path() {
         let content = r#"
@@ -508,7 +526,8 @@ Hello world"#;
 
     #[test]
     fn can_make_slug_from_non_slug_filename() {
-        let config = Config::default();
+        let mut config = Config::default();
+        config.slugify_paths = true;
         let res =
             Page::parse(Path::new(" file with space.md"), "+++\n+++", &config, &PathBuf::new());
         assert!(res.is_ok());
@@ -517,6 +536,17 @@ Hello world"#;
         assert_eq!(page.permalink, config.make_permalink(&page.slug));
     }
 
+    #[test]
+    fn can_make_path_from_utf8_filename() {
+        let mut config = Config::default();
+        config.slugify_paths = false;
+        let res = Page::parse(Path::new("日本.md"), "+++\n++++", &config, &PathBuf::new());
+        assert!(res.is_ok());
+        let page = res.unwrap();
+        assert_eq!(page.slug, "日本");
+        assert_eq!(page.permalink, config.make_permalink(&page.slug));
+    }
+
     #[test]
     fn can_specify_summary() {
         let config = Config::default();

components/library/src/content/page.rs

@@ -1,5 +1,4 @@
 extern crate serde;
-extern crate slug;
 extern crate tera;
 #[macro_use]
 extern crate serde_derive;