Multiple slugification strategies (#929)

Author: Keats

CHANGELOG.md

@@ -5,8 +5,8 @@
 ### Breaking
 - Remove `toc` variable in section/page context and pass it to `page.toc` and `section.toc` instead so they are
 accessible everywhere
-- [Slugification](https://en.wikipedia.org/wiki/Slug_(web_publishing)#Slug) of page paths is now optional. By default, every path will be slugified as it is happening right now.
-To keep non-ASCII characters,  set `slugify_paths = true` in your config.
+- [Slugification](https://en.wikipedia.org/wiki/Slug_(web_publishing)#Slug) of paths, taxonomies and anchors is now optional. By default, everything will still be slugified like in previous versions.
+See documentation for information on how to disable it.
 
 ### Other
 - Add zenburn syntax highlighting theme

Cargo.lock

@@ -12,6 +12,7 @@ use crate::highlighting::THEME_SET;
 use crate::theme::Theme;
 use errors::{bail, Error, Result};
 use utils::fs::read_file_with_error;
+use utils::slugs::SlugifyStrategy;
 
 // We want a default base url for tests
 static DEFAULT_BASE_URL: &str = "http://a-website.com";
@@ -23,6 +24,24 @@ pub enum Mode {
     Check,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(default)]
+pub struct Slugify {
+    pub paths: SlugifyStrategy,
+    pub taxonomies: SlugifyStrategy,
+    pub anchors: SlugifyStrategy,
+}
+
+impl Default for Slugify {
+    fn default() -> Self {
+        Slugify {
+            paths: SlugifyStrategy::On,
+            taxonomies: SlugifyStrategy::On,
+            anchors: SlugifyStrategy::On,
+        }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(default)]
 pub struct Language {
@@ -35,7 +54,7 @@ pub struct Language {
 }
 
 impl Default for Language {
-    fn default() -> Language {
+    fn default() -> Self {
         Language { code: String::new(), rss: false, search: false }
     }
 }
@@ -75,7 +94,7 @@ impl Taxonomy {
 }
 
 impl Default for Taxonomy {
-    fn default() -> Taxonomy {
+    fn default() -> Self {
         Taxonomy {
             name: String::new(),
             paginate_by: None,
@@ -130,8 +149,6 @@ pub struct Config {
     /// key into different language.
     translations: HashMap<String, TranslateTerm>,
 
-    /// Whether to slugify page and taxonomy URLs (disable for UTF-8 URLs)
-    pub slugify_paths: bool,
     /// Whether to highlight all code blocks found in markdown files. Defaults to false
     pub highlight_code: bool,
     /// Which themes to use for code highlighting. See Readme for supported themes
@@ -171,6 +188,9 @@ pub struct Config {
 
     pub link_checker: LinkChecker,
 
+    /// The setup for which slugification strategies to use for paths, taxonomies and anchors
+    pub slugify: Slugify,
+
     /// All user params set in [extra] in the config
     pub extra: HashMap<String, Toml>,
 
@@ -356,7 +376,6 @@ impl Default for Config {
             title: None,
             description: None,
             theme: None,
-            slugify_paths: true,
             highlight_code: false,
             highlight_theme: "base16-ocean-dark".to_string(),
             default_language: "en".to_string(),
@@ -374,6 +393,7 @@ impl Default for Config {
             extra_syntaxes: Vec::new(),
             extra_syntax_set: None,
             link_checker: LinkChecker::default(),
+            slugify: Slugify::default(),
             extra: HashMap::new(),
             build_timestamp: Some(1),
         }
@@ -382,7 +402,7 @@ impl Default for Config {
 
 #[cfg(test)]
 mod tests {
-    use super::{Config, Theme};
+    use super::{Config, SlugifyStrategy, Theme};
 
     #[test]
     fn can_import_valid_config() {
@@ -619,4 +639,22 @@ skip_prefixes = [
             vec!["http://[2001:db8::]/", "https://www.example.com/path",]
         );
     }
+
+    #[test]
+    fn slugify_strategies() {
+        let config_str = r#"
+title = "My site"
+base_url = "example.com"
+
+[slugify]
+paths = "on"
+taxonomies = "safe"
+anchors = "off"
+        "#;
+
+        let config = Config::parse(config_str).unwrap();
+        assert_eq!(config.slugify.paths, SlugifyStrategy::On);
+        assert_eq!(config.slugify.taxonomies, SlugifyStrategy::Safe);
+        assert_eq!(config.slugify.anchors, SlugifyStrategy::Off);
+    }
 }

components/config/src/config.rs

@@ -19,7 +19,7 @@ use utils::templates::render_template;
 use crate::content::file_info::FileInfo;
 use crate::content::has_anchor;
 use crate::content::ser::SerializingPage;
-use utils::slugs::maybe_slugify_paths;
+use utils::slugs::slugify_paths;
 
 lazy_static! {
     // Based on https://regex101.com/r/H2n38Z/1/tests
@@ -161,24 +161,24 @@ impl Page {
 
         page.slug = {
             if let Some(ref slug) = page.meta.slug {
-                maybe_slugify_paths(&slug.trim(), config.slugify_paths)
+                slugify_paths(slug, config.slugify.paths)
             } else if page.file.name == "index" {
                 if let Some(parent) = page.file.path.parent() {
                     if let Some(slug) = slug_from_dated_filename {
-                        maybe_slugify_paths(&slug, config.slugify_paths)
+                        slugify_paths(&slug, config.slugify.paths)
                     } else {
-                        maybe_slugify_paths(
+                        slugify_paths(
                             parent.file_name().unwrap().to_str().unwrap(),
-                            config.slugify_paths,
+                            config.slugify.paths,
                         )
                     }
                 } else {
-                    maybe_slugify_paths(&page.file.name, config.slugify_paths)
+                    slugify_paths(&page.file.name, config.slugify.paths)
                 }
             } else if let Some(slug) = slug_from_dated_filename {
-                maybe_slugify_paths(&slug, config.slugify_paths)
+                slugify_paths(&slug, config.slugify.paths)
             } else {
-                maybe_slugify_paths(&page.file.name, config.slugify_paths)
+                slugify_paths(&page.file.name, config.slugify.paths)
             }
         };
 
@@ -379,6 +379,7 @@ mod tests {
     use super::Page;
     use config::{Config, Language};
     use front_matter::InsertAnchor;
+    use utils::slugs::SlugifyStrategy;
 
     #[test]
     fn test_can_parse_a_valid_page() {
@@ -448,7 +449,7 @@ Hello world"#;
     +++
     Hello world"#;
         let mut config = Config::default();
-        config.slugify_paths = true;
+        config.slugify.paths = SlugifyStrategy::On;
         let res = Page::parse(Path::new("start.md"), content, &config, &PathBuf::new());
         assert!(res.is_ok());
         let page = res.unwrap();
@@ -465,7 +466,7 @@ Hello world"#;
     +++
     Hello world"#;
         let mut config = Config::default();
-        config.slugify_paths = false;
+        config.slugify.paths = SlugifyStrategy::Safe;
         let res = Page::parse(Path::new("start.md"), content, &config, &PathBuf::new());
         assert!(res.is_ok());
         let page = res.unwrap();
@@ -531,7 +532,7 @@ Hello world"#;
     #[test]
     fn can_make_slug_from_non_slug_filename() {
         let mut config = Config::default();
-        config.slugify_paths = true;
+        config.slugify.paths = SlugifyStrategy::On;
         let res =
             Page::parse(Path::new(" file with space.md"), "+++\n+++", &config, &PathBuf::new());
         assert!(res.is_ok());
@@ -543,7 +544,7 @@ Hello world"#;
     #[test]
     fn can_make_path_from_utf8_filename() {
         let mut config = Config::default();
-        config.slugify_paths = false;
+        config.slugify.paths = SlugifyStrategy::Safe;
         let res = Page::parse(Path::new("日本.md"), "+++\n++++", &config, &PathBuf::new());
         assert!(res.is_ok());
         let page = res.unwrap();

components/library/src/content/page.rs

@@ -11,7 +11,7 @@ use utils::templates::render_template;
 use crate::content::SerializingPage;
 use crate::library::Library;
 use crate::sorting::sort_pages_by_date;
-use utils::slugs::maybe_slugify_paths;
+use utils::slugs::slugify_paths;
 
 #[derive(Debug, Clone, PartialEq, Serialize)]
 pub struct SerializedTaxonomyItem<'a> {
@@ -70,7 +70,7 @@ impl TaxonomyItem {
             })
             .collect();
         let (mut pages, ignored_pages) = sort_pages_by_date(data);
-        let slug = maybe_slugify_paths(name, config.slugify_paths);
+        let slug = slugify_paths(name, config.slugify.taxonomies);
         let permalink = if taxonomy.lang != config.default_language {
             config.make_permalink(&format!("/{}/{}/{}", taxonomy.lang, taxonomy.name, slug))
         } else {
@@ -235,6 +235,7 @@ mod tests {
     use crate::content::Page;
     use crate::library::Library;
     use config::{Config, Language, Taxonomy as TaxonomyConfig};
+    use utils::slugs::SlugifyStrategy;
 
     #[test]
     fn can_make_taxonomies() {
@@ -565,7 +566,7 @@ mod tests {
     #[test]
     fn can_make_utf8_taxonomies() {
         let mut config = Config::default();
-        config.slugify_paths = false;
+        config.slugify.taxonomies = SlugifyStrategy::Safe;
         config.languages.push(Language {
             rss: false,
             code: "fr".to_string(),
@@ -598,7 +599,7 @@ mod tests {
     #[test]
     fn can_make_slugified_taxonomies_in_multiple_languages() {
         let mut config = Config::default();
-        config.slugify_paths = true;
+        config.slugify.taxonomies = SlugifyStrategy::On;
         config.languages.push(Language {
             rss: false,
             code: "fr".to_string(),

components/library/src/taxonomies/mod.rs

@@ -86,8 +86,7 @@ fn bench_render_content_with_highlighting(b: &mut test::Bencher) {
     tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
     let permalinks_ctx = HashMap::new();
     let config = Config::default();
-    let context =
-        RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
     b.iter(|| render_content(CONTENT, &context).unwrap());
 }
 
@@ -98,8 +97,7 @@ fn bench_render_content_without_highlighting(b: &mut test::Bencher) {
     let permalinks_ctx = HashMap::new();
     let mut config = Config::default();
     config.highlight_code = false;
-    let context =
-        RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
     b.iter(|| render_content(CONTENT, &context).unwrap());
 }
 
@@ -110,8 +108,7 @@ fn bench_render_content_no_shortcode(b: &mut test::Bencher) {
     let mut config = Config::default();
     config.highlight_code = false;
     let permalinks_ctx = HashMap::new();
-    let context =
-        RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
 
     b.iter(|| render_content(&content2, &context).unwrap());
 }
@@ -122,8 +119,7 @@ fn bench_render_shortcodes_one_present(b: &mut test::Bencher) {
     tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
     let config = Config::default();
     let permalinks_ctx = HashMap::new();
-    let context =
-        RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
 
     b.iter(|| render_shortcodes(CONTENT, &context));
 }

components/rendering/benches/all.rs

@@ -12,7 +12,7 @@ use config::highlighting::{get_highlighter, SYNTAX_SET, THEME_SET};
 use errors::{Error, Result};
 use front_matter::InsertAnchor;
 use utils::site::resolve_internal_link;
-use utils::slugs::maybe_slugify_anchors;
+use utils::slugs::slugify_anchors;
 use utils::vec::InsertMany;
 
 use self::cmark::{Event, LinkType, Options, Parser, Tag};
@@ -305,7 +305,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
             let id = heading_ref.id.unwrap_or_else(|| {
                 find_anchor(
                     &inserted_anchors,
-                    maybe_slugify_anchors(&title, context.config.slugify_paths),
+                    slugify_anchors(&title, context.config.slugify.anchors),
                     0,
                 )
             });

components/rendering/src/markdown.rs

@@ -6,6 +6,7 @@ use config::Config;
 use front_matter::InsertAnchor;
 use rendering::{render_content, RenderContext};
 use templates::ZOLA_TERA;
+use utils::slugs::SlugifyStrategy;
 
 #[test]
 fn can_do_render_content_simple() {
@@ -350,7 +351,7 @@ fn can_add_non_slug_id_to_headings() {
     let tera_ctx = Tera::default();
     let permalinks_ctx = HashMap::new();
     let mut config = Config::default();
-    config.slugify_paths = false;
+    config.slugify.anchors = SlugifyStrategy::Safe;
     let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
     let res = render_content(r#"# L'écologie et vous"#, &context).unwrap();
     assert_eq!(res.body, "<h1 id=\"L'écologie_et_vous\">L'écologie et vous</h1>\n");
@@ -880,4 +881,4 @@ fn stops_with_an_error_on_an_empty_link() {
 
     assert!(res.is_err());
     assert_eq!(res.unwrap_err().to_string(), expected);
-}
+}

components/rendering/tests/markdown.rs

@@ -686,14 +686,14 @@ fn can_ignore_markdown_content() {
 
 #[test]
 fn check_site() {
-   let (mut site, _tmp_dir, _public) = build_site("test_site");
+    let (mut site, _tmp_dir, _public) = build_site("test_site");
 
-   assert_eq!(
-       site.config.link_checker.skip_anchor_prefixes,
-       vec!["https://github.com/rust-lang/rust/blob/"]
-   );
-   assert_eq!(site.config.link_checker.skip_prefixes, vec!["http://[2001:db8::]/"]);
+    assert_eq!(
+        site.config.link_checker.skip_anchor_prefixes,
+        vec!["https://github.com/rust-lang/rust/blob/"]
+    );
+    assert_eq!(site.config.link_checker.skip_prefixes, vec!["http://[2001:db8::]/"]);
 
-   site.config.enable_check_mode();
-   site.load().expect("link check test_site");
+    site.config.enable_check_mode();
+    site.load().expect("link check test_site");
 }

components/site/tests/site.rs

@@ -346,6 +346,7 @@ mod tests {
 
     use config::{Config, Taxonomy as TaxonomyConfig};
     use library::{Library, Taxonomy, TaxonomyItem};
+    use utils::slugs::SlugifyStrategy;
 
     #[test]
     fn can_add_cachebust_to_url() {
@@ -390,7 +391,7 @@ mod tests {
     #[test]
     fn can_get_taxonomy() {
         let mut config = Config::default();
-        config.slugify_paths = true;
+        config.slugify.taxonomies = SlugifyStrategy::On;
         let taxo_config = TaxonomyConfig {
             name: "tags".to_string(),
             lang: config.default_language.clone(),
@@ -468,7 +469,7 @@ mod tests {
     #[test]
     fn can_get_taxonomy_url() {
         let mut config = Config::default();
-        config.slugify_paths = true;
+        config.slugify.taxonomies = SlugifyStrategy::On;
         let taxo_config = TaxonomyConfig {
             name: "tags".to_string(),
             lang: config.default_language.clone(),