chore(diff): add allowlist support for diffing content (#32)

* add allowlist support for diffing content
* more ignored files, added the csv output option for listing differences by file and json key
---------

Co-authored-by: Florian Dieminger <[email protected]>

Author: argl

crates/diff-test/src/main.rs

@@ -143,6 +143,8 @@ struct BuildArgs {
     #[arg(long)]
     html: bool,
     #[arg(long)]
+    csv: bool,
+    #[arg(long)]
     inline: bool,
     #[arg(long)]
     ignore_html_whitespace: bool,
@@ -175,7 +177,7 @@ fn is_html(s: &str) -> bool {
     s.trim_start().starts_with('<') && s.trim_end().ends_with('>')
 }
 
-const IGNORE: &[&str] = &[
+const IGNORED_KEYS: &[&str] = &[
     "doc.flaws",
     "blogMeta.readTime",
     "doc.modified",
@@ -189,9 +191,70 @@ const IGNORE: &[&str] = &[
     "doc.summary",
 ];
 
+static SKIP_GLOB_LIST: LazyLock<Vec<&str>> = LazyLock::new(|| {
+    vec![
+        "docs/mdn/writing_guidelines/",
+        "docs/mozilla/add-ons/webextensions/",
+        "docs/mozilla/firefox/releases/",
+    ]
+});
+
+static ALLOWLIST: LazyLock<HashSet<(&str, &str)>> = LazyLock::new(|| {
+    vec![
+        // Wrong auto-linking of example.com properly escaped link, unfixable in yari
+        ("docs/glossary/http/index.json", "doc.body.0.value.content"),
+        ("docs/learn/html/multimedia_and_embedding/other_embedding_technologies/index.json", "doc.body.4.value.content"),
+        // Relative link to MDN Playground gets rendered as dead link in yari, correct in rari
+        ("docs/learn/learning_and_getting_help/index.json", "doc.body.3.value.content"),
+        // 'unsupported templ: livesamplelink' in rari, remove when supported
+        ("docs/learn/forms/form_validation/index.json", "doc.body.12.value.content"),
+        ("docs/mdn/writing_guidelines/page_structures/live_samples/index.json", "doc.body.9.value.content"),
+        // p tag removal in lists
+        ("docs/learn/server-side/express_nodejs/deployment/index.json", "doc.body.11.value.content"),
+        // link element re-structure, better in rari
+        ("docs/learn/common_questions/design_and_accessibility/design_for_all_types_of_users/index.json", "doc.body.5.value.content"),
+        ("docs/learn/html/multimedia_and_embedding/video_and_audio_content/index.json", "doc.body.2.value.content"),
+        // id changes, no problem
+        ("docs/learn/css/howto/css_faq/index.json", "doc.body.11.value.id"),
+        ("docs/learn/forms/property_compatibility_table_for_form_controls/index.json", "doc.body.2.value.content"),
+        ("docs/learn/html/howto/define_terms_with_html/index.json", "doc.body.0.value.content"),
+        ("docs/learn/tools_and_testing/client-side_javascript_frameworks/react_interactivity_filtering_conditional_rendering/index.json", "doc.toc.3.id"),
+        ("docs/learn/tools_and_testing/client-side_javascript_frameworks/react_interactivity_filtering_conditional_rendering/index.json", "doc.body.4.value.id"),
+        ("docs/mdn/mdn_product_advisory_board/index.json", "doc.body.1.value.content"),
+        ("docs/mdn/writing_guidelines/page_structures/live_samples/index.json", "doc.body.11.value.content"),
+        ("docs/mdn/writing_guidelines/page_structures/live_samples/index.json", "doc.body.12.value.content"),
+        ("docs/mdn/writing_guidelines/page_structures/live_samples/index.json", "doc.body.3.value.content"),
+        // absolute to relative link change, no problem
+        ("docs/learn/forms/styling_web_forms/index.json", "doc.body.10.value.content"),
+        ("docs/mdn/kitchensink/index.json", "doc.body.24.value.content"),
+        // encoding changes, no problem
+        ("docs/learn/html/introduction_to_html/html_text_fundamentals/index.json", "doc.body.15.value.content"),
+        ("docs/learn/tools_and_testing/client-side_javascript_frameworks/vue_computed_properties/index.json", "doc.body.1.value.content"),
+        ("docs/learn/tools_and_testing/client-side_javascript_frameworks/react_interactivity_filtering_conditional_rendering/index.json", "doc.body.4.value.i"),
+        ("docs/mdn/writing_guidelines/page_structures/links/index.json", "doc.body.3.value.content"),
+        ("docs/mdn/writing_guidelines/page_structures/links/index.json", "doc.body.4.value.content"),
+        ("docs/mdn/writing_guidelines/page_structures/macros/commonly_used_macros/index.json", "doc.body.14.value.content"),
+        // internal linking fixed in rari
+        ("docs/mdn/community/discussions/index.json", "doc.body.0.value.content"),
+        // baseline change no problem
+        ("docs/mdn/kitchensink/index.json", "doc.baseline"),
+        ("docs/mdn/writing_guidelines/page_structures/compatibility_tables/index.json", "doc.baseline"),
+        // whitespace changes no problem
+        ("docs/mdn/kitchensink/index.json", "doc.body.23.value.title"),
+        ("docs/mdn/writing_guidelines/howto/write_an_api_reference/index.json", "doc.body.8.value.content"),
+        ("docs/mdn/writing_guidelines/page_structures/code_examples/index.json", "doc.body.7.value.content"),
+        // bug in yari
+        ("docs/mdn/writing_guidelines/howto/write_an_api_reference/information_contained_in_a_webidl_file/index.json", "doc.body.23.value.content"),
+        ]
+    .into_iter()
+    .collect()
+});
+
 static WS_DIFF: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r#"(?<x>>)[\n ]+|[\n ]+(?<y></)"#).unwrap());
 
+static EMPTY_P_DIFF: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"<p>[\n ]*</p>"#).unwrap());
+
 static DIFF_MAP: LazyLock<Arc<DashMap<String, String>>> =
     LazyLock::new(|| Arc::new(DashMap::new()));
 
@@ -204,6 +267,10 @@ fn pre_diff_element_massaging_handlers<'a>() -> Vec<(Cow<'a, Selector>, ElementC
             el.remove_attribute("data-flaw-src");
             Ok(())
         }),
+        element!("*[data-flaw]", |el| {
+            el.remove_attribute("data-flaw");
+            Ok(())
+        }),
         // remove ids from notecards, example-headers, code-examples
         element!("div.notecard, div.example-header, div.code-example", |el| {
             el.remove_attribute("id");
@@ -215,6 +282,7 @@ fn pre_diff_element_massaging_handlers<'a>() -> Vec<(Cow<'a, Selector>, ElementC
 fn full_diff(
     lhs: &Value,
     rhs: &Value,
+    file: &str,
     path: &[PathIndex],
     diff: &mut BTreeMap<String, String>,
     fast: bool,
@@ -227,9 +295,19 @@ fn full_diff(
             }
         }
     }
+    let key = make_key(path);
+
+    if SKIP_GLOB_LIST.iter().any(|i| file.starts_with(i)) {
+        return;
+    }
+
+    if ALLOWLIST.contains(&(file, &key)) {
+        return;
+    }
+
     if lhs != rhs {
-        let key = make_key(path);
-        if IGNORE.iter().any(|i| key.starts_with(i)) || key == "doc.sidebarHTML" && !sidebars {
+        if IGNORED_KEYS.iter().any(|i| key.starts_with(i)) || key == "doc.sidebarHTML" && !sidebars
+        {
             return;
         }
         match (lhs, rhs) {
@@ -241,6 +319,7 @@ fn full_diff(
                     full_diff(
                         lhs.get(i).unwrap_or(&Value::Null),
                         rhs.get(i).unwrap_or(&Value::Null),
+                        file,
                         &path,
                         diff,
                         fast,
@@ -257,6 +336,7 @@ fn full_diff(
                     full_diff(
                         lhs.get(key).unwrap_or(&Value::Null),
                         rhs.get(key).unwrap_or(&Value::Null),
+                        file,
                         &path,
                         diff,
                         fast,
@@ -281,6 +361,8 @@ fn full_diff(
                 if is_html(&lhs) && is_html(&rhs) {
                     let lhs_t = WS_DIFF.replace_all(&lhs, "$x$y");
                     let rhs_t = WS_DIFF.replace_all(&rhs, "$x$y");
+                    let lhs_t = EMPTY_P_DIFF.replace_all(&lhs_t, "");
+                    let rhs_t = EMPTY_P_DIFF.replace_all(&rhs_t, "");
                     let lhs_t = rewrite_str(
                         &lhs_t,
                         RewriteStrSettings {
@@ -360,7 +442,7 @@ fn main() -> Result<(), anyhow::Error> {
                         let left = v;
                         let right = b.get(k).unwrap_or(&Value::Null);
                         let mut diff = BTreeMap::new();
-                        full_diff(left, right, &[], &mut diff, arg.fast, arg.sidebars);
+                        full_diff(left, right, k, &[], &mut diff, arg.fast, arg.sidebars);
                         if !diff.is_empty() {
                             return Some(format!(
                                 r#"<li><span>{k}</span><div class="r"><pre><code>{}</code></pre></div></li>"#,
@@ -412,11 +494,45 @@ fn main() -> Result<(), anyhow::Error> {
                 let mut file = File::create(&arg.out)?;
                 file.write_all(html(&out.into_iter().collect::<String>()).as_bytes())?;
             }
+            if arg.csv {
+                let mut out = Vec::new();
+                out.push("File;JSON Path\n".to_string());
+                out.extend(
+                    a.par_iter()
+                        .filter_map(|(k, v)| {
+                            if b.get(k) == Some(v) {
+                                same.fetch_add(1, Relaxed);
+                                return None;
+                            }
+
+                            let left = v;
+                            let right = b.get(k).unwrap_or(&Value::Null);
+                            let mut diff = BTreeMap::new();
+                            full_diff(left, right, k, &[], &mut diff, arg.fast, arg.sidebars);
+                            if !diff.is_empty() {
+                                return Some(format!(
+                                    "{}\n",
+                                    diff.into_keys()
+                                        .map(|jsonpath| format!("{};{}", k, jsonpath))
+                                        .collect::<Vec<_>>()
+                                        .join("\n")
+                                ));
+                            } else {
+                                same.fetch_add(1, Relaxed);
+                            }
+                            None
+                        })
+                        .collect::<Vec<_>>(),
+                );
+                let mut file = File::create(&arg.out)?;
+                file.write_all(out.into_iter().collect::<String>().as_bytes())?;
+            }
 
             println!(
-                "Took: {:?} - {}/{hits}",
+                "Took: {:?} - {}/{hits} ok, {} remaining",
                 start.elapsed(),
-                same.load(Relaxed)
+                same.load(Relaxed),
+                hits - same.load(Relaxed)
             );
         }
     }