feature(cli): sync translated content (#24)

argl · fiji-flo · web-flow · commit a3e3e871d783 · 2024-10-24T09:47:21.000+02:00
---------

Co-authored-by: Florian Dieminger &lt;me@fiji-flo.de&gt;
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -2,3 +2,4 @@
 TESTING_CONTENT_ROOT = { value = "tests/data/content/files", relative = true }
 TESTING_CONTENT_TRANSLATED_ROOT = { value = "tests/data/translated_content/files", relative = true }
 TESTING_CACHE_CONTENT = "0"
+TESTING_READER_IGNORES_GITIGNORE = "1"
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/rari-cli/main.rs b/crates/rari-cli/main.rs
@@ -23,7 +23,9 @@ use rari_tools::history::gather_history;
 use rari_tools::popularities::update_popularities;
 use rari_tools::r#move::r#move;
 use rari_tools::remove::remove;
+use rari_tools::sync_translated_content::sync_translated_content;
 use rari_types::globals::{build_out_root, content_root, content_translated_root, SETTINGS};
+use rari_types::locale::Locale;
 use rari_types::settings::Settings;
 use self_update::cargo_crate_version;
 use tabwriter::TabWriter;
@@ -68,21 +70,22 @@ enum ContentSubcommand {
     Move(MoveArgs),
     Delete(DeleteArgs),
     AddRedirect(AddRedirectArgs),
+    SyncTranslatedContent(SyncTranslatedContentArgs),
 }
 
 #[derive(Args)]
 struct MoveArgs {
     old_slug: String,
     new_slug: String,
-    locale: Option<String>,
+    locale: Option<Locale>,
     #[arg(short = 'y', long, help = "Assume yes to all prompts")]
     assume_yes: bool,
 }
 
 #[derive(Args)]
 struct DeleteArgs {
     slug: String,
-    locale: Option<String>,
+    locale: Option<Locale>,
     #[arg(short, long, default_value_t = false)]
     recursive: bool,
     #[arg(long)]
@@ -97,6 +100,11 @@ struct AddRedirectArgs {
     to_url: String,
 }
 
+#[derive(Args)]
+struct SyncTranslatedContentArgs {
+    locales: Option<Vec<Locale>>,
+}
+
 #[derive(Args)]
 struct UpdateArgs {
     #[arg(long)]
@@ -349,17 +357,12 @@ fn main() -> Result<(), Error> {
         }
         Commands::Content(content_subcommand) => match content_subcommand {
             ContentSubcommand::Move(args) => {
-                r#move(
-                    &args.old_slug,
-                    &args.new_slug,
-                    args.locale.as_deref(),
-                    args.assume_yes,
-                )?;
+                r#move(&args.old_slug, &args.new_slug, args.locale, args.assume_yes)?;
             }
             ContentSubcommand::Delete(args) => {
                 remove(
                     &args.slug,
-                    args.locale.as_deref(),
+                    args.locale,
                     args.recursive,
                     args.redirect.as_deref(),
                     args.assume_yes,
@@ -368,6 +371,10 @@ fn main() -> Result<(), Error> {
             ContentSubcommand::AddRedirect(args) => {
                 add_redirect(&args.from_url, &args.to_url)?;
             }
+            ContentSubcommand::SyncTranslatedContent(args) => {
+                let locales = args.locales.as_deref().unwrap_or(Locale::translated());
+                sync_translated_content(locales, cli.verbose.is_present())?;
+            }
         },
         Commands::Update(args) => update(args.version)?,
     }
diff --git a/crates/rari-doc/src/cached_readers.rs b/crates/rari-doc/src/cached_readers.rs
@@ -318,17 +318,17 @@ pub fn read_and_cache_doc_pages() -> Result<Vec<Page>, DocError> {
         )
         .unwrap();
     if let Some(translated_root) = content_translated_root() {
-        let transted_docs = read_docs_parallel::<Doc>(&[translated_root], None)?;
+        let translated_docs = read_docs_parallel::<Doc>(&[translated_root], None)?;
         STATIC_DOC_PAGE_TRANSLATED_FILES
             .set(
-                transted_docs
+                translated_docs
                     .iter()
                     .cloned()
                     .map(|doc| ((doc.locale(), Cow::Owned(doc.slug().to_string())), doc))
                     .collect(),
             )
             .unwrap();
-        docs.extend(transted_docs)
+        docs.extend(translated_docs)
     }
     init_translations_from_static_docs();
     STATIC_DOC_PAGE_FILES_BY_PATH
diff --git a/crates/rari-doc/src/pages/page.rs b/crates/rari-doc/src/pages/page.rs
@@ -42,13 +42,26 @@ pub enum PageCategory {
 
 impl Page {
     pub fn from_url(url: &str) -> Result<Self, DocError> {
-        Self::from_url_with_other_locale_and_fallback(url, None)
+        Self::internal_from_url_with_other_locale_and_fallback(url, None, true)
+    }
+
+    pub fn from_url_no_fallback(url: &str) -> Result<Self, DocError> {
+        Self::internal_from_url_with_other_locale_and_fallback(url, None, false)
     }
 
     pub fn from_url_with_other_locale_and_fallback(
         url: &str,
         locale: Option<Locale>,
     ) -> Result<Self, DocError> {
+        Self::internal_from_url_with_other_locale_and_fallback(url, locale, true)
+    }
+
+    fn internal_from_url_with_other_locale_and_fallback(
+        url: &str,
+        locale: Option<Locale>,
+        fallback: bool,
+    ) -> Result<Self, DocError> {
+        let url = &url[..url.find('#').unwrap_or(url.len())];
         let UrlMeta {
             folder_path,
             slug,
@@ -61,7 +74,7 @@ impl Page {
                 .ok_or(DocError::PageNotFound(url.to_string(), PageCategory::SPA)),
             PageCategory::Doc => {
                 let doc = Doc::page_from_slug_path(&folder_path, locale);
-                if doc.is_err() && locale != Default::default() {
+                if doc.is_err() && locale != Default::default() && fallback {
                     Doc::page_from_slug_path(&folder_path, Default::default())
                 } else {
                     doc
diff --git a/crates/rari-doc/src/pages/types/doc.rs b/crates/rari-doc/src/pages/types/doc.rs
@@ -35,6 +35,10 @@ use crate::utils::{
  ]
 */
 
+fn is_page_type_none(page_type: &PageType) -> bool {
+    matches!(page_type, PageType::None)
+}
+
 #[derive(Deserialize, Serialize, Clone, Debug, Default, Validate)]
 #[serde(default)]
 pub struct FrontMatter {
@@ -46,7 +50,7 @@ pub struct FrontMatter {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub tags: Vec<String>,
     pub slug: String,
-    #[serde(rename = "page-type")]
+    #[serde(rename = "page-type", skip_serializing_if = "is_page_type_none")]
     pub page_type: PageType,
     #[serde(
         deserialize_with = "t_or_vec",
diff --git a/crates/rari-doc/src/walker.rs b/crates/rari-doc/src/walker.rs
@@ -2,7 +2,7 @@ use std::path::Path;
 
 use ignore::types::TypesBuilder;
 use ignore::WalkBuilder;
-use rari_types::globals::{content_root, content_translated_root};
+use rari_types::globals::{content_root, content_translated_root, settings};
 
 pub fn walk_builder(
     paths: &[impl AsRef<Path>],
@@ -25,6 +25,7 @@ pub fn walk_builder(
         }
         builder
     };
+    builder.git_ignore(!settings().reader_ignores_gitignore);
     builder.types(types.build()?);
     Ok(builder)
 }
diff --git a/crates/rari-tools/Cargo.toml b/crates/rari-tools/Cargo.toml
@@ -23,6 +23,7 @@ rayon.workspace = true
 console = "0"
 dialoguer = "0"
 csv = "1"
+sha2 = "0.10"
 
 [dev-dependencies]
 serial_test = { version = "3", features = ["file_locks"] }
diff --git a/crates/rari-tools/src/add_redirect.rs b/crates/rari-tools/src/add_redirect.rs
@@ -86,7 +86,7 @@ mod test {
     use super::*;
     use crate::tests::fixtures::docs::DocFixtures;
     use crate::tests::fixtures::redirects::RedirectFixtures;
-    use crate::utils::test_utils::get_redirects_map;
+    use crate::utils::get_redirects_map;
 
     #[test]
     fn test_add_redirect() {
diff --git a/crates/rari-tools/src/error.rs b/crates/rari-tools/src/error.rs
@@ -12,6 +12,10 @@ pub enum ToolError {
     InvalidSlug(Cow<'static, str>),
     #[error("Invalid url: {0}")]
     InvalidUrl(Cow<'static, str>),
+    #[error("Invalid locale: {0}")]
+    InvalidLocale(Cow<'static, str>),
+    #[error("Orphaned doc exists: {0}")]
+    OrphanedDocExists(Cow<'static, str>),
     #[error("Git error: {0}")]
     GitError(String),
 
diff --git a/crates/rari-tools/src/lib.rs b/crates/rari-tools/src/lib.rs
@@ -6,6 +6,7 @@ pub mod r#move;
 pub mod popularities;
 pub mod redirects;
 pub mod remove;
+pub mod sync_translated_content;
 #[cfg(test)]
 pub mod tests;
 mod utils;
diff --git a/crates/rari-tools/src/move.rs b/crates/rari-tools/src/move.rs
@@ -2,7 +2,6 @@ use std::borrow::Cow;
 use std::ffi::OsStr;
 use std::fs::create_dir_all;
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::sync::Arc;
 
 use console::{style, Style};
@@ -25,15 +24,11 @@ use crate::wikihistory::update_wiki_history;
 pub fn r#move(
     old_slug: &str,
     new_slug: &str,
-    locale: Option<&str>,
+    locale: Option<Locale>,
     assume_yes: bool,
 ) -> Result<(), ToolError> {
     validate_args(old_slug, new_slug)?;
-    let locale = if let Some(l) = locale {
-        Locale::from_str(l)?
-    } else {
-        Locale::default()
-    };
+    let locale = locale.unwrap_or_default();
 
     // Make a dry run to give some feedback on what would be done
     let green = Style::new().green();
@@ -244,7 +239,8 @@ mod test {
     use crate::tests::fixtures::docs::DocFixtures;
     use crate::tests::fixtures::redirects::RedirectFixtures;
     use crate::tests::fixtures::wikihistory::WikihistoryFixtures;
-    use crate::utils::test_utils::{check_file_existence, get_redirects_map};
+    use crate::utils::get_redirects_map;
+    use crate::utils::test_utils::check_file_existence;
 
     fn s(s: &str) -> String {
         s.to_string()
@@ -354,7 +350,6 @@ mod test {
             Locale::EnUs,
             false,
         );
-        println!("result: {:?}", result);
         assert!(result.is_ok());
         let result = result.unwrap();
         assert!(result.len() == 3);
diff --git a/crates/rari-tools/src/redirects.rs b/crates/rari-tools/src/redirects.rs
@@ -3,7 +3,7 @@ use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fs::File;
 use std::io::{self, BufRead, BufWriter, Write};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
 use rari_doc::pages::page::{Page, PageLike};
@@ -212,10 +212,7 @@ pub fn add_redirects(locale: Locale, update_pairs: &[(String, String)]) -> Resul
 
     // Read the redirects file for the locale and populate the map.
     let mut pairs = HashMap::new();
-    let path = root_for_locale(locale)?
-        .to_path_buf()
-        .join(locale.as_folder_str())
-        .join("_redirects.txt");
+    let path = redirects_path(locale)?;
 
     if let Err(e) = read_redirects_raw(&path, &mut pairs) {
         error!("Error reading redirects: {e}");
@@ -245,6 +242,12 @@ pub fn add_redirects(locale: Locale, update_pairs: &[(String, String)]) -> Resul
     Ok(())
 }
 
+/// Gets the path to the redirects file for a specific locale.
+pub(crate) fn redirects_path(locale: Locale) -> Result<PathBuf, ToolError> {
+    let root = root_for_locale(locale)?;
+    Ok(root.join(locale.as_folder_str()).join("_redirects.txt"))
+}
+
 /// Validates a list of redirect pairs.
 ///
 /// Iterates through each `(from, to)` pair and validates both URLs based on the locale.
@@ -318,8 +321,8 @@ fn validate_from_url(url: &str, locale: Locale) -> Result<(), ToolError> {
 
     check_url_invalid_symbols(&url)?;
 
-    // Check for existing file/folder, commented for now
-    if let Ok(page) = Page::from_url(&url) {
+    // Check for existing file/folder.
+    if let Ok(page) = Page::from_url_no_fallback(&url) {
         return Err(ToolError::InvalidRedirectFromURL(format!(
             "From-URL '{}' resolves to an existing folder at '{}'.",
             url,
@@ -857,16 +860,6 @@ mod tests {
         assert!(result.is_err());
     }
 
-    #[test]
-    fn test_validate_to_url_diff_locale() {
-        let slugs = vec!["A".to_string()];
-        let _docs = DocFixtures::new(&slugs, Locale::EnUs);
-        let url = "/en-US/docs/A";
-        let result = validate_to_url(url);
-        println!("{:?}", result);
-        assert!(result.is_ok());
-    }
-
     #[test]
     fn test_validate_from_url_happy_path() {
         let url = "/en-US/docs/A";
diff --git a/crates/rari-tools/src/remove.rs b/crates/rari-tools/src/remove.rs
@@ -2,7 +2,6 @@ use std::borrow::Cow;
 use std::collections::BTreeSet;
 use std::ffi::OsStr;
 use std::path::PathBuf;
-use std::str::FromStr;
 
 use console::Style;
 use dialoguer::theme::ColorfulTheme;
@@ -24,17 +23,13 @@ use crate::wikihistory::delete_from_wiki_history;
 
 pub fn remove(
     slug: &str,
-    locale: Option<&str>,
+    locale: Option<Locale>,
     recursive: bool,
     redirect: Option<&str>,
     assume_yes: bool,
 ) -> Result<(), ToolError> {
     validate_args(slug)?;
-    let locale = if let Some(l) = locale {
-        Locale::from_str(l)?
-    } else {
-        Locale::default()
-    };
+    let locale = locale.unwrap_or_default();
 
     let green = Style::new().green();
     let red = Style::new().red();
@@ -270,7 +265,8 @@ mod test {
     use crate::tests::fixtures::docs::DocFixtures;
     use crate::tests::fixtures::redirects::RedirectFixtures;
     use crate::tests::fixtures::wikihistory::WikihistoryFixtures;
-    use crate::utils::test_utils::{check_file_existence, get_redirects_map};
+    use crate::utils::get_redirects_map;
+    use crate::utils::test_utils::check_file_existence;
     use crate::wikihistory::test_get_wiki_history;
 
     #[test]
diff --git a/crates/rari-tools/src/sync_translated_content.rs b/crates/rari-tools/src/sync_translated_content.rs
diff --git a/crates/rari-tools/src/utils.rs b/crates/rari-tools/src/utils.rs
diff --git a/crates/rari-tools/src/wikihistory.rs b/crates/rari-tools/src/wikihistory.rs
diff --git a/crates/rari-types/src/locale.rs b/crates/rari-types/src/locale.rs
diff --git a/crates/rari-types/src/settings.rs b/crates/rari-types/src/settings.rs

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@ use std::path::Path;`
`2`	`2`
`3`	`3`	`use ignore::types::TypesBuilder;`
`4`	`4`	`use ignore::WalkBuilder;`
`5`		`-use rari_types::globals::{content_root, content_translated_root};`
	`5`	`+use rari_types::globals::{content_root, content_translated_root, settings};`
`6`	`6`
`7`	`7`	`pub fn walk_builder(`
`8`	`8`	`paths: &[impl AsRef<Path>],`
`@@ -25,6 +25,7 @@ pub fn walk_builder(`
`25`	`25`	`}`
`26`	`26`	`builder`
`27`	`27`	`};`
	`28`	`+ builder.git_ignore(!settings().reader_ignores_gitignore);`
`28`	`29`	`builder.types(types.build()?);`
`29`	`30`	`Ok(builder)`
`30`	`31`	`}`