From 5aab33ec6dd0d2eaa0047c33b4de0dbd04b6a3bf Mon Sep 17 00:00:00 2001 From: Tobias Bieniek Date: Thu, 9 Mar 2023 12:03:17 +0100 Subject: [PATCH] Add `FixFeature2` background worker job This job "fixes" the `features2` entries by moving all `features` into `feature2` if `feature2` already has content. This should avoid a mismatch of `features` pointing to `features2` for older cargo versions. --- src/admin/enqueue_job.rs | 5 +++ src/background_jobs.rs | 11 ++++++ src/worker/git.rs | 75 +++++++++++++++++++++++++++++++++++++++- src/worker/mod.rs | 6 ++-- 4 files changed, 93 insertions(+), 4 deletions(-) diff --git a/src/admin/enqueue_job.rs b/src/admin/enqueue_job.rs index a0a4275284f..2cb0d8fdc0c 100644 --- a/src/admin/enqueue_job.rs +++ b/src/admin/enqueue_job.rs @@ -23,6 +23,10 @@ pub enum Command { #[arg(long = "dry-run")] dry_run: bool, }, + FixFeatures2 { + #[arg(long = "dry-run")] + dry_run: bool, + }, } pub fn run(command: Command) -> Result<()> { @@ -51,5 +55,6 @@ pub fn run(command: Command) -> Result<()> { Command::DailyDbMaintenance => Ok(worker::daily_db_maintenance().enqueue(conn)?), Command::SquashIndex => Ok(worker::squash_index().enqueue(conn)?), Command::NormalizeIndex { dry_run } => Ok(worker::normalize_index(dry_run).enqueue(conn)?), + Command::FixFeatures2 { dry_run } => Ok(worker::fix_features2(dry_run).enqueue(conn)?), } } diff --git a/src/background_jobs.rs b/src/background_jobs.rs index 3e09ed14a10..a8f26422abe 100644 --- a/src/background_jobs.rs +++ b/src/background_jobs.rs @@ -15,6 +15,7 @@ use cargo_registry_index::Repository; pub enum Job { DailyDbMaintenance, DumpDb(DumpDbJob), + FixFeatures2(FixFeatures2Job), IndexAddCrate(IndexAddCrateJob), IndexSquash, IndexSyncToHttp(IndexSyncToHttpJob), @@ -41,6 +42,7 @@ pub(crate) struct PerformState<'a> { impl Job { const DAILY_DB_MAINTENANCE: &str = "daily_db_maintenance"; const DUMP_DB: &str = "dump_db"; + const FIX_FEATURES2: &str = "fix_features2"; const INDEX_ADD_CRATE: &str = "add_crate"; const INDEX_SQUASH: &str = "squash_index"; const INDEX_SYNC_TO_HTTP: &str = "update_crate_index"; @@ -53,6 +55,7 @@ impl Job { match self { Job::DailyDbMaintenance => Self::DAILY_DB_MAINTENANCE, Job::DumpDb(_) => Self::DUMP_DB, + Job::FixFeatures2(_) => Self::FIX_FEATURES2, Job::IndexAddCrate(_) => Self::INDEX_ADD_CRATE, Job::IndexSquash => Self::INDEX_SQUASH, Job::IndexSyncToHttp(_) => Self::INDEX_SYNC_TO_HTTP, @@ -67,6 +70,7 @@ impl Job { match self { Job::DailyDbMaintenance => Ok(serde_json::Value::Null), Job::DumpDb(inner) => serde_json::to_value(inner), + Job::FixFeatures2(inner) => serde_json::to_value(inner), Job::IndexAddCrate(inner) => serde_json::to_value(inner), Job::IndexSquash => Ok(serde_json::Value::Null), Job::IndexSyncToHttp(inner) => serde_json::to_value(inner), @@ -95,6 +99,7 @@ impl Job { Ok(match job_type { Self::DAILY_DB_MAINTENANCE => Job::DailyDbMaintenance, Self::DUMP_DB => Job::DumpDb(from_value(value)?), + Self::FIX_FEATURES2 => Job::FixFeatures2(from_value(value)?), Self::INDEX_ADD_CRATE => Job::IndexAddCrate(from_value(value)?), Self::INDEX_SQUASH => Job::IndexSquash, Self::INDEX_SYNC_TO_HTTP => Job::IndexSyncToHttp(from_value(value)?), @@ -120,6 +125,7 @@ impl Job { worker::perform_daily_db_maintenance(&mut *fresh_connection(pool)?) } Job::DumpDb(args) => worker::perform_dump_db(env, args.database_url, args.target_name), + Job::FixFeatures2(args) => worker::perform_fix_features2(env, args), Job::IndexAddCrate(args) => worker::perform_index_add_crate(env, conn, &args.krate), Job::IndexSquash => worker::perform_index_squash(env), Job::IndexSyncToHttp(args) => worker::perform_index_sync_to_http(env, args.crate_name), @@ -183,6 +189,11 @@ pub struct NormalizeIndexJob { pub dry_run: bool, } +#[derive(Serialize, Deserialize)] +pub struct FixFeatures2Job { + pub dry_run: bool, +} + #[derive(Serialize, Deserialize)] pub struct RenderAndUploadReadmeJob { pub(super) version_id: i32, diff --git a/src/worker/git.rs b/src/worker/git.rs index f9b80cbd764..458cc2cf8c4 100644 --- a/src/worker/git.rs +++ b/src/worker/git.rs @@ -1,5 +1,6 @@ use crate::background_jobs::{ - Environment, IndexAddCrateJob, IndexSyncToHttpJob, IndexUpdateYankedJob, Job, NormalizeIndexJob, + Environment, FixFeatures2Job, IndexAddCrateJob, IndexSyncToHttpJob, IndexUpdateYankedJob, Job, + NormalizeIndexJob, }; use crate::schema; use crate::swirl::PerformError; @@ -252,3 +253,75 @@ pub fn perform_normalize_index( pub fn normalize_index(dry_run: bool) -> Job { Job::NormalizeIndex(NormalizeIndexJob { dry_run }) } + +pub fn perform_fix_features2(env: &Environment, args: FixFeatures2Job) -> Result<(), PerformError> { + info!("Fixing `features2` usage in the index"); + + let repo = env.lock_index()?; + + let files = repo.get_files_modified_since(None)?; + let num_files = files.len(); + + for (i, file) in files.iter().enumerate() { + if i % 50 == 0 { + info!(num_files, i, ?file); + } + + let crate_name = file.file_name().unwrap().to_str().unwrap(); + let path = repo.index_file(crate_name); + if !path.exists() { + continue; + } + + let file = fs::File::open(&path)?; + let reader = BufReader::new(file); + let mut versions = Vec::new(); + let mut changed = false; + for line in reader.lines() { + let line = line?; + if line.is_empty() { + continue; + } + + let mut krate: Crate = serde_json::from_str(&line)?; + if !krate.features.is_empty() { + if let Some(features2) = krate.features2.as_mut() { + features2.extend(std::mem::take(&mut krate.features)); + changed = true; + } + } + + versions.push(krate); + } + + if changed { + let mut body: Vec = Vec::new(); + for version in versions { + serde_json::to_writer(&mut body, &version).unwrap(); + body.push(b'\n'); + } + fs::write(path, body)?; + } + } + + info!("Committing `features2` fix"); + let msg = "Fix `features2` usage\n\n\ + More information can be found at https://github.com/rust-lang/crates.io/issues/6135"; + repo.run_command(Command::new("git").args(["commit", "-am", msg]))?; + + let branch = match args.dry_run { + false => "master", + true => "features2-dry-run", + }; + + info!(?branch, "Pushing to upstream repository"); + repo.run_command(Command::new("git").args(["push", "origin", &format!("HEAD:{branch}")]))?; + + info!("`features2` fix completed"); + + Ok(()) +} + +pub fn fix_features2(dry_run: bool) -> Job { + Job::FixFeatures2(FixFeatures2Job { dry_run }) +} diff --git a/src/worker/mod.rs b/src/worker/mod.rs index 51f6eb3f9a0..a3d325cdf9e 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -12,15 +12,15 @@ mod update_downloads; pub use daily_db_maintenance::daily_db_maintenance; pub use dump_db::dump_db; -pub use git::{add_crate, normalize_index, squash_index, sync_yanked}; +pub use git::{add_crate, fix_features2, normalize_index, squash_index, sync_yanked}; pub use readmes::render_and_upload_readme; pub use update_downloads::update_downloads; pub(crate) use daily_db_maintenance::perform_daily_db_maintenance; pub(crate) use dump_db::perform_dump_db; pub(crate) use git::{ - perform_index_add_crate, perform_index_squash, perform_index_sync_to_http, - perform_index_update_yanked, perform_normalize_index, + perform_fix_features2, perform_index_add_crate, perform_index_squash, + perform_index_sync_to_http, perform_index_update_yanked, perform_normalize_index, }; pub(crate) use readmes::perform_render_and_upload_readme; pub(crate) use update_downloads::perform_update_downloads;