From ffe8a033692356d14c4b5073e686847959cb88a0 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Mon, 9 Feb 2026 01:18:37 +0100 Subject: [PATCH] gracefully handle 404 errors from crates.io --- Cargo.lock | 4 + .../src/docbuilder/rustwide_builder.rs | 2 +- .../bin/docs_rs_import_release/src/import.rs | 6 +- crates/lib/docs_rs_registry_api/Cargo.toml | 7 + crates/lib/docs_rs_registry_api/src/api.rs | 131 ++++++++++++++---- crates/lib/docs_rs_registry_api/src/models.rs | 2 +- 6 files changed, 119 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9d8c6bb5..c652881c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2245,9 +2245,13 @@ dependencies = [ "docs_rs_env_vars", "docs_rs_types", "docs_rs_utils", + "mime", + "mockito", "reqwest 0.13.2", "serde", + "serde_json", "sqlx", + "tokio", "tracing", "url", ] diff --git a/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs b/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs index 0de95fbfe..75e49cb09 100644 --- a/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs +++ b/crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs @@ -786,7 +786,7 @@ impl RustwideBuilder { .runtime .block_on(self.registry_api.get_release_data(name, version)) { - Ok(data) => Some(data), + Ok(data) => data, Err(err) => { error!(%name, %version, ?err, "could not fetch releases-data"); None diff --git a/crates/bin/docs_rs_import_release/src/import.rs b/crates/bin/docs_rs_import_release/src/import.rs index dbf90a921..46f2b6be6 100644 --- a/crates/bin/docs_rs_import_release/src/import.rs +++ b/crates/bin/docs_rs_import_release/src/import.rs @@ -4,6 +4,7 @@ use crate::{ rustdoc::{download_static_files, find_static_paths, find_successful_build_targets}, rustdoc_status::fetch_rustdoc_status, }; +use anyhow::anyhow; use anyhow::{Result, bail}; use docs_rs_cargo_metadata::CargoMetadata; use docs_rs_database::releases::{ @@ -117,7 +118,10 @@ async fn import_test_release_inner( (files_list, source_size) }; - let registry_data = registry_api.get_release_data(name, version).await?; + let registry_data = registry_api + .get_release_data(name, version) + .await? + .ok_or_else(|| anyhow!("registry data not found"))?; let rustdoc_dir = { info!("download & extract rustdoc archive..."); diff --git a/crates/lib/docs_rs_registry_api/Cargo.toml b/crates/lib/docs_rs_registry_api/Cargo.toml index dd80266be..42efc2a00 100644 --- a/crates/lib/docs_rs_registry_api/Cargo.toml +++ b/crates/lib/docs_rs_registry_api/Cargo.toml @@ -18,3 +18,10 @@ serde = { workspace = true } sqlx = { workspace = true } tracing = { workspace = true } url = { workspace = true } + +[dev-dependencies] +docs_rs_types = { path = "../docs_rs_types", features = ["testing"] } +mime = { workspace = true } +mockito = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } diff --git a/crates/lib/docs_rs_registry_api/src/api.rs b/crates/lib/docs_rs_registry_api/src/api.rs index 1a32a8737..2528db0bf 100644 --- a/crates/lib/docs_rs_registry_api/src/api.rs +++ b/crates/lib/docs_rs_registry_api/src/api.rs @@ -6,7 +6,10 @@ use anyhow::{Context, Result, anyhow, bail}; use chrono::{DateTime, Utc}; use docs_rs_types::{KrateName, Version}; use docs_rs_utils::{APP_USER_AGENT, retry_async}; -use reqwest::header::{ACCEPT, HeaderValue, USER_AGENT}; +use reqwest::{ + StatusCode, + header::{ACCEPT, HeaderValue, USER_AGENT}, +}; use serde::Deserialize; use tracing::instrument; use url::Url; @@ -60,25 +63,7 @@ impl RegistryApi { &self, name: &KrateName, version: &Version, - ) -> Result { - let (release_time, yanked, downloads) = self - .get_release_time_yanked_downloads(name, version) - .await - .context(format!("Failed to get crate data for {name}-{version}"))?; - - Ok(ReleaseData { - release_time, - yanked, - downloads, - }) - } - - /// Get release_time, yanked and downloads from the registry's API - async fn get_release_time_yanked_downloads( - &self, - name: &KrateName, - version: &Version, - ) -> Result<(DateTime, bool, i32)> { + ) -> Result> { let url = { let mut url = self.api_base.clone(); url.path_segments_mut() @@ -103,20 +88,31 @@ impl RegistryApi { downloads: i32, } - let response: Response = retry_async( + let response: Response = match retry_async( || async { - Ok(self - .client - .get(url.clone()) - .send() - .await? - .error_for_status()?) + Ok( + match self + .client + .get(url.clone()) + .send() + .await? + .error_for_status() + { + Ok(resp) => Some(resp), + Err(err) if matches!(err.status(), Some(StatusCode::NOT_FOUND)) => None, + Err(err) => return Err(err.into()), + }, + ) }, self.max_retries, ) .await? - .json() - .await?; + { + Some(resp) => resp.json().await?, + None => { + return Ok(None); + } + }; let version = response .versions @@ -124,7 +120,11 @@ impl RegistryApi { .find(|data| data.num == *version) .with_context(|| anyhow!("Could not find version in response"))?; - Ok((version.created_at, version.yanked, version.downloads)) + Ok(Some(ReleaseData { + release_time: version.created_at, + yanked: version.yanked, + downloads: version.downloads, + })) } /// Fetch owners from the registry's API @@ -241,3 +241,74 @@ impl RegistryApi { Ok(Search { crates, meta }) } } + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + use docs_rs_types::testing::{KRATE, V1, V2}; + use reqwest::header::CONTENT_TYPE; + + #[tokio::test] + async fn test_get_release_data() -> Result<()> { + let mut server = mockito::Server::new_async().await; + + let created = Utc::now(); + + let _m = server + .mock("GET", "/api/v1/crates/krate/versions") + .with_status(200) + .with_header(CONTENT_TYPE, mime::APPLICATION_JSON.as_ref()) + .with_body( + serde_json::json!({ + "versions": [ + { + "num": V1.to_string(), + "created_at": created.to_rfc3339(), + "yanked": false, + "downloads": 42 + }, + { + "num": V2.to_string(), + "created_at": "2025-01-01T00:00:00Z", + "yanked": true, + "downloads": 22 + } + ] + }) + .to_string(), + ) + .create_async() + .await; + + let api = RegistryApi::new(server.url().parse().unwrap(), 0)?; + + assert_eq!( + api.get_release_data(&KRATE, &V1).await?, + Some(ReleaseData { + release_time: created, + yanked: false, + downloads: 42 + }) + ); + + Ok(()) + } + + #[tokio::test] + async fn test_404_in_release_data_returns_none() -> Result<()> { + let mut server = mockito::Server::new_async().await; + + let _m = server + .mock("GET", "/api/v1/crates/krate/versions") + .with_status(404) + .create_async() + .await; + + let api = RegistryApi::new(server.url().parse().unwrap(), 0)?; + + assert_eq!(api.get_release_data(&KRATE, &V1).await?, None,); + + Ok(()) + } +} diff --git a/crates/lib/docs_rs_registry_api/src/models.rs b/crates/lib/docs_rs_registry_api/src/models.rs index 89cfc4e8e..3d7ce0bdf 100644 --- a/crates/lib/docs_rs_registry_api/src/models.rs +++ b/crates/lib/docs_rs_registry_api/src/models.rs @@ -7,7 +7,7 @@ pub struct CrateData { pub owners: Vec, } -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct ReleaseData { pub release_time: DateTime, pub yanked: bool,