Add rust_version field to the index

The rust_version field is read from the uploaded tarball, breaking from
convention.

Author: cassaundra

cargo-registry-index/lib.rs

@@ -125,6 +125,8 @@ pub struct Crate {
     pub yanked: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub links: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rust_version: Option<String>,
     /// The schema version for this entry.
     ///
     /// If this is None, it defaults to version 1. Entries with unknown

migrations/2023-03-23-211241_add_rust_version_to_versions/down.sql

@@ -0,0 +1 @@
+ALTER TABLE versions DROP COLUMN rust_version;

migrations/2023-03-23-211241_add_rust_version_to_versions/up.sql

@@ -0,0 +1 @@
+ALTER TABLE versions ADD COLUMN rust_version VARCHAR;

src/admin/render_readmes.rs

@@ -3,6 +3,7 @@ use crate::{
     models::Version,
     schema::{crates, readme_renderings, versions},
     uploaders::Uploader,
+    util::{find_file_by_path, manifest::Manifest},
 };
 use anyhow::{anyhow, Context};
 use std::{io::Read, path::Path, sync::Arc, thread};
@@ -200,38 +201,7 @@ fn render_pkg_readme<R: Read>(mut archive: Archive<R>, pkg_name: &str) -> anyhow
             pkg_path_in_vcs,
         )
     };
-    return Ok(rendered);
-
-    #[derive(Debug, Deserialize)]
-    struct Package {
-        readme: Option<String>,
-        repository: Option<String>,
-    }
-
-    #[derive(Debug, Deserialize)]
-    struct Manifest {
-        package: Package,
-    }
-}
-
-/// Search an entry by its path in a Tar archive.
-fn find_file_by_path<R: Read>(
-    entries: &mut tar::Entries<'_, R>,
-    path: &Path,
-) -> anyhow::Result<String> {
-    let mut file = entries
-        .filter_map(|entry| entry.ok())
-        .find(|file| match file.path() {
-            Ok(p) => p == path,
-            Err(_) => false,
-        })
-        .ok_or_else(|| anyhow!("Failed to find tarball entry: {}", path.display()))?;
-
-    let mut contents = String::new();
-    file.read_to_string(&mut contents)
-        .context("Failed to read file contents")?;
-
-    Ok(contents)
+    Ok(rendered)
 }
 
 #[cfg(test)]

src/controllers/krate/publish.rs

@@ -1,15 +1,13 @@
 //! Functionality related to publishing a new crate or version of a crate.
 
-use crate::auth::AuthCheck;
-use crate::background_jobs::Job;
 use axum::body::Bytes;
 use flate2::read::GzDecoder;
 use hex::ToHex;
 use hyper::body::Buf;
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
 use std::io::Read;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use crate::controllers::cargo_prelude::*;
 use crate::controllers::util::RequestPartsExt;
@@ -18,10 +16,13 @@ use crate::models::{
     Rights, VersionAction,
 };
 
+use crate::auth::AuthCheck;
+use crate::background_jobs::Job;
 use crate::middleware::log_request::RequestLogExt;
 use crate::models::token::EndpointScope;
 use crate::schema::*;
 use crate::util::errors::{cargo_err, AppResult};
+use crate::util::manifest::Manifest;
 use crate::util::{CargoVcsInfo, LimitErrorReader, Maximums};
 use crate::views::{
     EncodableCrate, EncodableCrateDependency, EncodableCrateUpload, GoodCrate, PublishWarnings,
@@ -188,6 +189,13 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult<Json<GoodCra
             // Read tarball from request
             let hex_cksum: String = Sha256::digest(&tarball_bytes).encode_hex();
 
+            let pkg_name = format!("{}-{}", krate.name, vers);
+            let tarball_info = verify_tarball(&pkg_name, &tarball_bytes, maximums.max_unpack_size)?;
+
+            let rust_version = tarball_info
+                .manifest
+                .and_then(|m| m.package.rust_version.map(|rv| rv.0));
+
             // Persist the new version of this crate
             let version = NewVersion::new(
                 krate.id,
@@ -201,6 +209,7 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult<Json<GoodCra
                 user.id,
                 hex_cksum.clone(),
                 links.clone(),
+                rust_version.as_deref(),
             )?
             .save(conn, &verified_email_address)?;
 
@@ -224,10 +233,7 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult<Json<GoodCra
 
             let top_versions = krate.top_versions(conn)?;
 
-            let pkg_name = format!("{}-{}", krate.name, vers);
-            let cargo_vcs_info =
-                verify_tarball(&pkg_name, &tarball_bytes, maximums.max_unpack_size)?;
-            let pkg_path_in_vcs = cargo_vcs_info.map(|info| info.path_in_vcs);
+            let pkg_path_in_vcs = tarball_info.vcs_info.map(|info| info.path_in_vcs);
 
             if let Some(readme) = new_crate.readme {
                 Job::render_and_upload_readme(
@@ -269,6 +275,7 @@ pub async fn publish(app: AppState, req: BytesRequest) -> AppResult<Json<GoodCra
                 deps: git_deps,
                 yanked: Some(false),
                 links,
+                rust_version,
                 v,
             };
 
@@ -426,12 +433,14 @@ pub fn add_dependencies(
     Ok(git_deps)
 }
 
+#[derive(Debug)]
+struct TarballInfo {
+    manifest: Option<Manifest>,
+    vcs_info: Option<CargoVcsInfo>,
+}
+
 #[instrument(skip_all, fields(%pkg_name))]
-fn verify_tarball(
-    pkg_name: &str,
-    tarball: &[u8],
-    max_unpack: u64,
-) -> AppResult<Option<CargoVcsInfo>> {
+fn verify_tarball(pkg_name: &str, tarball: &[u8], max_unpack: u64) -> AppResult<TarballInfo> {
     // All our data is currently encoded with gzip
     let decoder = GzDecoder::new(tarball);
 
@@ -442,10 +451,15 @@ fn verify_tarball(
     // Use this I/O object now to take a peek inside
     let mut archive = tar::Archive::new(decoder);
 
+    let entries = archive.entries()?;
+
     let vcs_info_path = Path::new(&pkg_name).join(".cargo_vcs_info.json");
     let mut vcs_info = None;
 
-    for entry in archive.entries()? {
+    let manifest_path = PathBuf::from(format!("{pkg_name}/Cargo.toml"));
+    let mut manifest: Option<Manifest> = None;
+
+    for entry in entries {
         let mut entry = entry.map_err(|err| {
             err.chain(cargo_err(
                 "uploaded tarball is malformed or too large when decompressed",
@@ -467,6 +481,15 @@ fn verify_tarball(
             vcs_info = CargoVcsInfo::from_contents(&contents).ok();
         }
 
+        // Try to extract and read the Cargo.toml from the tarball, silently
+        // erroring if it cannot be read.
+        let entry_path = entry.path()?;
+        if entry_path == manifest_path {
+            let mut contents = String::new();
+            entry.read_to_string(&mut contents)?;
+            manifest = toml::from_str(&contents).ok();
+        }
+
         // Historical versions of the `tar` crate which Cargo uses internally
         // don't properly prevent hard links and symlinks from overwriting
         // arbitrary files on the filesystem. As a bit of a hammer we reject any
@@ -477,7 +500,8 @@ fn verify_tarball(
             return Err(cargo_err("invalid tarball uploaded"));
         }
     }
-    Ok(vcs_info)
+
+    Ok(TarballInfo { manifest, vcs_info })
 }
 
 #[cfg(test)]
@@ -505,7 +529,9 @@ mod tests {
 
         let limit = 512 * 1024 * 1024;
         assert_eq!(
-            verify_tarball("foo-0.0.1", &serialized_archive, limit).unwrap(),
+            verify_tarball("foo-0.0.1", &serialized_archive, limit)
+                .unwrap()
+                .vcs_info,
             None
         );
         assert_err!(verify_tarball("bar-0.0.1", &serialized_archive, limit));
@@ -527,6 +553,7 @@ mod tests {
         let limit = 512 * 1024 * 1024;
         let vcs_info = verify_tarball("foo-0.0.1", &serialized_archive, limit)
             .unwrap()
+            .vcs_info
             .unwrap();
         assert_eq!(vcs_info.path_in_vcs, "");
     }
@@ -547,7 +574,51 @@ mod tests {
         let limit = 512 * 1024 * 1024;
         let vcs_info = verify_tarball("foo-0.0.1", &serialized_archive, limit)
             .unwrap()
+            .vcs_info
             .unwrap();
         assert_eq!(vcs_info.path_in_vcs, "path/in/vcs");
     }
+
+    #[test]
+    fn verify_tarball_test_manifest() {
+        let mut pkg = tar::Builder::new(vec![]);
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/Cargo.toml",
+            br#"
+[package]
+rust_version = "1.59"
+readme = "README.md"
+repository = "https://github.com/foo/bar"
+"#,
+        );
+        let mut serialized_archive = vec![];
+        GzEncoder::new(pkg.into_inner().unwrap().as_slice(), Default::default())
+            .read_to_end(&mut serialized_archive)
+            .unwrap();
+
+        let limit = 512 * 1024 * 1024;
+        let manifest = verify_tarball("foo-0.0.1", &serialized_archive, limit)
+            .unwrap()
+            .manifest;
+        assert_eq!(
+            manifest
+                .as_ref()
+                .unwrap()
+                .package
+                .rust_version
+                .as_ref()
+                .unwrap()
+                .0,
+            "1.59".to_owned()
+        );
+        assert_eq!(
+            manifest.as_ref().unwrap().package.readme,
+            Some("README.md".to_owned())
+        );
+        assert_eq!(
+            manifest.as_ref().unwrap().package.repository,
+            Some("https://github.com/foo/bar".to_owned())
+        );
+    }
 }

src/downloads_counter.rs

@@ -459,6 +459,7 @@ mod tests {
                 self.user.id,
                 "0000000000000000000000000000000000000000000000000000000000000000".to_string(),
                 None,
+                None,
             )
             .expect("failed to create version")
             .save(conn, "[email protected]")

src/models/krate.rs

@@ -510,6 +510,7 @@ impl Crate {
                     deps,
                     features,
                     links: version.links,
+                    rust_version: version.rust_version,
                     features2,
                     v,
                 };

src/models/version.rs

@@ -25,6 +25,7 @@ pub struct Version {
     pub published_by: Option<i32>,
     pub checksum: String,
     pub links: Option<String>,
+    pub rust_version: Option<String>,
 }
 
 #[derive(Insertable, Debug)]
@@ -38,6 +39,7 @@ pub struct NewVersion {
     published_by: i32,
     checksum: String,
     links: Option<String>,
+    rust_version: Option<String>,
 }
 
 /// The highest version (semver order) and the most recently updated version.
@@ -139,6 +141,7 @@ impl NewVersion {
         published_by: i32,
         checksum: String,
         links: Option<String>,
+        rust_version: Option<&str>,
     ) -> AppResult<Self> {
         let features = serde_json::to_value(features)?;
 
@@ -151,6 +154,7 @@ impl NewVersion {
             published_by,
             checksum,
             links,
+            rust_version: rust_version.map(ToOwned::to_owned),
         };
 
         new_version.validate_license(license_file)?;

src/schema.rs

@@ -966,6 +966,12 @@ diesel::table! {
         ///
         /// (Automatically generated by Diesel.)
         links -> Nullable<Varchar>,
+        /// The `rust_version` column of the `versions` table.
+        ///
+        /// Its SQL type is `Nullable<Varchar>`.
+        ///
+        /// (Automatically generated by Diesel.)
+        rust_version -> Nullable<Varchar>,
     }
 }
 

src/tests/builders/version.rs

@@ -20,6 +20,7 @@ pub struct VersionBuilder<'a> {
     yanked: bool,
     checksum: String,
     links: Option<String>,
+    rust_version: Option<String>,
 }
 
 impl<'a> VersionBuilder<'a> {
@@ -45,6 +46,7 @@ impl<'a> VersionBuilder<'a> {
             yanked: false,
             checksum: String::new(),
             links: None,
+            rust_version: None,
         }
     }
 
@@ -83,6 +85,12 @@ impl<'a> VersionBuilder<'a> {
         self
     }
 
+    /// Sets the version's `rust_version` value.
+    pub fn rust_version(mut self, rust_version: &str) -> Self {
+        self.rust_version = Some(rust_version.to_owned());
+        self
+    }
+
     pub fn build(
         self,
         crate_id: i32,
@@ -103,6 +111,7 @@ impl<'a> VersionBuilder<'a> {
             published_by,
             self.checksum,
             self.links,
+            self.rust_version.as_deref(),
         )?
         .save(connection, "[email protected]")?;
 

src/tests/krate/versions.rs

@@ -1,4 +1,4 @@
-use crate::builders::CrateBuilder;
+use crate::builders::{CrateBuilder, VersionBuilder};
 use crate::util::{RequestHelper, TestApp};
 use cargo_registry::schema::versions;
 use cargo_registry::views::EncodableVersion;
@@ -16,7 +16,7 @@ fn versions() {
     app.db(|conn| {
         CrateBuilder::new("foo_versions", user.id)
             .version("0.5.1")
-            .version("1.0.0")
+            .version(VersionBuilder::new("1.0.0").rust_version("1.64"))
             .version("0.5.0")
             .expect_build(conn);
         // Make version 1.0.0 mimic a version published before we started recording who published
@@ -33,6 +33,7 @@ fn versions() {
 
     assert_eq!(json.versions.len(), 3);
     assert_eq!(json.versions[0].num, "1.0.0");
+    assert_eq!(json.versions[0].rust_version, Some("1.64".to_owned()));
     assert_eq!(json.versions[1].num, "0.5.1");
     assert_eq!(json.versions[2].num, "0.5.0");
     assert_none!(&json.versions[0].published_by);

src/tests/routes/crates/versions/read.rs

@@ -14,6 +14,7 @@ fn show_by_crate_name_and_version() {
         VersionBuilder::new("2.0.0")
             .size(1234)
             .checksum("c241cd77c3723ccf1aa453f169ee60c0a888344da504bee0142adb859092acb4")
+            .rust_version("1.64")
             .expect_build(krate.id, user.id, conn)
     });