controllers::krate::publish: Extract cargo vcs info from tarball

Supported in rust-lang/cargo#9866

Author: nipunn1313

src/admin/render_readmes.rs

@@ -3,6 +3,7 @@ use crate::{
     models::Version,
     schema::{crates, readme_renderings, versions},
     uploaders::Uploader,
+    util::CargoVcsInfo,
 };
 use std::{collections::HashMap, ffi::OsString, io::Read, sync::Arc, thread};
 
@@ -212,6 +213,15 @@ fn render_pkg_readme<R: Read>(mut pkg: Archive<R>, pkg_name: &str) -> Option<Str
         })
         .collect();
 
+    let pkg_path_in_vcs = {
+        let path = format!("{}/.cargo_vcs_info.json", pkg_name);
+        entries.get(&OsString::from(path)).map(|contents| {
+            CargoVcsInfo::from_contents(contents)
+                .map(|info| info.path_in_vcs)
+                .unwrap_or_else(|e| panic!("[{}] invalid .cargo_vcs_info.json: {}", pkg_name, e))
+        })
+    };
+
     let manifest: Manifest = {
         let path = format!("{}/Cargo.toml", pkg_name);
         let contents = entries
@@ -242,12 +252,11 @@ fn render_pkg_readme<R: Read>(mut pkg: Archive<R>, pkg_name: &str) -> Option<Str
 
         let path = format!("{}/{}", pkg_name, readme_path);
         let contents = entries.get(&OsString::from(path))?;
-        let pkg_path_in_vcs = None;
         text_to_html(
             contents,
             readme_path,
             manifest.package.repository.as_deref(),
-            pkg_path_in_vcs,
+            pkg_path_in_vcs.as_deref(),
         )
     };
     return Some(rendered);
@@ -441,4 +450,60 @@ repository = "https://github.com/foo/foo"
         assert!(result.contains("docs/readme"));
         assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/docs/./Other.md\""))
     }
+
+    #[test]
+    fn test_render_pkg_pkg_not_at_root_of_repo() {
+        let mut pkg = tar::Builder::new(vec![]);
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/Cargo.toml",
+            br#"
+[package]
+readme = "docs/README.md"
+repository = "https://github.com/foo/foo"
+"#,
+        );
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/docs/README.md",
+            b"docs/readme [link](./Other.md)",
+        );
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/.cargo_vcs_info.json",
+            br#"{"path_in_vcs": "path/in/vcs"}"#,
+        );
+        let serialized_archive = pkg.into_inner().unwrap();
+        let result =
+            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
+        assert!(result.contains("docs/readme"));
+        assert!(
+            result.contains("\"https://github.com/foo/foo/blob/HEAD/path/in/vcs/docs/./Other.md\"")
+        )
+    }
+
+    #[test]
+    fn test_render_pkg_blank_vcs_info() {
+        let mut pkg = tar::Builder::new(vec![]);
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/Cargo.toml",
+            br#"
+[package]
+readme = "docs/README.md"
+repository = "https://github.com/foo/foo"
+"#,
+        );
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/docs/README.md",
+            b"docs/readme [link](./Other.md)",
+        );
+        add_file(&mut pkg, "foo-0.0.1/.cargo_vcs_info.json", br#"{}"#);
+        let serialized_archive = pkg.into_inner().unwrap();
+        let result =
+            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
+        assert!(result.contains("docs/readme"));
+        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/docs/./Other.md\""))
+    }
 }

src/controllers/krate/publish.rs

@@ -4,6 +4,7 @@ use flate2::read::GzDecoder;
 use hex::ToHex;
 use sha2::{Digest, Sha256};
 use std::io::Read;
+use std::path::Path;
 use std::sync::Arc;
 use swirl::Job;
 
@@ -17,7 +18,7 @@ use crate::models::{
 use crate::render;
 use crate::schema::*;
 use crate::util::errors::{cargo_err, AppResult};
-use crate::util::{read_fill, read_le_u32, LimitErrorReader, Maximums};
+use crate::util::{read_fill, read_le_u32, CargoVcsInfo, LimitErrorReader, Maximums};
 use crate::views::{
     EncodableCrate, EncodableCrateDependency, EncodableCrateUpload, GoodCrate, PublishWarnings,
 };
@@ -193,9 +194,8 @@ pub fn publish(req: &mut dyn RequestExt) -> EndpointResult {
         LimitErrorReader::new(req.body(), maximums.max_upload_size).read_to_end(&mut tarball)?;
         let hex_cksum: String = Sha256::digest(&tarball).encode_hex();
         let pkg_name = format!("{}-{}", krate.name, vers);
-        verify_tarball(&pkg_name, &tarball, maximums.max_unpack_size)?;
-
-        let pkg_path_in_vcs = None;
+        let cargo_vcs_info = verify_tarball(&pkg_name, &tarball, maximums.max_unpack_size)?;
+        let pkg_path_in_vcs = cargo_vcs_info.map(|info| info.path_in_vcs);
 
         if let Some(readme) = new_crate.readme {
             render::render_and_upload_readme(
@@ -366,7 +366,11 @@ pub fn add_dependencies(
     Ok(git_deps)
 }
 
-fn verify_tarball(pkg_name: &str, tarball: &[u8], max_unpack: u64) -> AppResult<()> {
+fn verify_tarball(
+    pkg_name: &str,
+    tarball: &[u8],
+    max_unpack: u64,
+) -> AppResult<Option<CargoVcsInfo>> {
     // All our data is currently encoded with gzip
     let decoder = GzDecoder::new(tarball);
 
@@ -376,8 +380,12 @@ fn verify_tarball(pkg_name: &str, tarball: &[u8], max_unpack: u64) -> AppResult<
 
     // Use this I/O object now to take a peek inside
     let mut archive = tar::Archive::new(decoder);
+
+    let vcs_info_path = Path::new(&pkg_name).join(".cargo_vcs_info.json");
+    let mut vcs_info = None;
+
     for entry in archive.entries()? {
-        let entry = entry.map_err(|err| {
+        let mut entry = entry.map_err(|err| {
             err.chain(cargo_err(
                 "uploaded tarball is malformed or too large when decompressed",
             ))
@@ -388,9 +396,15 @@ fn verify_tarball(pkg_name: &str, tarball: &[u8], max_unpack: u64) -> AppResult<
         // upload a tarball that contains both `foo-0.1.0/` source code as well
         // as `bar-0.1.0/` source code, and this could overwrite other crates in
         // the registry!
-        if !entry.path()?.starts_with(&pkg_name) {
+        let entry_path = entry.path()?;
+        if !entry_path.starts_with(&pkg_name) {
             return Err(cargo_err("invalid tarball uploaded"));
         }
+        if entry_path == vcs_info_path {
+            let mut contents = String::new();
+            entry.read_to_string(&mut contents)?;
+            vcs_info = CargoVcsInfo::from_contents(&contents).ok();
+        }
 
         // Historical versions of the `tar` crate which Cargo uses internally
         // don't properly prevent hard links and symlinks from overwriting
@@ -402,7 +416,7 @@ fn verify_tarball(pkg_name: &str, tarball: &[u8], max_unpack: u64) -> AppResult<
             return Err(cargo_err("invalid tarball uploaded"));
         }
     }
-    Ok(())
+    Ok(vcs_info)
 }
 
 #[cfg(test)]
@@ -422,12 +436,51 @@ mod tests {
     #[test]
     fn verify_tarball_test() {
         let mut pkg = tar::Builder::new(vec![]);
-        add_file(&mut pkg, "foo-0.0.1/.cargo_vcs_info.json", br#"{}"#);
+        add_file(&mut pkg, "foo-0.0.1/Cargo.toml", b"");
         let mut serialized_archive = vec![];
         GzEncoder::new(pkg.into_inner().unwrap().as_slice(), Default::default())
             .read_to_end(&mut serialized_archive)
             .unwrap();
-        verify_tarball("foo-0.0.1", &serialized_archive, 512 * 1024 * 1024).unwrap();
+        let vcs_info = verify_tarball("foo-0.0.1", &serialized_archive, 512 * 1024 * 1024).unwrap();
+        assert!(vcs_info.is_none());
         verify_tarball("bar-0.0.1", &serialized_archive, 512 * 1024 * 1024).unwrap_err();
     }
+
+    #[test]
+    fn verify_tarball_test_incomplete_vcs_info() {
+        let mut pkg = tar::Builder::new(vec![]);
+        add_file(&mut pkg, "foo-0.0.1/Cargo.toml", b"");
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/.cargo_vcs_info.json",
+            br#"{"unknown": "field"}"#,
+        );
+        let mut serialized_archive = vec![];
+        GzEncoder::new(pkg.into_inner().unwrap().as_slice(), Default::default())
+            .read_to_end(&mut serialized_archive)
+            .unwrap();
+        let vcs_info = verify_tarball("foo-0.0.1", &serialized_archive, 512 * 1024 * 1024)
+            .unwrap()
+            .unwrap();
+        assert_eq!(vcs_info.path_in_vcs, "");
+    }
+
+    #[test]
+    fn verify_tarball_test_vcs_info() {
+        let mut pkg = tar::Builder::new(vec![]);
+        add_file(&mut pkg, "foo-0.0.1/Cargo.toml", b"");
+        add_file(
+            &mut pkg,
+            "foo-0.0.1/.cargo_vcs_info.json",
+            br#"{"path_in_vcs": "path/in/vcs"}"#,
+        );
+        let mut serialized_archive = vec![];
+        GzEncoder::new(pkg.into_inner().unwrap().as_slice(), Default::default())
+            .read_to_end(&mut serialized_archive)
+            .unwrap();
+        let vcs_info = verify_tarball("foo-0.0.1", &serialized_archive, 512 * 1024 * 1024)
+            .unwrap()
+            .unwrap();
+        assert_eq!(vcs_info.path_in_vcs, "path/in/vcs");
+    }
 }

src/util.rs

@@ -53,3 +53,46 @@ impl Maximums {
         }
     }
 }
+
+/// Represents relevant contents of .cargo_vcs_info.json file when uploaded from cargo
+/// or downloaded from crates.io
+#[derive(Debug, Deserialize, Eq, PartialEq)]
+pub struct CargoVcsInfo {
+    /// Path to the package within repo (empty string if root). / not \
+    #[serde(default)]
+    pub path_in_vcs: String,
+}
+
+impl CargoVcsInfo {
+    pub fn from_contents(contents: &str) -> serde_json::Result<Self> {
+        serde_json::from_str(contents)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::CargoVcsInfo;
+
+    #[test]
+    fn test_cargo_vcs_info() {
+        assert_eq!(CargoVcsInfo::from_contents("").ok(), None);
+        assert_eq!(
+            CargoVcsInfo::from_contents("{}").unwrap(),
+            CargoVcsInfo {
+                path_in_vcs: "".into()
+            }
+        );
+        assert_eq!(
+            CargoVcsInfo::from_contents(r#"{"path_in_vcs": "hi"}"#).unwrap(),
+            CargoVcsInfo {
+                path_in_vcs: "hi".into()
+            }
+        );
+        assert_eq!(
+            CargoVcsInfo::from_contents(r#"{"path_in_vcs": "hi", "future": "field"}"#).unwrap(),
+            CargoVcsInfo {
+                path_in_vcs: "hi".into()
+            }
+        );
+    }
+}