From 25cc75c9245664f239a52de60a0c0baa8b4c81d5 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 9 Oct 2020 15:34:57 +0200
Subject: [PATCH 1/7] build-manifest: accept the Rust version instead of the
monorepo path
This commit changes the way build-manifest is invoked, to let it accept
the Rust version directly instead of requiring the path of the Rust
monorepo and letting build-manifest figure out the path on its own.
This allows to run build-manifest without a clone of the monorepo.
---
src/bootstrap/dist.rs | 2 +-
src/bootstrap/run.rs | 2 +-
src/tools/build-manifest/README.md | 4 ++--
src/tools/build-manifest/src/main.rs | 4 ++--
src/tools/build-manifest/src/versions.rs | 13 +++----------
5 files changed, 9 insertions(+), 16 deletions(-)
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 3a0743da7a415..8f4968d5ac69f 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -2353,7 +2353,7 @@ impl Step for HashSign {
cmd.arg(today.trim());
cmd.arg(addr);
cmd.arg(&builder.config.channel);
- cmd.arg(&builder.src);
+ cmd.arg(&builder.version);
cmd.env("BUILD_MANIFEST_LEGACY", "1");
builder.create_dir(&distdir(builder));
diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs
index 80c093e713eff..d8169549c07ba 100644
--- a/src/bootstrap/run.rs
+++ b/src/bootstrap/run.rs
@@ -77,7 +77,7 @@ impl Step for BuildManifest {
cmd.arg(today.trim());
cmd.arg(addr);
cmd.arg(&builder.config.channel);
- cmd.arg(&builder.src);
+ cmd.arg(&builder.version);
builder.create_dir(&distdir(builder));
builder.run(&mut cmd);
diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md
index 26e96c9fd8fda..b77c5a907c118 100644
--- a/src/tools/build-manifest/README.md
+++ b/src/tools/build-manifest/README.md
@@ -21,8 +21,8 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
```
$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \
- CHANNEL path/to/rust/repo
+ CHANNEL VERSION
```
Remember to replace `CHANNEL` with the channel you produced dist artifacts of
-and `path/to/rust/repo` with the path to your checkout of the Rust repository.
+and `VERSION` with the current Rust version.
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 7ee28cd6e5d95..0efebe363f88f 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -221,7 +221,7 @@ fn main() {
let date = args.next().unwrap();
let s3_address = args.next().unwrap();
let channel = args.next().unwrap();
- let monorepo_path = args.next().unwrap();
+ let rustc_version = args.next().unwrap();
// Do not ask for a passphrase while manually testing
let mut passphrase = String::new();
@@ -231,7 +231,7 @@ fn main() {
}
Builder {
- versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(),
+ versions: Versions::new(&channel, &rustc_version, &input).unwrap(),
input,
output,
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 75b6979b54a78..c2bce0f1cf39c 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -1,4 +1,4 @@
-use anyhow::{Context, Error};
+use anyhow::Error;
use flate2::read::GzDecoder;
use std::collections::HashMap;
use std::fs::File;
@@ -93,17 +93,10 @@ pub(crate) struct Versions {
}
impl Versions {
- pub(crate) fn new(
- channel: &str,
- dist_path: &Path,
- monorepo_root: &Path,
- ) -> Result<Self, Error> {
+ pub(crate) fn new(channel: &str, rustc_version: &str, dist_path: &Path) -> Result<Self, Error> {
Ok(Self {
channel: channel.into(),
- rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
- .context("failed to read the rustc version from src/version")?
- .trim()
- .to_string(),
+ rustc_version: rustc_version.into(),
dist_path: dist_path.into(),
versions: HashMap::new(),
})
From 2f387e9d11f1ea6222af0ff00e4de956496fc83f Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 9 Oct 2020 19:49:13 +0200
Subject: [PATCH 2/7] bootstrap: add disabled by default build-manifest dist
component
---
src/bootstrap/builder.rs | 1 +
src/bootstrap/dist.rs | 67 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 68 insertions(+)
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 4bc162abee6c1..73761b57a8d44 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -462,6 +462,7 @@ impl<'a> Builder<'a> {
dist::LlvmTools,
dist::RustDev,
dist::Extended,
+ dist::BuildManifest,
dist::HashSign
),
Kind::Install => describe!(
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 8f4968d5ac69f..12533d29809cf 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -2584,3 +2584,70 @@ impl Step for RustDev {
Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)))
}
}
+
+/// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
+/// release process to avoid cloning the monorepo and building stuff.
+///
+/// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct BuildManifest {
+ pub target: TargetSelection,
+}
+
+impl Step for BuildManifest {
+ type Output = PathBuf;
+ const DEFAULT: bool = false;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/build-manifest")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(BuildManifest { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ let build_manifest = builder.tool_exe(Tool::BuildManifest);
+
+ let name = pkgname(builder, "build-manifest");
+ let tmp = tmpdir(builder);
+
+ // Prepare the image.
+ let image = tmp.join("build-manifest-image");
+ let image_bin = image.join("bin");
+ let _ = fs::remove_dir_all(&image);
+ t!(fs::create_dir_all(&image_bin));
+ builder.install(&build_manifest, &image_bin.join("build-manifest"), 0o755);
+
+ // Prepare the overlay.
+ let overlay = tmp.join("build-manifest-overlay");
+ let _ = fs::remove_dir_all(&overlay);
+ builder.create_dir(&overlay);
+ builder.create(&overlay.join("version"), &builder.rust_version());
+ for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] {
+ builder.install(&builder.src.join(file), &overlay, 0o644);
+ }
+
+ // Create the final tarball.
+ let mut cmd = rust_installer(builder);
+ cmd.arg("generate")
+ .arg("--product-name=Rust")
+ .arg("--rel-manifest-dir=rustlib")
+ .arg("--success-message=build-manifest installed.")
+ .arg("--image-dir")
+ .arg(&image)
+ .arg("--work-dir")
+ .arg(&tmpdir(builder))
+ .arg("--output-dir")
+ .arg(&distdir(builder))
+ .arg("--non-installed-overlay")
+ .arg(&overlay)
+ .arg(format!("--package-name={}-{}", name, self.target.triple))
+ .arg("--legacy-manifest-dirs=rustlib,cargo")
+ .arg("--component-name=build-manifest");
+
+ builder.run(&mut cmd);
+ distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple))
+ }
+}
From 60ae018bf1e0d6d372ac545b56c8992a1365e917 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 9 Oct 2020 19:51:07 +0200
Subject: [PATCH 3/7] bootstrap: add --include-default-paths to ./x.py
---
src/bootstrap/builder.rs | 42 ++++++++++++++++++++--------------------
src/bootstrap/config.rs | 2 ++
src/bootstrap/flags.rs | 7 +++++++
3 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 73761b57a8d44..707c1ff3efad9 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -193,37 +193,37 @@ impl StepDescription {
);
}
- if paths.is_empty() {
- for (desc, should_run) in v.iter().zip(should_runs) {
+ if paths.is_empty() || builder.config.include_default_paths {
+ for (desc, should_run) in v.iter().zip(&should_runs) {
if desc.default && should_run.is_really_default {
for pathset in &should_run.paths {
desc.maybe_run(builder, pathset);
}
}
}
- } else {
- for path in paths {
- // strip CurDir prefix if present
- let path = match path.strip_prefix(".") {
- Ok(p) => p,
- Err(_) => path,
- };
+ }
- let mut attempted_run = false;
- for (desc, should_run) in v.iter().zip(&should_runs) {
- if let Some(suite) = should_run.is_suite_path(path) {
- attempted_run = true;
- desc.maybe_run(builder, suite);
- } else if let Some(pathset) = should_run.pathset_for_path(path) {
- attempted_run = true;
- desc.maybe_run(builder, pathset);
- }
- }
+ for path in paths {
+ // strip CurDir prefix if present
+ let path = match path.strip_prefix(".") {
+ Ok(p) => p,
+ Err(_) => path,
+ };
- if !attempted_run {
- panic!("error: no rules matched {}", path.display());
+ let mut attempted_run = false;
+ for (desc, should_run) in v.iter().zip(&should_runs) {
+ if let Some(suite) = should_run.is_suite_path(path) {
+ attempted_run = true;
+ desc.maybe_run(builder, suite);
+ } else if let Some(pathset) = should_run.pathset_for_path(path) {
+ attempted_run = true;
+ desc.maybe_run(builder, pathset);
}
}
+
+ if !attempted_run {
+ panic!("error: no rules matched {}", path.display());
+ }
}
}
}
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 6265bbaf5c22c..db82155bd6ad2 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -61,6 +61,7 @@ pub struct Config {
pub profiler: bool,
pub ignore_git: bool,
pub exclude: Vec<PathBuf>,
+ pub include_default_paths: bool,
pub rustc_error_format: Option<String>,
pub json_output: bool,
pub test_compare_mode: bool,
@@ -532,6 +533,7 @@ impl Config {
let mut config = Config::default_opts();
config.exclude = flags.exclude;
+ config.include_default_paths = flags.include_default_paths;
config.rustc_error_format = flags.rustc_error_format;
config.json_output = flags.json_output;
config.on_fail = flags.on_fail;
diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
index 319a0b4e611eb..c10188875fbc4 100644
--- a/src/bootstrap/flags.rs
+++ b/src/bootstrap/flags.rs
@@ -30,6 +30,7 @@ pub struct Flags {
pub cmd: Subcommand,
pub incremental: bool,
pub exclude: Vec<PathBuf>,
+ pub include_default_paths: bool,
pub rustc_error_format: Option<String>,
pub json_output: bool,
pub dry_run: bool,
@@ -137,6 +138,11 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
opts.optmulti("", "host", "host targets to build", "HOST");
opts.optmulti("", "target", "target targets to build", "TARGET");
opts.optmulti("", "exclude", "build paths to exclude", "PATH");
+ opts.optflag(
+ "",
+ "include-default-paths",
+ "include default paths in addition to the provided ones",
+ );
opts.optopt("", "on-fail", "command to run on failure", "CMD");
opts.optflag("", "dry-run", "dry run; don't build anything");
opts.optopt(
@@ -618,6 +624,7 @@ Arguments:
.into_iter()
.map(|p| p.into())
.collect::<Vec<_>>(),
+ include_default_paths: matches.opt_present("include-default-paths"),
deny_warnings: parse_deny_warnings(&matches),
llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map(
|s| s.parse::<bool>().expect("`llvm-skip-rebuild` should be either true or false"),
From 24d04ccd3977d4eca676439067f096de484f6fa7 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 9 Oct 2020 19:53:48 +0200
Subject: [PATCH 4/7] ci: also build the build-manifest component on
dist-x86_64-linux
---
src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
index 58e2567a58f08..14700aeea05af 100644
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
@@ -98,7 +98,9 @@ ENV RUST_CONFIGURE_ARGS \
--set llvm.thin-lto=true \
--set llvm.ninja=false \
--set rust.jemalloc
-ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS \
+ --include-default-paths \
+ src/tools/build-manifest
ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
# This is the only builder which will create source tarballs
From f3d07b36ed7609a7826200479d8d472d36f0a995 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Mon, 12 Oct 2020 17:47:37 +0200
Subject: [PATCH 5/7] build-manifest: allow configuring the number of threads
---
Cargo.lock | 1 +
src/tools/build-manifest/Cargo.toml | 1 +
src/tools/build-manifest/src/main.rs | 19 ++++++++++++-------
3 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index bfc2d3e066a89..83ac8396a16e3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -243,6 +243,7 @@ dependencies = [
"anyhow",
"flate2",
"hex 0.4.2",
+ "num_cpus",
"rayon",
"serde",
"serde_json",
diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml
index 4ae4dbfc06ede..4a2c710811f61 100644
--- a/src/tools/build-manifest/Cargo.toml
+++ b/src/tools/build-manifest/Cargo.toml
@@ -14,3 +14,4 @@ tar = "0.4.29"
sha2 = "0.9.1"
rayon = "1.3.1"
hex = "0.4.2"
+num_cpus = "1.13.0"
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 0efebe363f88f..6fda9f4e59f52 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -207,13 +207,18 @@ fn main() {
// related code in this tool and ./x.py dist hash-and-sign can be removed.
let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok();
- // Avoid overloading the old server in legacy mode.
- if legacy {
- rayon::ThreadPoolBuilder::new()
- .num_threads(1)
- .build_global()
- .expect("failed to initialize Rayon");
- }
+ let num_threads = if legacy {
+ // Avoid overloading the old server in legacy mode.
+ 1
+ } else if let Ok(num) = env::var("BUILD_MANIFEST_NUM_THREADS") {
+ num.parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
+ } else {
+ num_cpus::get()
+ };
+ rayon::ThreadPoolBuilder::new()
+ .num_threads(num_threads)
+ .build_global()
+ .expect("failed to initialize Rayon");
let mut args = env::args().skip(1);
let input = PathBuf::from(args.next().unwrap());
From cbded3e193ba7acc4611e9b8612bbc98608e7800 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Mon, 12 Oct 2020 19:34:01 +0200
Subject: [PATCH 6/7] build-manifest: use var_os instead of var to check if
vars exist
This will prevent the tool mistakenly ignoring the variables if they
happen to contain non-utf8 data.
---
src/tools/build-manifest/src/main.rs | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 6fda9f4e59f52..1515f46e03ac2 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -205,13 +205,13 @@ fn main() {
//
// Once the old release process is fully decommissioned, the environment variable, all the
// related code in this tool and ./x.py dist hash-and-sign can be removed.
- let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok();
+ let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some();
let num_threads = if legacy {
// Avoid overloading the old server in legacy mode.
1
- } else if let Ok(num) = env::var("BUILD_MANIFEST_NUM_THREADS") {
- num.parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
+ } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
+ num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
} else {
num_cpus::get()
};
From 0b7ee9d522242d6320d1066d3ba9d2314a576e8b Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Mon, 12 Oct 2020 19:40:35 +0200
Subject: [PATCH 7/7] build-manifest: bundle the rustc version in the binary
---
src/bootstrap/dist.rs | 1 -
src/bootstrap/run.rs | 1 -
src/tools/build-manifest/src/main.rs | 3 +--
src/tools/build-manifest/src/versions.rs | 17 ++++++-----------
4 files changed, 7 insertions(+), 15 deletions(-)
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 12533d29809cf..dd4cf9d595323 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -2353,7 +2353,6 @@ impl Step for HashSign {
cmd.arg(today.trim());
cmd.arg(addr);
cmd.arg(&builder.config.channel);
- cmd.arg(&builder.version);
cmd.env("BUILD_MANIFEST_LEGACY", "1");
builder.create_dir(&distdir(builder));
diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs
index d8169549c07ba..7c64e5a0aadc8 100644
--- a/src/bootstrap/run.rs
+++ b/src/bootstrap/run.rs
@@ -77,7 +77,6 @@ impl Step for BuildManifest {
cmd.arg(today.trim());
cmd.arg(addr);
cmd.arg(&builder.config.channel);
- cmd.arg(&builder.version);
builder.create_dir(&distdir(builder));
builder.run(&mut cmd);
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 1515f46e03ac2..cb04900c737e0 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -226,7 +226,6 @@ fn main() {
let date = args.next().unwrap();
let s3_address = args.next().unwrap();
let channel = args.next().unwrap();
- let rustc_version = args.next().unwrap();
// Do not ask for a passphrase while manually testing
let mut passphrase = String::new();
@@ -236,7 +235,7 @@ fn main() {
}
Builder {
- versions: Versions::new(&channel, &rustc_version, &input).unwrap(),
+ versions: Versions::new(&channel, &input).unwrap(),
input,
output,
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index c2bce0f1cf39c..79f2ef8dfc450 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
use tar::Archive;
const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
+const RUSTC_VERSION: &str = include_str!("../../../version");
#[derive(Debug, Hash, Eq, PartialEq, Clone)]
pub(crate) enum PkgType {
@@ -87,19 +88,13 @@ pub(crate) struct VersionInfo {
pub(crate) struct Versions {
channel: String,
- rustc_version: String,
dist_path: PathBuf,
versions: HashMap<PkgType, VersionInfo>,
}
impl Versions {
- pub(crate) fn new(channel: &str, rustc_version: &str, dist_path: &Path) -> Result<Self, Error> {
- Ok(Self {
- channel: channel.into(),
- rustc_version: rustc_version.into(),
- dist_path: dist_path.into(),
- versions: HashMap::new(),
- })
+ pub(crate) fn new(channel: &str, dist_path: &Path) -> Result<Self, Error> {
+ Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() })
}
pub(crate) fn channel(&self) -> &str {
@@ -177,10 +172,10 @@ impl Versions {
) -> Result<String, Error> {
let component_name = package.tarball_component_name();
let version = match self.channel.as_str() {
- "stable" => self.rustc_version.clone(),
+ "stable" => RUSTC_VERSION.into(),
"beta" => "beta".into(),
"nightly" => "nightly".into(),
- _ => format!("{}-dev", self.rustc_version),
+ _ => format!("{}-dev", RUSTC_VERSION),
};
if package.target_independent() {
@@ -191,6 +186,6 @@ impl Versions {
}
pub(crate) fn rustc_version(&self) -> &str {
- &self.rustc_version
+ RUSTC_VERSION
}
}