Auto merge of rust-lang#119899 - onur-ozkan:redesign-stage0-std, r=<try>

redesign stage 0 std

### Summary

This PR changes how bootstrap builds the stage 1 compiler by switching to precompiled stage 0 standard library instead of building the in-tree one. The goal was to update bootstrap to use the beta standard library at stage 0 rather than compiling it from source (see the motivation at rust-lang/compiler-team#619).

Previously, to build a stage 1 compiler bootstrap followed this path:

```
download stage0 compiler -> build in-tree std -> compile stage1 compiler with in-tree std
```

With this PR, the new path is:

```
download stage0 compiler -> compile stage1 compiler with precompiled stage0 std
```

This also means that `cfg(bootstrap)`/`cfg(not(bootstrap))` is no longer needed for library development.

### Building "library"

Since stage0 `std` is no longer in-tree `x build/test/check library --stage 0` is now no-op. The minimum supported stage to build `std` is now 1. For the same reason, default stage values in the library profile is no longer 0.

Because building the in-tree library now requires a stage1 compiler, I highly recommend library developers to enable `download-rustc` to speed up compilation time.

<hr>

If you encounter a bug or unexpected results please open a topic in the [#t-infra/bootstrap](https://rust-lang.zulipchat.com/#narrow/channel/326414-t-infra.2Fbootstrap) Zulip channel or create a [bootstrap issue](https://github.com/rust-lang/rust/issues/new?template=bootstrap.md).

(Review thread: https://rust-lang.zulipchat.com/#narrow/channel/326414-t-infra.2Fbootstrap/topic/Review.20thread.3A.20stage.200.20redesign.20PR/with/508271433)

~~Blocked on rust-lang#122709

try-job: dist-x86_64-linux

Author: bors

compiler/rustc_span/src/analyze_source_file.rs

@@ -84,6 +84,7 @@ cfg_match! {
                 // For character in the chunk, see if its byte value is < 0, which
                 // indicates that it's part of a UTF-8 char.
                 let multibyte_test = _mm_cmplt_epi8(chunk, _mm_set1_epi8(0));
+
                 // Create a bit mask from the comparison results.
                 let multibyte_mask = _mm_movemask_epi8(multibyte_test);
 
@@ -93,6 +94,7 @@ cfg_match! {
 
                     // Check for newlines in the chunk
                     let newlines_test = _mm_cmpeq_epi8(chunk, _mm_set1_epi8(b'\n' as i8));
+
                     let mut newlines_mask = _mm_movemask_epi8(newlines_test);
 
                     let output_offset = RelativeBytePos::from_usize(chunk_index * CHUNK_SIZE + 1);

library/core/src/fmt/rt.rs

@@ -16,14 +16,6 @@ pub struct Placeholder {
     pub width: Count,
 }
 
-#[cfg(bootstrap)]
-impl Placeholder {
-    #[inline]
-    pub const fn new(position: usize, flags: u32, precision: Count, width: Count) -> Self {
-        Self { position, flags, precision, width }
-    }
-}
-
 /// Used by [width](https://doc.rust-lang.org/std/fmt/#width)
 /// and [precision](https://doc.rust-lang.org/std/fmt/#precision) specifiers.
 #[lang = "format_count"]

src/bootstrap/defaults/bootstrap.library.toml

@@ -1,17 +1,18 @@
 # These defaults are meant for contributors to the standard library and documentation.
 [build]
-# When building the standard library, you almost never want to build the compiler itself.
-build-stage = 0
-test-stage = 0
-bench-stage = 0
+build-stage = 1
+test-stage = 1
+bench-stage = 1
 
 [rust]
 # This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower.
 incremental = true
 # Make the compiler and standard library faster to build, at the expense of a ~20% runtime slowdown.
 lto = "off"
-# Download rustc by default for library profile if compiler-affecting
-# directories are not modified. For CI this is disabled.
+# When building the standard library, you almost never want to build the compiler itself.
+#
+# If compiler-affecting directories are not modified, use precompiled rustc to speed up
+# library development by skipping compiler builds.
 download-rustc = "if-unchanged"
 
 [llvm]

src/bootstrap/src/core/build_steps/check.rs

@@ -1,5 +1,6 @@
 //! Implementation of compiling the compiler and standard library, in "check"-based modes.
 
+use crate::core::build_steps::compile;
 use crate::core::build_steps::compile::{
     add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make,
 };
@@ -45,10 +46,12 @@ impl Step for Std {
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let stage = run.builder.top_stage;
         run.crate_or_deps("sysroot")
             .crate_or_deps("coretests")
             .crate_or_deps("alloctests")
             .path("library")
+            .default_condition(stage != 0)
     }
 
     fn make_run(run: RunConfig<'_>) {
@@ -62,6 +65,12 @@ impl Step for Std {
         let target = self.target;
         let compiler = builder.compiler(builder.top_stage, builder.config.build);
 
+        if builder.top_stage == 0 {
+            // Reuse the stage0 libstd
+            builder.ensure(compile::Std::new(compiler, target));
+            return;
+        }
+
         let mut cargo = builder::Cargo::new(
             builder,
             compiler,

src/bootstrap/src/core/build_steps/clippy.rs

@@ -207,16 +207,18 @@ impl Step for Rustc {
         let compiler = builder.compiler(builder.top_stage, builder.config.build);
         let target = self.target;
 
-        if compiler.stage != 0 {
-            // If we're not in stage 0, then we won't have a std from the beta
-            // compiler around. That means we need to make sure there's one in
-            // the sysroot for the compiler to find. Otherwise, we're going to
-            // fail when building crates that need to generate code (e.g., build
-            // scripts and their dependencies).
-            builder.ensure(compile::Std::new(compiler, compiler.host));
-            builder.ensure(compile::Std::new(compiler, target));
-        } else {
-            builder.ensure(check::Std::new(target).build_kind(Some(Kind::Check)));
+        if !builder.download_rustc() {
+            if compiler.stage != 0 {
+                // If we're not in stage 0, then we won't have a std from the beta
+                // compiler around. That means we need to make sure there's one in
+                // the sysroot for the compiler to find. Otherwise, we're going to
+                // fail when building crates that need to generate code (e.g., build
+                // scripts and their dependencies).
+                builder.ensure(compile::Std::new(compiler, compiler.host));
+                builder.ensure(compile::Std::new(compiler, target));
+            } else {
+                builder.ensure(check::Std::new(target).build_kind(Some(Kind::Check)));
+            }
         }
 
         let mut cargo = builder::Cargo::new(
@@ -286,7 +288,9 @@ macro_rules! lint_any {
                 let compiler = builder.compiler(builder.top_stage, builder.config.build);
                 let target = self.target;
 
-                builder.ensure(check::Rustc::new(target, builder).build_kind(Some(Kind::Check)));
+                if !builder.download_rustc() {
+                    builder.ensure(check::Rustc::new(target, builder).build_kind(Some(Kind::Check)));
+                };
 
                 let cargo = prepare_tool_cargo(
                     builder,

src/bootstrap/src/core/build_steps/compile.rs

@@ -151,16 +151,26 @@ impl Step for Std {
     )]
     fn run(self, builder: &Builder<'_>) {
         let target = self.target;
-        let compiler = self.compiler;
+
+        // We already have std ready to be used for stage 0.
+        if self.compiler.stage == 0 {
+            let compiler = self.compiler;
+            builder.ensure(StdLink::from_std(self, compiler));
+
+            return;
+        }
+
+        let compiler = if builder.download_rustc() && self.force_recompile {
+            // When there are changes in the library tree with CI-rustc, we want to build
+            // the stageN library and that requires using stageN-1 compiler.
+            builder.compiler(self.compiler.stage.saturating_sub(1), builder.config.build)
+        } else {
+            self.compiler
+        };
 
         // When using `download-rustc`, we already have artifacts for the host available. Don't
         // recompile them.
-        if builder.download_rustc() && builder.is_builder_target(target)
-            // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so
-            // its artifacts can't be reused.
-            && compiler.stage != 0
-            && !self.force_recompile
-        {
+        if builder.download_rustc() && builder.is_builder_target(target) && !self.force_recompile {
             let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false });
             cp_rustc_component_to_ci_sysroot(
                 builder,
@@ -193,7 +203,13 @@ impl Step for Std {
         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
         trace!(?compiler_to_use);
 
-        if compiler_to_use != compiler {
+        if compiler_to_use != compiler
+            // Never uplift std unless we have compiled stage 1; if stage 1 is compiled,
+            // uplift it from there.
+            //
+            // FIXME: improve `fn compiler_for` to avoid adding stage condition here.
+            && compiler.stage > 1
+        {
             trace!(?compiler_to_use, ?compiler, "compiler != compiler_to_use, uplifting library");
 
             builder.ensure(Std::new(compiler_to_use, target));
@@ -226,27 +242,6 @@ impl Step for Std {
 
         target_deps.extend(self.copy_extra_objects(builder, &compiler, target));
 
-        // The LLD wrappers and `rust-lld` are self-contained linking components that can be
-        // necessary to link the stdlib on some targets. We'll also need to copy these binaries to
-        // the `stage0-sysroot` to ensure the linker is found when bootstrapping on such a target.
-        if compiler.stage == 0 && builder.is_builder_target(compiler.host) {
-            trace!(
-                "(build == host) copying linking components to `stage0-sysroot` for bootstrapping"
-            );
-            // We want to copy the host `bin` folder within the `rustlib` folder in the sysroot.
-            let src_sysroot_bin = builder
-                .rustc_snapshot_sysroot()
-                .join("lib")
-                .join("rustlib")
-                .join(compiler.host)
-                .join("bin");
-            if src_sysroot_bin.exists() {
-                let target_sysroot_bin = builder.sysroot_target_bindir(compiler, target);
-                t!(fs::create_dir_all(&target_sysroot_bin));
-                builder.cp_link_r(&src_sysroot_bin, &target_sysroot_bin);
-            }
-        }
-
         // We build a sysroot for mir-opt tests using the same trick that Miri does: A check build
         // with -Zalways-encode-mir. This frees us from the need to have a target linker, and the
         // fact that this is a check build integrates nicely with run_cargo.
@@ -741,7 +736,7 @@ impl Step for StdLink {
         let target = self.target;
 
         // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`.
-        let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() {
+        let (libdir, hostdir) = if !self.force_recompile && builder.download_rustc() {
             // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too
             let lib = builder.sysroot_libdir_relative(self.compiler);
             let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot {
@@ -757,23 +752,16 @@ impl Step for StdLink {
             (libdir, hostdir)
         };
 
-        add_to_sysroot(
-            builder,
-            &libdir,
-            &hostdir,
-            &build_stamp::libstd_stamp(builder, compiler, target),
-        );
+        let is_downloaded_beta_stage0 = builder
+            .build
+            .config
+            .initial_rustc
+            .starts_with(builder.out.join(compiler.host).join("stage0/bin"));
 
         // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0`
         // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta,
         // and is not set to a custom path.
-        if compiler.stage == 0
-            && builder
-                .build
-                .config
-                .initial_rustc
-                .starts_with(builder.out.join(compiler.host).join("stage0/bin"))
-        {
+        if compiler.stage == 0 && is_downloaded_beta_stage0 {
             // Copy bin files from stage0/bin to stage0-sysroot/bin
             let sysroot = builder.out.join(compiler.host).join("stage0-sysroot");
 
@@ -783,21 +771,9 @@ impl Step for StdLink {
             t!(fs::create_dir_all(&sysroot_bin_dir));
             builder.cp_link_r(&stage0_bin_dir, &sysroot_bin_dir);
 
-            // Copy all files from stage0/lib to stage0-sysroot/lib
             let stage0_lib_dir = builder.out.join(host).join("stage0/lib");
-            if let Ok(files) = fs::read_dir(stage0_lib_dir) {
-                for file in files {
-                    let file = t!(file);
-                    let path = file.path();
-                    if path.is_file() {
-                        builder.copy_link(
-                            &path,
-                            &sysroot.join("lib").join(path.file_name().unwrap()),
-                            FileType::Regular,
-                        );
-                    }
-                }
-            }
+            t!(fs::create_dir_all(sysroot.join("lib")));
+            builder.cp_link_r(&stage0_lib_dir, &sysroot.join("lib"));
 
             // Copy codegen-backends from stage0
             let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler);
@@ -811,6 +787,22 @@ impl Step for StdLink {
             if stage0_codegen_backends.exists() {
                 builder.cp_link_r(&stage0_codegen_backends, &sysroot_codegen_backends);
             }
+        } else if compiler.stage == 0 {
+            let sysroot = builder.out.join(compiler.host.triple).join("stage0-sysroot");
+            builder.cp_link_r(&builder.initial_sysroot.join("lib"), &sysroot.join("lib"));
+        } else {
+            if builder.download_rustc() {
+                // Ensure there are no CI-rustc std artifacts.
+                let _ = fs::remove_dir_all(&libdir);
+                let _ = fs::remove_dir_all(&hostdir);
+            }
+
+            add_to_sysroot(
+                builder,
+                &libdir,
+                &hostdir,
+                &build_stamp::libstd_stamp(builder, compiler, target),
+            );
         }
     }
 }
@@ -1033,7 +1025,7 @@ impl Step for Rustc {
         let compiler = self.compiler;
         let target = self.target;
 
-        // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
+        // NOTE: the ABI of the stage0 compiler is different from the ABI of the downloaded compiler,
         // so its artifacts can't be reused.
         if builder.download_rustc() && compiler.stage != 0 {
             trace!(stage = compiler.stage, "`download_rustc` requested");
@@ -1788,9 +1780,9 @@ impl Step for Sysroot {
         t!(fs::create_dir_all(&sysroot));
 
         // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0
-        // compiler relies on more recent version of LLVM than the beta compiler, it may not
+        // compiler relies on more recent version of LLVM than the stage0 compiler, it may not
         // be able to locate the correct LLVM in the sysroot. This situation typically occurs
-        // when we upgrade LLVM version while the beta compiler continues to use an older version.
+        // when we upgrade LLVM version while the stage0 compiler continues to use an older version.
         //
         // Make sure to add the correct version of LLVM into the stage0 sysroot.
         if compiler.stage == 0 {

src/bootstrap/src/core/build_steps/test.rs

@@ -1558,7 +1558,7 @@ impl Step for Compiletest {
 
         if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() {
             eprintln!("\
-ERROR: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail
+ERROR: `--stage 0` runs compiletest on the stage0 (precompiled) compiler, not your local changes, and will almost always cause tests to fail
 HELP: to test the compiler, use `--stage 1` instead
 HELP: to test the standard library, use `--stage 0 library/std` instead
 NOTE: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`."
@@ -1586,9 +1586,9 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
         // NOTE: Only stage 1 is special cased because we need the rustc_private artifacts to match the
         // running compiler in stage 2 when plugins run.
         let (stage, stage_id) = if suite == "ui-fulldeps" && compiler.stage == 1 {
-            // At stage 0 (stage - 1) we are using the beta compiler. Using `self.target` can lead
-            // finding an incorrect compiler path on cross-targets, as the stage 0 beta compiler is
-            // always equal to `build.build` in the configuration.
+            // At stage 0 (stage - 1) we are using the stage0 compiler. Using `self.target` can lead
+            // finding an incorrect compiler path on cross-targets, as the stage 0 is always equal to
+            // `build.build` in the configuration.
             let build = builder.build.build;
             compiler = builder.compiler(compiler.stage - 1, build);
             let test_stage = compiler.stage + 1;
@@ -1674,7 +1674,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
         }
 
         if mode == "rustdoc-json" {
-            // Use the beta compiler for jsondocck
+            // Use the stage0 compiler for jsondocck
             let json_compiler = compiler.with_stage(0);
             cmd.arg("--jsondocck-path")
                 .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }).tool_path);

src/bootstrap/src/core/build_steps/tool.rs

@@ -330,9 +330,9 @@ pub(crate) fn get_tool_rustc_compiler(
         return target_compiler;
     }
 
-    if builder.download_rustc() && target_compiler.stage > 0 {
-        // We already have the stage N compiler, we don't need to cut the stage.
-        return builder.compiler(target_compiler.stage, builder.config.build);
+    if builder.download_rustc() && target_compiler.stage == 1 {
+        // We shouldn't drop to stage0 compiler when using CI rustc.
+        return builder.compiler(1, builder.config.build);
     }
 
     // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise