From 2c752bcf559975995eb8086a7fa6a7f9b5ba0de8 Mon Sep 17 00:00:00 2001 From: Arlo Siemsen Date: Thu, 20 Feb 2025 11:47:14 -0600 Subject: [PATCH 01/30] Undeprecate env::home_dir --- library/std/src/env.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/library/std/src/env.rs b/library/std/src/env.rs index adbd68896241c..5a931b077cdb0 100644 --- a/library/std/src/env.rs +++ b/library/std/src/env.rs @@ -641,11 +641,6 @@ impl Error for JoinPathsError { /// None => println!("Impossible to get your home dir!"), /// } /// ``` -#[deprecated( - since = "1.29.0", - note = "This function's behavior may be unexpected on Windows. \ - Consider using a crate from crates.io instead." -)] #[must_use] #[stable(feature = "env", since = "1.0.0")] pub fn home_dir() -> Option { From b34054511401f7aed8e76b8d5663651b5b4ced2a Mon Sep 17 00:00:00 2001 From: Rain Date: Tue, 11 Feb 2025 22:50:26 +0000 Subject: [PATCH 02/30] [illumos] attempt to use posix_spawn to spawn processes illumos has `posix_spawn`, and the very newest versions also have `_addchdir`, so use that. This is a nice ~4x performance improvement for process creation. My go-to as usual is nextest against the clap repo, which acts as a stress test for process creation -- with [this commit]: ```console $ cargo nextest run -E 'not test(ui_tests) and not test(example_tests)' before: Summary [ 1.747s] 879 tests run: 879 passed, 2 skipped after: Summary [ 0.445s] 879 tests run: 879 passed, 2 skipped ``` [this commit]: https://github.com/clap-rs/clap/commit/fde45f9aea766fb8de46e3d46e6575f393c3b6b9 --- library/Cargo.lock | 4 ++-- library/std/Cargo.toml | 2 +- .../src/sys/pal/unix/process/process_unix.rs | 17 ++++++++++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/library/Cargo.lock b/library/Cargo.lock index 0be2f9a154939..6bc3cf4da9f72 100644 --- a/library/Cargo.lock +++ b/library/Cargo.lock @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" dependencies = [ "rustc-std-workspace-core", ] diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml index 228ee6eea05fa..cec5cdbb92c27 100644 --- a/library/std/Cargo.toml +++ b/library/std/Cargo.toml @@ -35,7 +35,7 @@ miniz_oxide = { version = "0.8.0", optional = true, default-features = false } addr2line = { version = "0.24.0", optional = true, default-features = false } [target.'cfg(not(all(windows, target_env = "msvc")))'.dependencies] -libc = { version = "0.2.169", default-features = false, features = [ +libc = { version = "0.2.170", default-features = false, features = [ 'rustc-dep-of-std', ], public = true } diff --git a/library/std/src/sys/pal/unix/process/process_unix.rs b/library/std/src/sys/pal/unix/process/process_unix.rs index aa7406dd54874..7c5b0fca11a10 100644 --- a/library/std/src/sys/pal/unix/process/process_unix.rs +++ b/library/std/src/sys/pal/unix/process/process_unix.rs @@ -410,6 +410,7 @@ impl Command { #[cfg(not(any( target_os = "freebsd", + target_os = "illumos", all(target_os = "linux", target_env = "gnu"), all(target_os = "linux", target_env = "musl"), target_os = "nto", @@ -427,6 +428,7 @@ impl Command { // directly. #[cfg(any( target_os = "freebsd", + target_os = "illumos", all(target_os = "linux", target_env = "gnu"), all(target_os = "linux", target_env = "musl"), target_os = "nto", @@ -584,6 +586,10 @@ impl Command { fn get_posix_spawn_addchdir() -> Option { use crate::sys::weak::weak; + // POSIX.1-2024 standardizes this function: + // https://pubs.opengroup.org/onlinepubs/9799919799/functions/posix_spawn_file_actions_addchdir.html. + // The _np version is more widely available, though, so try that first. + weak! { fn posix_spawn_file_actions_addchdir_np( *mut libc::posix_spawn_file_actions_t, @@ -591,7 +597,16 @@ impl Command { ) -> libc::c_int } - posix_spawn_file_actions_addchdir_np.get() + weak! { + fn posix_spawn_file_actions_addchdir( + *mut libc::posix_spawn_file_actions_t, + *const libc::c_char + ) -> libc::c_int + } + + posix_spawn_file_actions_addchdir_np + .get() + .or_else(|| posix_spawn_file_actions_addchdir.get()) } /// Get the function pointer for adding a chdir action to a From 78615ff2ae2813a1fd4e26ab1d9f7e914f6c7902 Mon Sep 17 00:00:00 2001 From: Alona Enraght-Moony Date: Mon, 24 Feb 2025 23:34:46 +0000 Subject: [PATCH 03/30] Stablize `string_extend_from_within` --- library/alloc/src/string.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index 9446afd4b24b0..88227688ede05 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -1137,7 +1137,6 @@ impl String { /// # Examples /// /// ``` - /// #![feature(string_extend_from_within)] /// let mut string = String::from("abcde"); /// /// string.extend_from_within(2..); @@ -1150,7 +1149,7 @@ impl String { /// assert_eq!(string, "abcdecdeabecde"); /// ``` #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "string_extend_from_within", issue = "103806")] + #[stable(feature = "string_extend_from_within", since = "CURRENT_RUSTC_VERSION")] pub fn extend_from_within(&mut self, src: R) where R: RangeBounds, From 0ca1c9c1dd184da0786e76d674e8f0c7f64ef124 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Thu, 6 Feb 2025 13:19:11 -0800 Subject: [PATCH 04/30] Count char width at most once in Formatter::pad When both width and precision flags are specified, then the character width is counted twice. Instead, record the character width when truncating it to the precision, so it does not need to be recomputed. Simplify control flow so the cases are more clear. --- library/core/src/fmt/mod.rs | 66 ++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs index 764e7fff33ec7..3f60bb067d6e6 100644 --- a/library/core/src/fmt/mod.rs +++ b/library/core/src/fmt/mod.rs @@ -1693,49 +1693,41 @@ impl<'a> Formatter<'a> { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn pad(&mut self, s: &str) -> Result { - // Make sure there's a fast path up front + // Make sure there's a fast path up front. if self.options.width.is_none() && self.options.precision.is_none() { return self.buf.write_str(s); } - // The `precision` field can be interpreted as a `max-width` for the + + // The `precision` field can be interpreted as a maximum width for the // string being formatted. - let s = if let Some(max) = self.options.precision { - // If our string is longer that the precision, then we must have - // truncation. However other flags like `fill`, `width` and `align` - // must act as always. - if let Some((i, _)) = s.char_indices().nth(max) { - // LLVM here can't prove that `..i` won't panic `&s[..i]`, but - // we know that it can't panic. Use `get` + `unwrap_or` to avoid - // `unsafe` and otherwise don't emit any panic-related code - // here. - s.get(..i).unwrap_or(s) - } else { - &s - } + let (s, char_count) = if let Some(max_char_count) = self.options.precision { + let mut iter = s.char_indices(); + let remaining = match iter.advance_by(max_char_count) { + Ok(()) => 0, + Err(remaining) => remaining.get(), + }; + // SAFETY: The offset of `.char_indices()` is guaranteed to be + // in-bounds and between character boundaries. + let truncated = unsafe { s.get_unchecked(..iter.offset()) }; + (truncated, max_char_count - remaining) } else { - &s + // Use the optimized char counting algorithm for the full string. + (s, s.chars().count()) }; - // The `width` field is more of a `min-width` parameter at this point. - match self.options.width { - // If we're under the maximum length, and there's no minimum length - // requirements, then we can just emit the string - None => self.buf.write_str(s), - Some(width) => { - let chars_count = s.chars().count(); - // If we're under the maximum width, check if we're over the minimum - // width, if so it's as easy as just emitting the string. - if chars_count >= width { - self.buf.write_str(s) - } - // If we're under both the maximum and the minimum width, then fill - // up the minimum width with the specified string + some alignment. - else { - let align = Alignment::Left; - let post_padding = self.padding(width - chars_count, align)?; - self.buf.write_str(s)?; - post_padding.write(self) - } - } + + // The `width` field is more of a minimum width parameter at this point. + if let Some(width) = self.options.width + && char_count < width + { + // If we're under the minimum width, then fill up the minimum width + // with the specified string + some alignment. + let post_padding = self.padding(width - char_count, Alignment::Left)?; + self.buf.write_str(s)?; + post_padding.write(self) + } else { + // If we're over the minimum width or there is no minimum width, we + // can just emit the string. + self.buf.write_str(s) } } From 4c452857e88549008b5ae720b358339a114f86a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 26 Feb 2025 11:59:00 +0100 Subject: [PATCH 05/30] Add `dist:Gcc` build step To distribute the prebuilt libgccjit.so from CI. --- src/bootstrap/src/core/build_steps/dist.rs | 27 ++++++++++++++++++++++ src/bootstrap/src/core/builder/mod.rs | 1 + src/tools/opt-dist/src/main.rs | 1 + 3 files changed, 29 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 26b3e0b701c4b..e0ad6856dc72c 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -2464,3 +2464,30 @@ impl Step for ReproducibleArtifacts { if added_anything { Some(tarball.generate()) } else { None } } } + +/// Tarball containing a prebuilt version of the libgccjit library, +/// needed as a dependency for the GCC codegen backend (similarly to the LLVM +/// backend needing a prebuilt libLLVM). +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct Gcc { + pub target: TargetSelection, +} + +impl Step for Gcc { + type Output = GeneratedTarball; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("gcc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Gcc { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let tarball = Tarball::new(builder, "gcc", &self.target.triple); + let output = builder.ensure(super::gcc::Gcc { target: self.target }); + tarball.add_file(output.libgccjit, ".", 0o644); + tarball.generate() + } +} diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index 9c04f097bee27..10bd67c5b6c13 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1072,6 +1072,7 @@ impl<'a> Builder<'a> { dist::PlainSourceTarball, dist::BuildManifest, dist::ReproducibleArtifacts, + dist::Gcc ), Kind::Install => describe!( install::Docs, diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 565721a909344..3111149f2ae20 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -389,6 +389,7 @@ fn main() -> anyhow::Result<()> { "clippy", "miri", "rustfmt", + "gcc", ] { build_args.extend(["--skip".to_string(), target.to_string()]); } From e4bfad2ba1ee8df8147a41e480d0c2735e12dbdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 28 Feb 2025 09:43:05 +0100 Subject: [PATCH 06/30] Build GCC on the Linux x64 dist runner --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 0b4682ac32ba0..f54ecef1e308a 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -101,7 +101,7 @@ ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ - build-manifest bootstrap + build-manifest bootstrap gcc ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang # This is the only builder which will create source tarballs From 6563437773dcf87a71c9a41a8aec90d23efeb4b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:02:41 +0100 Subject: [PATCH 07/30] Create the `OptimizedDist` tool with a macro --- src/bootstrap/src/core/build_steps/tool.rs | 40 +--------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 39acb646dff4d..27cc49d3be7c8 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -445,51 +445,13 @@ bootstrap_tool!( WasmComponentLd, "src/tools/wasm-component-ld", "wasm-component-ld", is_unstable_tool = true, allow_features = "min_specialization"; UnicodeTableGenerator, "src/tools/unicode-table-generator", "unicode-table-generator"; FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump"; + OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"]; ); /// These are the submodules that are required for rustbook to work due to /// depending on mdbook plugins. pub static SUBMODULES_FOR_RUSTBOOK: &[&str] = &["src/doc/book", "src/doc/reference"]; -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct OptimizedDist { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for OptimizedDist { - type Output = ToolBuildResult; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/opt-dist") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(OptimizedDist { - compiler: run.builder.compiler(0, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> ToolBuildResult { - // We need to ensure the rustc-perf submodule is initialized when building opt-dist since - // the tool requires it to be in place to run. - builder.require_submodule("src/tools/rustc-perf", None); - - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "opt-dist", - mode: Mode::ToolBootstrap, - path: "src/tools/opt-dist", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - /// The [rustc-perf](https://github.com/rust-lang/rustc-perf) benchmark suite, which is added /// as a submodule at `src/tools/rustc-perf`. #[derive(Debug, Clone, Hash, PartialEq, Eq)] From 96a91975f1a2695bba3ed633f825062be748007b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:20:32 +0100 Subject: [PATCH 08/30] Allow specifying that tools build a library, not a binary --- src/bootstrap/src/core/build_steps/tool.rs | 40 +++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 27cc49d3be7c8..e401a0bdfc842 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -34,6 +34,12 @@ pub enum SourceType { Submodule, } +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum ToolArtifactKind { + Binary, + Library, +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] struct ToolBuild { compiler: Compiler, @@ -47,6 +53,8 @@ struct ToolBuild { allow_features: &'static str, /// Additional arguments to pass to the `cargo` invocation. cargo_args: Vec, + /// Whether the tool builds a binary or a library. + artifact_kind: ToolArtifactKind, } impl Builder<'_> { @@ -79,7 +87,7 @@ impl Builder<'_> { /// for using this type as `type Output = ToolBuildResult;` #[derive(Clone)] pub struct ToolBuildResult { - /// Executable path of the corresponding tool that was built. + /// Artifact path of the corresponding tool that was built. pub tool_path: PathBuf, /// Compiler used to build the tool. For non-`ToolRustc` tools this is equal to `target_compiler`. /// For `ToolRustc` this is one stage before of the `target_compiler`. @@ -179,8 +187,14 @@ impl Step for ToolBuild { if tool == "tidy" { tool = "rust-tidy"; } - let tool_path = - copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool); + let tool_path = match self.artifact_kind { + ToolArtifactKind::Binary => { + copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool) + } + ToolArtifactKind::Library => builder + .cargo_out(self.compiler, self.mode, self.target) + .join(format!("lib{tool}.rlib")), + }; ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler } } @@ -330,6 +344,7 @@ macro_rules! bootstrap_tool { $(,is_unstable_tool = $unstable:expr)* $(,allow_features = $allow_features:expr)? $(,submodules = $submodules:expr)? + $(,artifact_kind = $artifact_kind:expr)? ; )+) => { #[derive(PartialEq, Eq, Clone)] @@ -389,6 +404,7 @@ macro_rules! bootstrap_tool { builder.require_submodule(submodule, None); } )* + builder.ensure(ToolBuild { compiler: self.compiler, target: self.target, @@ -407,7 +423,12 @@ macro_rules! bootstrap_tool { }, extra_features: vec![], allow_features: concat!($($allow_features)*), - cargo_args: vec![] + cargo_args: vec![], + artifact_kind: if false $(|| $artifact_kind == ToolArtifactKind::Library)* { + ToolArtifactKind::Library + } else { + ToolArtifactKind::Binary + } }) } } @@ -491,6 +512,7 @@ impl Step for RustcPerf { // Only build the collector package, which is used for benchmarking through // a CLI. cargo_args: vec!["-p".to_string(), "collector".to_string()], + artifact_kind: ToolArtifactKind::Binary, }; let res = builder.ensure(tool.clone()); // We also need to symlink the `rustc-fake` binary to the corresponding directory, @@ -548,6 +570,7 @@ impl Step for ErrorIndex { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -583,6 +606,7 @@ impl Step for RemoteTestServer { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -687,6 +711,7 @@ impl Step for Rustdoc { extra_features, allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); // don't create a stage0-sysroot/bin directory. @@ -741,6 +766,7 @@ impl Step for Cargo { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -789,6 +815,7 @@ impl Step for LldWrapper { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target); @@ -849,6 +876,7 @@ impl Step for RustAnalyzer { source_type: SourceType::InTree, allow_features: RustAnalyzer::ALLOW_FEATURES, cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -893,6 +921,7 @@ impl Step for RustAnalyzerProcMacroSrv { source_type: SourceType::InTree, allow_features: RustAnalyzer::ALLOW_FEATURES, cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` @@ -947,6 +976,7 @@ impl Step for LlvmBitcodeLinker { extra_features: self.extra_features, allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); if tool_result.target_compiler.stage > 0 { @@ -1126,6 +1156,7 @@ fn run_tool_build_step( source_type: SourceType::InTree, allow_features: "", cargo_args: vec![], + artifact_kind: ToolArtifactKind::Binary, }); // FIXME: This should just be an if-let-chain, but those are unstable. @@ -1204,6 +1235,7 @@ impl Step for TestFloatParse { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } From 807f6ffa91268f72f738b2b1776a19e8a0a025f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:36:29 +0100 Subject: [PATCH 09/30] Implement `RunMakeSupport` tool using the `bootstrap_tool!` macro --- src/bootstrap/src/core/build_steps/test.rs | 53 ---------------------- src/bootstrap/src/core/build_steps/tool.rs | 1 + 2 files changed, 1 insertion(+), 53 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 709711d9d34c1..efdac694aee2a 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1230,59 +1230,6 @@ macro_rules! test { }; } -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct RunMakeSupport { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RunMakeSupport { - type Output = PathBuf; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); - } - - /// Builds run-make-support and returns the path to the resulting rlib. - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(compile::Std::new(self.compiler, self.target)); - - let cargo = tool::prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolStd, - self.target, - Kind::Build, - "src/tools/run-make-support", - SourceType::InTree, - &[], - ); - - let _guard = builder.msg_tool( - Kind::Build, - Mode::ToolStd, - "run-make-support", - self.compiler.stage, - &self.compiler.host, - &self.target, - ); - cargo.into_cmd().run(builder); - - let lib_name = "librun_make_support.rlib"; - let lib = builder.tools_dir(self.compiler).join(lib_name); - - let cargo_out = builder.cargo_out(self.compiler, Mode::ToolStd, self.target).join(lib_name); - builder.copy_link(&cargo_out, &lib); - lib - } -} - /// Runs `cargo test` on the `src/tools/run-make-support` crate. /// That crate is used by run-make tests. #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index e401a0bdfc842..0149709f5c51b 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -467,6 +467,7 @@ bootstrap_tool!( UnicodeTableGenerator, "src/tools/unicode-table-generator", "unicode-table-generator"; FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump"; OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"]; + RunMakeSupport, "src/tools/run-make-support", "run_make_support", artifact_kind = ToolArtifactKind::Library; ); /// These are the submodules that are required for rustbook to work due to From 4af94f842d37c15a0428e50c10fc2293f3b7a9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:36:45 +0100 Subject: [PATCH 10/30] Implement `RunMake` test suite using the `test!` macro --- src/bootstrap/src/core/build_steps/test.rs | 38 +++------------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index efdac694aee2a..79be42cdd2f2d 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1381,40 +1381,7 @@ test!(Pretty { only_hosts: true, }); -/// Special-handling is needed for `run-make`, so don't use `test!` for defining `RunMake` -/// tests. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RunMake { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RunMake { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/run-make") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); - run.builder.ensure(RunMake { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: "run-make", - suite: "run-make", - path: "tests/run-make", - compare_mode: None, - }); - } -} +test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make", default: true }); test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly", default: true }); @@ -1657,6 +1624,9 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the host: target, }); } + if suite == "run-make" { + builder.tool_exe(Tool::RunMakeSupport); + } // Also provide `rust_test_helpers` for the host. builder.ensure(TestHelpers { target: compiler.host }); From 86aae8e2d626258104cae2725bcdd1a92e572daf Mon Sep 17 00:00:00 2001 From: Ayush Singh Date: Sun, 23 Feb 2025 16:16:10 +0530 Subject: [PATCH 11/30] uefi: Add Service Binding Protocol abstraction - Some UEFI protocols such as TCP4, TCP6, UDP4, UDP6, etc are managed by service binding protocol. - A new instance of such protocols is created and destroyed using the corresponding service binding protocol. - This PR adds abstractions to make using such protocols simpler using Rust Drop trait. - The reason to add these abstractions in a seperate PR from TCP4 Protocol is to make review easier. [EFI_SERVICE_BINDING_PROTCOL](https://uefi.org/specs/UEFI/2.11/11_Protocols_UEFI_Driver_Model.html#efi-service-binding-protocol) Signed-off-by: Ayush Singh --- library/std/src/sys/pal/uefi/helpers.rs | 61 ++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/library/std/src/sys/pal/uefi/helpers.rs b/library/std/src/sys/pal/uefi/helpers.rs index ec2da4e4ee7a4..55f0509896368 100644 --- a/library/std/src/sys/pal/uefi/helpers.rs +++ b/library/std/src/sys/pal/uefi/helpers.rs @@ -10,7 +10,7 @@ //! - More information about protocols can be found [here](https://edk2-docs.gitbook.io/edk-ii-uefi-driver-writer-s-guide/3_foundation/36_protocols_and_handles) use r_efi::efi::{self, Guid}; -use r_efi::protocols::{device_path, device_path_to_text, shell}; +use r_efi::protocols::{device_path, device_path_to_text, service_binding, shell}; use crate::ffi::{OsStr, OsString}; use crate::io::{self, const_error}; @@ -500,3 +500,62 @@ pub(crate) fn get_device_path_from_map(map: &Path) -> io::Result, + child_handle: NonNull, +} + +impl ServiceProtocol { + #[expect(dead_code)] + pub(crate) fn open(service_guid: r_efi::efi::Guid) -> io::Result { + let handles = locate_handles(service_guid)?; + + for handle in handles { + if let Ok(protocol) = open_protocol::(handle, service_guid) { + let Ok(child_handle) = Self::create_child(protocol) else { + continue; + }; + + return Ok(Self { service_guid, handle, child_handle }); + } + } + + Err(io::const_error!(io::ErrorKind::NotFound, "no service binding protocol found")) + } + + #[expect(dead_code)] + pub(crate) fn child_handle(&self) -> NonNull { + self.child_handle + } + + fn create_child( + sbp: NonNull, + ) -> io::Result> { + let mut child_handle: r_efi::efi::Handle = crate::ptr::null_mut(); + // SAFETY: A new handle is allocated if a pointer to NULL is passed. + let r = unsafe { ((*sbp.as_ptr()).create_child)(sbp.as_ptr(), &mut child_handle) }; + + if r.is_error() { + Err(crate::io::Error::from_raw_os_error(r.as_usize())) + } else { + NonNull::new(child_handle) + .ok_or(const_error!(io::ErrorKind::Other, "null child handle")) + } + } +} + +impl Drop for ServiceProtocol { + fn drop(&mut self) { + if let Ok(sbp) = open_protocol::(self.handle, self.service_guid) + { + // SAFETY: Child handle must be allocated by the current service binding protocol. + let _ = unsafe { + ((*sbp.as_ptr()).destroy_child)(sbp.as_ptr(), self.child_handle.as_ptr()) + }; + } + } +} From 0034d6c928803076225867c6fea19a3e2c8e9f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 24 Feb 2025 09:52:53 +0100 Subject: [PATCH 12/30] Compile run-make recipes using the stage0 compiler --- src/bootstrap/src/core/build_steps/test.rs | 5 + src/tools/compiletest/src/common.rs | 3 + src/tools/compiletest/src/lib.rs | 7 ++ src/tools/compiletest/src/runtest/run_make.rs | 119 ++++++------------ .../src/external_deps/rustc.rs | 8 +- .../src/external_deps/rustdoc.rs | 24 +--- src/tools/run-make-support/src/lib.rs | 2 +- src/tools/run-make-support/src/run.rs | 19 +-- src/tools/run-make-support/src/util.rs | 6 +- .../run-make/rustdoc-default-output/rmake.rs | 4 +- .../version-verbose-commit-hash/rmake.rs | 4 +- 11 files changed, 78 insertions(+), 123 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 79be42cdd2f2d..99ab876ccfa5d 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1679,6 +1679,11 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the }; cmd.arg("--cargo-path").arg(cargo_path); + + // We need to pass the compiler that was used to compile run-make-support, + // because we have to use the same compiler to compile rmake.rs recipes. + let stage0_rustc_path = builder.compiler(0, compiler.host); + cmd.arg("--stage0-rustc-path").arg(builder.rustc(stage0_rustc_path)); } // Avoid depending on rustdoc when we don't need it. diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index b302c6a49f58e..978836cb6636a 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -190,6 +190,9 @@ pub struct Config { /// The cargo executable. pub cargo_path: Option, + /// Rustc executable used to compile run-make recipes. + pub stage0_rustc_path: Option, + /// The rustdoc executable. pub rustdoc_path: Option, diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs index 9dff7047bc4ab..e93f32c86bc41 100644 --- a/src/tools/compiletest/src/lib.rs +++ b/src/tools/compiletest/src/lib.rs @@ -54,6 +54,12 @@ pub fn parse_config(args: Vec) -> Config { .reqopt("", "run-lib-path", "path to target shared libraries", "PATH") .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH") .optopt("", "cargo-path", "path to cargo to use for compiling", "PATH") + .optopt( + "", + "stage0-rustc-path", + "path to rustc to use for compiling run-make recipes", + "PATH", + ) .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH") .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH") .reqopt("", "python", "path to python to use for doc tests", "PATH") @@ -320,6 +326,7 @@ pub fn parse_config(args: Vec) -> Config { run_lib_path: make_absolute(opt_path(matches, "run-lib-path")), rustc_path: opt_path(matches, "rustc-path"), cargo_path: matches.opt_str("cargo-path").map(PathBuf::from), + stage0_rustc_path: matches.opt_str("stage0-rustc-path").map(PathBuf::from), rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from), coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from), python: matches.opt_str("python").unwrap(), diff --git a/src/tools/compiletest/src/runtest/run_make.rs b/src/tools/compiletest/src/runtest/run_make.rs index 9bb3993223e21..07f4d7c494265 100644 --- a/src/tools/compiletest/src/runtest/run_make.rs +++ b/src/tools/compiletest/src/runtest/run_make.rs @@ -173,7 +173,8 @@ impl TestCx<'_> { fn run_rmake_v2_test(&self) { // For `run-make` V2, we need to perform 2 steps to build and run a `run-make` V2 recipe // (`rmake.rs`) to run the actual tests. The support library is already built as a tool rust - // library and is available under `build/$TARGET/stageN-tools-bin/librun_make_support.rlib`. + // library and is available under + // `build/$HOST/stage0-bootstrap-tools/$TARGET/release/librun_make_support.rlib`. // // 1. We need to build the recipe `rmake.rs` as a binary and link in the `run_make_support` // library. @@ -224,25 +225,21 @@ impl TestCx<'_> { // // ``` // build// - // ├── stageN-tools-bin/ - // │ └── librun_make_support.rlib // <- support rlib itself - // ├── stageN-tools/ - // │ ├── release/deps/ // <- deps of deps - // │ └── /release/deps/ // <- deps + // ├── stage0-bootstrap-tools/ + // │ ├── /release/librun_make_support.rlib // <- support rlib itself + // │ ├── /release/deps/ // <- deps + // │ └── release/deps/ // <- deps of deps // ``` // // FIXME(jieyouxu): there almost certainly is a better way to do this (specifically how the - // support lib and its deps are organized, can't we copy them to the tools-bin dir as - // well?), but this seems to work for now. + // support lib and its deps are organized), but this seems to work for now. - let stage_number = self.config.stage; + let tools_bin = host_build_root.join("stage0-bootstrap-tools"); + let support_host_path = tools_bin.join(&self.config.host).join("release"); + let support_lib_path = support_host_path.join("librun_make_support.rlib"); - let stage_tools_bin = host_build_root.join(format!("stage{stage_number}-tools-bin")); - let support_lib_path = stage_tools_bin.join("librun_make_support.rlib"); - - let stage_tools = host_build_root.join(format!("stage{stage_number}-tools")); - let support_lib_deps = stage_tools.join(&self.config.host).join("release").join("deps"); - let support_lib_deps_deps = stage_tools.join("release").join("deps"); + let support_lib_deps = support_host_path.join("deps"); + let support_lib_deps_deps = tools_bin.join("release").join("deps"); // To compile the recipe with rustc, we need to provide suitable dynamic library search // paths to rustc. This includes both: @@ -253,12 +250,6 @@ impl TestCx<'_> { let base_dylib_search_paths = Vec::from_iter(env::split_paths(&env::var(dylib_env_var()).unwrap())); - let host_dylib_search_paths = { - let mut paths = vec![self.config.compile_lib_path.clone()]; - paths.extend(base_dylib_search_paths.iter().cloned()); - paths - }; - // Calculate the paths of the recipe binary. As previously discussed, this is placed at // `/` with `bin_name` being `rmake` or `rmake.exe` depending on // platform. @@ -268,7 +259,15 @@ impl TestCx<'_> { p }; - let mut rustc = Command::new(&self.config.rustc_path); + // run-make-support and run-make tests are compiled using the stage0 compiler + // If the stage is 0, then the compiler that we test (either bootstrap or an explicitly + // set compiler) is the one that actually compiled run-make-support. + let stage0_rustc = self + .config + .stage0_rustc_path + .as_ref() + .expect("stage0 rustc is required to run run-make tests"); + let mut rustc = Command::new(&stage0_rustc); rustc .arg("-o") .arg(&recipe_bin) @@ -282,35 +281,12 @@ impl TestCx<'_> { .arg(format!("run_make_support={}", &support_lib_path.to_string_lossy())) .arg("--edition=2021") .arg(&self.testpaths.file.join("rmake.rs")) - .arg("-Cprefer-dynamic") - // Provide necessary library search paths for rustc. - .env(dylib_env_var(), &env::join_paths(host_dylib_search_paths).unwrap()); + .arg("-Cprefer-dynamic"); // In test code we want to be very pedantic about values being silently discarded that are // annotated with `#[must_use]`. rustc.arg("-Dunused_must_use"); - // > `cg_clif` uses `COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0` for running the rustc - // > test suite. With the introduction of rmake.rs this broke. `librun_make_support.rlib` is - // > compiled using the bootstrap rustc wrapper which sets `--sysroot - // > build/aarch64-unknown-linux-gnu/stage0-sysroot`, but then compiletest will compile - // > `rmake.rs` using the sysroot of the bootstrap compiler causing it to not find the - // > `libstd.rlib` against which `librun_make_support.rlib` is compiled. - // - // The gist here is that we have to pass the proper stage0 sysroot if we want - // - // ``` - // $ COMPILETEST_FORCE_STAGE0=1 ./x test run-make --stage 0 - // ``` - // - // to work correctly. - // - // See for more background. - let stage0_sysroot = host_build_root.join("stage0-sysroot"); - if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() { - rustc.arg("--sysroot").arg(&stage0_sysroot); - } - // Now run rustc to build the recipe. let res = self.run_command_to_procres(&mut rustc); if !res.status.success() { @@ -320,35 +296,24 @@ impl TestCx<'_> { // To actually run the recipe, we have to provide the recipe with a bunch of information // provided through env vars. - // Compute stage-specific standard library paths. - let stage_std_path = host_build_root.join(format!("stage{stage_number}")).join("lib"); - // Compute dynamic library search paths for recipes. + // These dylib directories are needed to **execute the recipe**. let recipe_dylib_search_paths = { let mut paths = base_dylib_search_paths.clone(); - - // For stage 0, we need to explicitly include the stage0-sysroot libstd dylib. - // See . - if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() { - paths.push( - stage0_sysroot.join("lib").join("rustlib").join(&self.config.host).join("lib"), - ); - } - - paths.push(support_lib_path.parent().unwrap().to_path_buf()); - paths.push(stage_std_path.join("rustlib").join(&self.config.host).join("lib")); - paths - }; - - // Compute runtime library search paths for recipes. This is target-specific. - let target_runtime_dylib_search_paths = { - let mut paths = vec![rmake_out_dir.clone()]; - paths.extend(base_dylib_search_paths.iter().cloned()); + paths.push( + stage0_rustc + .parent() + .unwrap() + .parent() + .unwrap() + .join("lib") + .join("rustlib") + .join(&self.config.host) + .join("lib"), + ); paths }; - // FIXME(jieyouxu): please rename `TARGET_RPATH_ENV`, `HOST_RPATH_DIR` and - // `TARGET_RPATH_DIR`, it is **extremely** confusing! let mut cmd = Command::new(&recipe_bin); cmd.current_dir(&rmake_out_dir) .stdout(Stdio::piped()) @@ -357,9 +322,14 @@ impl TestCx<'_> { // example, this could be `LD_LIBRARY_PATH` on some linux distros but `PATH` on Windows. .env("LD_LIB_PATH_ENVVAR", dylib_env_var()) // Provide the dylib search paths. + // This is required to run the **recipe** itself. .env(dylib_env_var(), &env::join_paths(recipe_dylib_search_paths).unwrap()) - // Provide runtime dylib search paths. - .env("TARGET_RPATH_ENV", &env::join_paths(target_runtime_dylib_search_paths).unwrap()) + // Provide the directory to libraries that are needed to run the *compiler* invoked + // by the recipe. + .env("HOST_RUSTC_DYLIB_PATH", &self.config.compile_lib_path) + // Provide the directory to libraries that might be needed to run binaries created + // by a compiler invoked by the recipe. + .env("TARGET_EXE_DYLIB_PATH", &self.config.run_lib_path) // Provide the target. .env("TARGET", &self.config.target) // Some tests unfortunately still need Python, so provide path to a Python interpreter. @@ -370,13 +340,6 @@ impl TestCx<'_> { .env("BUILD_ROOT", &host_build_root) // Provide path to stage-corresponding rustc. .env("RUSTC", &self.config.rustc_path) - // Provide the directory to libraries that are needed to run the *compiler*. This is not - // to be confused with `TARGET_RPATH_ENV` or `TARGET_RPATH_DIR`. This is needed if the - // recipe wants to invoke rustc. - .env("HOST_RPATH_DIR", &self.config.compile_lib_path) - // Provide the directory to libraries that might be needed to run compiled binaries - // (further compiled by the recipe!). - .env("TARGET_RPATH_DIR", &self.config.run_lib_path) // Provide which LLVM components are available (e.g. which LLVM components are provided // through a specific CI runner). .env("LLVM_COMPONENTS", &self.config.llvm_components); diff --git a/src/tools/run-make-support/src/external_deps/rustc.rs b/src/tools/run-make-support/src/external_deps/rustc.rs index 710ba025830a8..cb2bd83815a35 100644 --- a/src/tools/run-make-support/src/external_deps/rustc.rs +++ b/src/tools/run-make-support/src/external_deps/rustc.rs @@ -5,7 +5,7 @@ use std::str::FromStr as _; use crate::command::Command; use crate::env::env_var; use crate::path_helpers::cwd; -use crate::util::set_host_rpath; +use crate::util::set_host_compiler_dylib_path; use crate::{is_aix, is_darwin, is_msvc, is_windows, uname}; /// Construct a new `rustc` invocation. This will automatically set the library @@ -15,8 +15,8 @@ pub fn rustc() -> Rustc { Rustc::new() } -/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_rpath`] -/// still presets the environment variable `HOST_RPATH_DIR` by default. +/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_compiler_dylib_path`] +/// still presets the environment variable `HOST_RUSTC_DYLIB_PATH` by default. #[track_caller] pub fn bare_rustc() -> Rustc { Rustc::bare() @@ -44,7 +44,7 @@ pub fn rustc_path() -> String { #[track_caller] fn setup_common() -> Command { let mut cmd = Command::new(rustc_path()); - set_host_rpath(&mut cmd); + set_host_compiler_dylib_path(&mut cmd); cmd } diff --git a/src/tools/run-make-support/src/external_deps/rustdoc.rs b/src/tools/run-make-support/src/external_deps/rustdoc.rs index 3c0e9c82f0bbc..61351ace12c9b 100644 --- a/src/tools/run-make-support/src/external_deps/rustdoc.rs +++ b/src/tools/run-make-support/src/external_deps/rustdoc.rs @@ -2,16 +2,10 @@ use std::ffi::OsStr; use std::path::Path; use crate::command::Command; -use crate::env::{env_var, env_var_os}; -use crate::util::set_host_rpath; +use crate::env::env_var; +use crate::util::set_host_compiler_dylib_path; -/// Construct a plain `rustdoc` invocation with no flags set. -#[track_caller] -pub fn bare_rustdoc() -> Rustdoc { - Rustdoc::bare() -} - -/// Construct a new `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set. +/// Construct a new `rustdoc` invocation. #[track_caller] pub fn rustdoc() -> Rustdoc { Rustdoc::new() @@ -29,23 +23,15 @@ crate::macros::impl_common_helpers!(Rustdoc); fn setup_common() -> Command { let rustdoc = env_var("RUSTDOC"); let mut cmd = Command::new(rustdoc); - set_host_rpath(&mut cmd); + set_host_compiler_dylib_path(&mut cmd); cmd } impl Rustdoc { /// Construct a bare `rustdoc` invocation. #[track_caller] - pub fn bare() -> Self { - let cmd = setup_common(); - Self { cmd } - } - - /// Construct a `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set. - #[track_caller] pub fn new() -> Self { - let mut cmd = setup_common(); - cmd.arg("-L").arg(env_var_os("TARGET_RPATH_DIR")); + let cmd = setup_common(); Self { cmd } } diff --git a/src/tools/run-make-support/src/lib.rs b/src/tools/run-make-support/src/lib.rs index 7e63ab3159ace..d40ec9c4116e7 100644 --- a/src/tools/run-make-support/src/lib.rs +++ b/src/tools/run-make-support/src/lib.rs @@ -67,7 +67,7 @@ pub use llvm::{ }; pub use python::python_command; pub use rustc::{aux_build, bare_rustc, rustc, rustc_path, Rustc}; -pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc}; +pub use rustdoc::{rustdoc, Rustdoc}; /// [`diff`][mod@diff] is implemented in terms of the [similar] library. /// diff --git a/src/tools/run-make-support/src/run.rs b/src/tools/run-make-support/src/run.rs index 3eeba6fd5263d..7812863ccc2c8 100644 --- a/src/tools/run-make-support/src/run.rs +++ b/src/tools/run-make-support/src/run.rs @@ -1,10 +1,10 @@ use std::ffi::OsStr; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::{env, panic}; use crate::command::{Command, CompletedProcess}; -use crate::util::{handle_failed_output, set_host_rpath}; -use crate::{cwd, env_var, is_windows}; +use crate::util::handle_failed_output; +use crate::{cwd, env_var}; #[track_caller] fn run_common(name: &str, args: Option<&[&str]>) -> Command { @@ -18,10 +18,11 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command { cmd.arg(arg); } } + cmd.env(&ld_lib_path_envvar, { let mut paths = vec![]; paths.push(cwd()); - for p in env::split_paths(&env_var("TARGET_RPATH_ENV")) { + for p in env::split_paths(&env_var("TARGET_EXE_DYLIB_PATH")) { paths.push(p.to_path_buf()); } for p in env::split_paths(&env_var(&ld_lib_path_envvar)) { @@ -31,15 +32,6 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command { }); cmd.env("LC_ALL", "C"); // force english locale - if is_windows() { - let mut paths = vec![]; - for p in env::split_paths(&std::env::var("PATH").unwrap_or(String::new())) { - paths.push(p.to_path_buf()); - } - paths.push(Path::new(&env_var("TARGET_RPATH_DIR")).to_path_buf()); - cmd.env("PATH", env::join_paths(paths.iter()).unwrap()); - } - cmd } @@ -84,7 +76,6 @@ pub fn run_fail(name: &str) -> CompletedProcess { #[track_caller] pub fn cmd>(program: S) -> Command { let mut command = Command::new(program); - set_host_rpath(&mut command); command.env("LC_ALL", "C"); // force english locale command } diff --git a/src/tools/run-make-support/src/util.rs b/src/tools/run-make-support/src/util.rs index 703e3ad1c6cd2..af01758447b93 100644 --- a/src/tools/run-make-support/src/util.rs +++ b/src/tools/run-make-support/src/util.rs @@ -24,13 +24,13 @@ pub(crate) fn handle_failed_output( std::process::exit(1) } -/// Set the runtime library path as needed for running the host rustc/rustdoc/etc. -pub(crate) fn set_host_rpath(cmd: &mut Command) { +/// Set the runtime library paths as needed for running the host compilers (rustc/rustdoc/etc). +pub(crate) fn set_host_compiler_dylib_path(cmd: &mut Command) { let ld_lib_path_envvar = env_var("LD_LIB_PATH_ENVVAR"); cmd.env(&ld_lib_path_envvar, { let mut paths = vec![]; paths.push(cwd()); - paths.push(PathBuf::from(env_var("HOST_RPATH_DIR"))); + paths.push(PathBuf::from(env_var("HOST_RUSTC_DYLIB_PATH"))); for p in std::env::split_paths(&env_var(&ld_lib_path_envvar)) { paths.push(p.to_path_buf()); } diff --git a/tests/run-make/rustdoc-default-output/rmake.rs b/tests/run-make/rustdoc-default-output/rmake.rs index 06720445a35ce..5f9c501e5286b 100644 --- a/tests/run-make/rustdoc-default-output/rmake.rs +++ b/tests/run-make/rustdoc-default-output/rmake.rs @@ -3,10 +3,10 @@ // ensures the output of rustdoc's help menu is as expected. // See https://github.com/rust-lang/rust/issues/88756 -use run_make_support::{bare_rustdoc, diff}; +use run_make_support::{diff, rustdoc}; fn main() { - let out = bare_rustdoc().run().stdout_utf8(); + let out = rustdoc().run().stdout_utf8(); diff() .expected_file("output-default.stdout") .actual_text("actual", out) diff --git a/tests/run-make/version-verbose-commit-hash/rmake.rs b/tests/run-make/version-verbose-commit-hash/rmake.rs index 733c0e2cdb14f..ae8dda38ab418 100644 --- a/tests/run-make/version-verbose-commit-hash/rmake.rs +++ b/tests/run-make/version-verbose-commit-hash/rmake.rs @@ -5,13 +5,13 @@ //@ needs-git-hash -use run_make_support::{bare_rustc, bare_rustdoc, regex}; +use run_make_support::{bare_rustc, regex, rustdoc}; fn main() { let out_rustc = bare_rustc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); let out_rustdoc = - bare_rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); + rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); let re = regex::Regex::new(r#"commit-hash: [0-9a-f]{40}\ncommit-date: [0-9]{4}-[0-9]{2}-[0-9]{2}"#) .unwrap(); From 9d6ca5f286dac25ac168aa77709f5a0cbd40d30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 25 Feb 2025 11:10:59 +0100 Subject: [PATCH 13/30] Ignore a-b-a-linker-guard during cross-compilation --- tests/run-make/a-b-a-linker-guard/rmake.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/run-make/a-b-a-linker-guard/rmake.rs b/tests/run-make/a-b-a-linker-guard/rmake.rs index ee6d655bc760c..4428685a6cf16 100644 --- a/tests/run-make/a-b-a-linker-guard/rmake.rs +++ b/tests/run-make/a-b-a-linker-guard/rmake.rs @@ -1,7 +1,8 @@ -// ignore-tidy-linelength +// Test that if we build `b` against a version of `a` that has +// one set of types, it will not run with a dylib that has a different set of types. -// Test that if we build `b` against a version of `a` that has one set of types, it will not run -// with a dylib that has a different set of types. +//@ ignore-cross-compile +// Reason: the compiled binary is executed use run_make_support::{run, run_fail, rustc}; From a0ed304c2101b4efdaf9982189e8d6ca73690712 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 9 Dec 2024 09:25:21 +0000 Subject: [PATCH 14/30] float: Update some constants to `pub(crate)` These constants can be useful outside of their current module. Make them `pub(crate)` to allow for this. --- library/core/src/num/f32.rs | 6 +++--- library/core/src/num/f64.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index a200fd5318669..de1557ccc9028 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -493,13 +493,13 @@ impl f32 { pub const NEG_INFINITY: f32 = -1.0_f32 / 0.0_f32; /// Sign bit - const SIGN_MASK: u32 = 0x8000_0000; + pub(crate) const SIGN_MASK: u32 = 0x8000_0000; /// Exponent mask - const EXP_MASK: u32 = 0x7f80_0000; + pub(crate) const EXP_MASK: u32 = 0x7f80_0000; /// Mantissa mask - const MAN_MASK: u32 = 0x007f_ffff; + pub(crate) const MAN_MASK: u32 = 0x007f_ffff; /// Minimum representable positive value (min subnormal) const TINY_BITS: u32 = 0x1; diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index de63a462b61ac..65b5f3b9af093 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -492,13 +492,13 @@ impl f64 { pub const NEG_INFINITY: f64 = -1.0_f64 / 0.0_f64; /// Sign bit - const SIGN_MASK: u64 = 0x8000_0000_0000_0000; + pub(crate) const SIGN_MASK: u64 = 0x8000_0000_0000_0000; /// Exponent mask - const EXP_MASK: u64 = 0x7ff0_0000_0000_0000; + pub(crate) const EXP_MASK: u64 = 0x7ff0_0000_0000_0000; /// Mantissa mask - const MAN_MASK: u64 = 0x000f_ffff_ffff_ffff; + pub(crate) const MAN_MASK: u64 = 0x000f_ffff_ffff_ffff; /// Minimum representable positive value (min subnormal) const TINY_BITS: u64 = 0x1; From 5a2da96a44abad6be752c7ee1cac173aa1ca2f7c Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 9 Dec 2024 09:25:22 +0000 Subject: [PATCH 15/30] dec2flt: Update documentation of existing methods Fix or elaborate existing float parsing documentation. This includes introducing a convention that should make naming more consistent. --- library/core/src/num/dec2flt/common.rs | 14 ++++++++------ library/core/src/num/dec2flt/decimal.rs | 22 +++++++++++++--------- library/core/src/num/dec2flt/mod.rs | 16 ++++++++++++++-- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/library/core/src/num/dec2flt/common.rs b/library/core/src/num/dec2flt/common.rs index 4dadf406ae8c7..1646d3a95d3b1 100644 --- a/library/core/src/num/dec2flt/common.rs +++ b/library/core/src/num/dec2flt/common.rs @@ -8,12 +8,12 @@ pub(crate) trait ByteSlice { /// Writes a 64-bit integer as 8 bytes in little-endian order. fn write_u64(&mut self, value: u64); - /// Calculate the offset of a slice from another. + /// Calculate the difference in length between two slices. fn offset_from(&self, other: &Self) -> isize; /// Iteratively parse and consume digits from bytes. - /// Returns the same bytes with consumed digits being - /// elided. + /// + /// Returns the same bytes with consumed digits being elided. Breaks on invalid digits. fn parse_digits(&self, func: impl FnMut(u8)) -> &Self; } @@ -39,11 +39,11 @@ impl ByteSlice for [u8] { fn parse_digits(&self, mut func: impl FnMut(u8)) -> &Self { let mut s = self; - while let Some((c, s_next)) = s.split_first() { + while let Some((c, rest)) = s.split_first() { let c = c.wrapping_sub(b'0'); if c < 10 { func(c); - s = s_next; + s = rest; } else { break; } @@ -53,7 +53,9 @@ impl ByteSlice for [u8] { } } -/// Determine if 8 bytes are all decimal digits. +/// Determine if all characters in an 8-byte byte string (represented as a `u64`) are all decimal +/// digits. +/// /// This does not care about the order in which the bytes were loaded. pub(crate) fn is_8digits(v: u64) -> bool { let a = v.wrapping_add(0x4646_4646_4646_4646); diff --git a/library/core/src/num/dec2flt/decimal.rs b/library/core/src/num/dec2flt/decimal.rs index b37724ba62d5e..a84e11ec8e710 100644 --- a/library/core/src/num/dec2flt/decimal.rs +++ b/library/core/src/num/dec2flt/decimal.rs @@ -1,4 +1,4 @@ -//! Arbitrary-precision decimal class for fallback algorithms. +//! Arbitrary-precision decimal type used by fallback algorithms. //! //! This is only used if the fast-path (native floats) and //! the Eisel-Lemire algorithm are unable to unambiguously @@ -11,6 +11,7 @@ use crate::num::dec2flt::common::{ByteSlice, is_8digits}; +/// A decimal floating-point number. #[derive(Clone)] pub(super) struct Decimal { /// The number of significant digits in the decimal. @@ -30,18 +31,17 @@ impl Default for Decimal { } impl Decimal { - /// The maximum number of digits required to unambiguously round a float. + /// The maximum number of digits required to unambiguously round up to a 64-bit float. /// - /// For a double-precision IEEE 754 float, this required 767 digits, - /// so we store the max digits + 1. + /// For an IEEE 754 binary64 float, this required 767 digits. So we store the max digits + 1. /// /// We can exactly represent a float in radix `b` from radix 2 if /// `b` is divisible by 2. This function calculates the exact number of /// digits required to exactly represent that float. /// /// According to the "Handbook of Floating Point Arithmetic", - /// for IEEE754, with emin being the min exponent, p2 being the - /// precision, and b being the radix, the number of digits follows as: + /// for IEEE754, with `emin` being the min exponent, `p2` being the + /// precision, and `b` being the radix, the number of digits follows as: /// /// `−emin + p2 + ⌊(emin + 1) log(2, b) − log(1 − 2^(−p2), b)⌋` /// @@ -56,11 +56,14 @@ impl Decimal { /// In Python: /// `-emin + p2 + math.floor((emin+ 1)*math.log(2, b)-math.log(1-2**(-p2), b))` pub(super) const MAX_DIGITS: usize = 768; - /// The max digits that can be exactly represented in a 64-bit integer. + /// The max decimal digits that can be exactly represented in a 64-bit integer. pub(super) const MAX_DIGITS_WITHOUT_OVERFLOW: usize = 19; pub(super) const DECIMAL_POINT_RANGE: i32 = 2047; - /// Append a digit to the buffer. + /// Append a digit to the buffer if it fits. + // FIXME(tgross35): it may be better for this to return an option + // FIXME(tgross35): incrementing the digit counter even if we don't push anything + // seems incorrect. pub(super) fn try_add_digit(&mut self, digit: u8) { if self.num_digits < Self::MAX_DIGITS { self.digits[self.num_digits] = digit; @@ -69,6 +72,7 @@ impl Decimal { } /// Trim trailing zeros from the buffer. + // FIXME(tgross35): this could be `.rev().position()` if perf is okay pub(super) fn trim(&mut self) { // All of the following calls to `Decimal::trim` can't panic because: // @@ -86,7 +90,7 @@ impl Decimal { pub(super) fn round(&self) -> u64 { if self.num_digits == 0 || self.decimal_point < 0 { return 0; - } else if self.decimal_point > 18 { + } else if self.decimal_point >= Self::MAX_DIGITS_WITHOUT_OVERFLOW as i32 { return 0xFFFF_FFFF_FFFF_FFFF_u64; } let dp = self.decimal_point as usize; diff --git a/library/core/src/num/dec2flt/mod.rs b/library/core/src/num/dec2flt/mod.rs index 6dca740684537..91bfe1bef2e77 100644 --- a/library/core/src/num/dec2flt/mod.rs +++ b/library/core/src/num/dec2flt/mod.rs @@ -3,8 +3,8 @@ //! # Problem statement //! //! We are given a decimal string such as `12.34e56`. This string consists of integral (`12`), -//! fractional (`34`), and exponent (`56`) parts. All parts are optional and interpreted as zero -//! when missing. +//! fractional (`34`), and exponent (`56`) parts. All parts are optional and interpreted as a +//! default value (1 or 0) when missing. //! //! We seek the IEEE 754 floating point number that is closest to the exact value of the decimal //! string. It is well-known that many decimal strings do not have terminating representations in @@ -67,6 +67,18 @@ //! "such that the exponent +/- the number of decimal digits fits into a 64 bit integer". //! Larger exponents are accepted, but we don't do arithmetic with them, they are immediately //! turned into {positive,negative} {zero,infinity}. +//! +//! # Notation +//! +//! This module uses the same notation as the Lemire paper: +//! +//! - `m`: binary mantissa; always nonnegative +//! - `p`: binary exponent; a signed integer +//! - `w`: decimal significand; always nonnegative +//! - `q`: decimal exponent; a signed integer +//! +//! This gives `m * 2^p` for the binary floating-point number, with `w * 10^q` as the decimal +//! equivalent. #![doc(hidden)] #![unstable( From 49a2d4c757e27f915d15664ba92bbe0d3f590a8e Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 9 Dec 2024 09:25:22 +0000 Subject: [PATCH 16/30] dec2flt: Rename `Decimal` to `DecimalSeq` This module currently contains two decimal types, `Decimal` and `Number`. These names don't provide a whole lot of insight into what exactly they are, and `Number` is actually the one that is more like an expected `Decimal` type. In accordance with this, rename the existing `Decimal` to `DecimalSeq`. This highlights that it contains a sequence of decimal digits, rather than representing a base-10 floating point (decimal) number. Additionally, add some tests to validate internal behavior. --- .../dec2flt/{decimal.rs => decimal_seq.rs} | 51 ++++++++++++------- library/core/src/num/dec2flt/mod.rs | 2 +- library/core/src/num/dec2flt/slow.rs | 8 +-- .../tests/num/dec2flt/decimal_seq.rs | 30 +++++++++++ library/coretests/tests/num/dec2flt/mod.rs | 1 + 5 files changed, 70 insertions(+), 22 deletions(-) rename library/core/src/num/dec2flt/{decimal.rs => decimal_seq.rs} (94%) create mode 100644 library/coretests/tests/num/dec2flt/decimal_seq.rs diff --git a/library/core/src/num/dec2flt/decimal.rs b/library/core/src/num/dec2flt/decimal_seq.rs similarity index 94% rename from library/core/src/num/dec2flt/decimal.rs rename to library/core/src/num/dec2flt/decimal_seq.rs index a84e11ec8e710..de22280c001c9 100644 --- a/library/core/src/num/dec2flt/decimal.rs +++ b/library/core/src/num/dec2flt/decimal_seq.rs @@ -11,9 +11,9 @@ use crate::num::dec2flt::common::{ByteSlice, is_8digits}; -/// A decimal floating-point number. -#[derive(Clone)] -pub(super) struct Decimal { +/// A decimal floating-point number, represented as a sequence of decimal digits. +#[derive(Clone, Debug, PartialEq)] +pub struct DecimalSeq { /// The number of significant digits in the decimal. pub num_digits: usize, /// The offset of the decimal point in the significant digits. @@ -24,13 +24,13 @@ pub(super) struct Decimal { pub digits: [u8; Self::MAX_DIGITS], } -impl Default for Decimal { +impl Default for DecimalSeq { fn default() -> Self { Self { num_digits: 0, decimal_point: 0, truncated: false, digits: [0; Self::MAX_DIGITS] } } } -impl Decimal { +impl DecimalSeq { /// The maximum number of digits required to unambiguously round up to a 64-bit float. /// /// For an IEEE 754 binary64 float, this required 767 digits. So we store the max digits + 1. @@ -55,7 +55,8 @@ impl Decimal { /// /// In Python: /// `-emin + p2 + math.floor((emin+ 1)*math.log(2, b)-math.log(1-2**(-p2), b))` - pub(super) const MAX_DIGITS: usize = 768; + pub const MAX_DIGITS: usize = 768; + /// The max decimal digits that can be exactly represented in a 64-bit integer. pub(super) const MAX_DIGITS_WITHOUT_OVERFLOW: usize = 19; pub(super) const DECIMAL_POINT_RANGE: i32 = 2047; @@ -73,12 +74,12 @@ impl Decimal { /// Trim trailing zeros from the buffer. // FIXME(tgross35): this could be `.rev().position()` if perf is okay - pub(super) fn trim(&mut self) { - // All of the following calls to `Decimal::trim` can't panic because: + pub fn trim(&mut self) { + // All of the following calls to `DecimalSeq::trim` can't panic because: // - // 1. `parse_decimal` sets `num_digits` to a max of `Decimal::MAX_DIGITS`. + // 1. `parse_decimal` sets `num_digits` to a max of `DecimalSeq::MAX_DIGITS`. // 2. `right_shift` sets `num_digits` to `write_index`, which is bounded by `num_digits`. - // 3. `left_shift` `num_digits` to a max of `Decimal::MAX_DIGITS`. + // 3. `left_shift` `num_digits` to a max of `DecimalSeq::MAX_DIGITS`. // // Trim is only called in `right_shift` and `left_shift`. debug_assert!(self.num_digits <= Self::MAX_DIGITS); @@ -93,21 +94,26 @@ impl Decimal { } else if self.decimal_point >= Self::MAX_DIGITS_WITHOUT_OVERFLOW as i32 { return 0xFFFF_FFFF_FFFF_FFFF_u64; } + let dp = self.decimal_point as usize; let mut n = 0_u64; + for i in 0..dp { n *= 10; if i < self.num_digits { n += self.digits[i] as u64; } } + let mut round_up = false; + if dp < self.num_digits { round_up = self.digits[dp] >= 5; if self.digits[dp] == 5 && dp + 1 == self.num_digits { round_up = self.truncated || ((dp != 0) && (1 & self.digits[dp - 1] != 0)) } } + if round_up { n += 1; } @@ -123,6 +129,7 @@ impl Decimal { let mut read_index = self.num_digits; let mut write_index = self.num_digits + num_new_digits; let mut n = 0_u64; + while read_index != 0 { read_index -= 1; write_index -= 1; @@ -136,6 +143,7 @@ impl Decimal { } n = quotient; } + while n > 0 { write_index -= 1; let quotient = n / 10; @@ -147,10 +155,13 @@ impl Decimal { } n = quotient; } + self.num_digits += num_new_digits; + if self.num_digits > Self::MAX_DIGITS { self.num_digits = Self::MAX_DIGITS; } + self.decimal_point += num_new_digits as i32; self.trim(); } @@ -206,8 +217,8 @@ impl Decimal { } /// Parse a big integer representation of the float as a decimal. -pub(super) fn parse_decimal(mut s: &[u8]) -> Decimal { - let mut d = Decimal::default(); +pub fn parse_decimal_seq(mut s: &[u8]) -> DecimalSeq { + let mut d = DecimalSeq::default(); let start = s; while let Some((&b'0', s_next)) = s.split_first() { @@ -225,7 +236,7 @@ pub(super) fn parse_decimal(mut s: &[u8]) -> Decimal { s = s_next; } } - while s.len() >= 8 && d.num_digits + 8 < Decimal::MAX_DIGITS { + while s.len() >= 8 && d.num_digits + 8 < DecimalSeq::MAX_DIGITS { let v = s.read_u64(); if !is_8digits(v) { break; @@ -237,6 +248,7 @@ pub(super) fn parse_decimal(mut s: &[u8]) -> Decimal { s = s.parse_digits(|digit| d.try_add_digit(digit)); d.decimal_point = s.len() as i32 - first.len() as i32; } + if d.num_digits != 0 { // Ignore the trailing zeros if there are any let mut n_trailing_zeros = 0; @@ -250,11 +262,12 @@ pub(super) fn parse_decimal(mut s: &[u8]) -> Decimal { d.decimal_point += n_trailing_zeros as i32; d.num_digits -= n_trailing_zeros; d.decimal_point += d.num_digits as i32; - if d.num_digits > Decimal::MAX_DIGITS { + if d.num_digits > DecimalSeq::MAX_DIGITS { d.truncated = true; - d.num_digits = Decimal::MAX_DIGITS; + d.num_digits = DecimalSeq::MAX_DIGITS; } } + if let Some((&ch, s_next)) = s.split_first() { if ch == b'e' || ch == b'E' { s = s_next; @@ -276,13 +289,15 @@ pub(super) fn parse_decimal(mut s: &[u8]) -> Decimal { d.decimal_point += if neg_exp { -exp_num } else { exp_num }; } } - for i in d.num_digits..Decimal::MAX_DIGITS_WITHOUT_OVERFLOW { + + for i in d.num_digits..DecimalSeq::MAX_DIGITS_WITHOUT_OVERFLOW { d.digits[i] = 0; } + d } -fn number_of_digits_decimal_left_shift(d: &Decimal, mut shift: usize) -> usize { +fn number_of_digits_decimal_left_shift(d: &DecimalSeq, mut shift: usize) -> usize { #[rustfmt::skip] const TABLE: [u16; 65] = [ 0x0000, 0x0800, 0x0801, 0x0803, 0x1006, 0x1009, 0x100D, 0x1812, 0x1817, 0x181D, 0x2024, @@ -347,6 +362,7 @@ fn number_of_digits_decimal_left_shift(d: &Decimal, mut shift: usize) -> usize { let pow5_a = (0x7FF & x_a) as usize; let pow5_b = (0x7FF & x_b) as usize; let pow5 = &TABLE_POW5[pow5_a..]; + for (i, &p5) in pow5.iter().enumerate().take(pow5_b - pow5_a) { if i >= d.num_digits { return num_new_digits - 1; @@ -358,5 +374,6 @@ fn number_of_digits_decimal_left_shift(d: &Decimal, mut shift: usize) -> usize { return num_new_digits; } } + num_new_digits } diff --git a/library/core/src/num/dec2flt/mod.rs b/library/core/src/num/dec2flt/mod.rs index 91bfe1bef2e77..471505d249403 100644 --- a/library/core/src/num/dec2flt/mod.rs +++ b/library/core/src/num/dec2flt/mod.rs @@ -97,7 +97,7 @@ use crate::fmt; use crate::str::FromStr; mod common; -mod decimal; +pub mod decimal_seq; mod fpu; mod slow; mod table; diff --git a/library/core/src/num/dec2flt/slow.rs b/library/core/src/num/dec2flt/slow.rs index 85d4b13284b7d..e274e1aa0aa1b 100644 --- a/library/core/src/num/dec2flt/slow.rs +++ b/library/core/src/num/dec2flt/slow.rs @@ -1,7 +1,7 @@ //! Slow, fallback algorithm for cases the Eisel-Lemire algorithm cannot round. use crate::num::dec2flt::common::BiasedFp; -use crate::num::dec2flt::decimal::{Decimal, parse_decimal}; +use crate::num::dec2flt::decimal_seq::{DecimalSeq, parse_decimal_seq}; use crate::num::dec2flt::float::RawFloat; /// Parse the significant digits and biased, binary exponent of a float. @@ -36,7 +36,7 @@ pub(crate) fn parse_long_mantissa(s: &[u8]) -> BiasedFp { let fp_zero = BiasedFp::zero_pow2(0); let fp_inf = BiasedFp::zero_pow2(F::INFINITE_POWER); - let mut d = parse_decimal(s); + let mut d = parse_decimal_seq(s); // Short-circuit if the value can only be a literal 0 or infinity. if d.num_digits == 0 || d.decimal_point < -324 { @@ -50,7 +50,7 @@ pub(crate) fn parse_long_mantissa(s: &[u8]) -> BiasedFp { let n = d.decimal_point as usize; let shift = get_shift(n); d.right_shift(shift); - if d.decimal_point < -Decimal::DECIMAL_POINT_RANGE { + if d.decimal_point < -DecimalSeq::DECIMAL_POINT_RANGE { return fp_zero; } exp2 += shift as i32; @@ -67,7 +67,7 @@ pub(crate) fn parse_long_mantissa(s: &[u8]) -> BiasedFp { get_shift((-d.decimal_point) as _) }; d.left_shift(shift); - if d.decimal_point > Decimal::DECIMAL_POINT_RANGE { + if d.decimal_point > DecimalSeq::DECIMAL_POINT_RANGE { return fp_inf; } exp2 -= shift as i32; diff --git a/library/coretests/tests/num/dec2flt/decimal_seq.rs b/library/coretests/tests/num/dec2flt/decimal_seq.rs new file mode 100644 index 0000000000000..f46ba7c465a6e --- /dev/null +++ b/library/coretests/tests/num/dec2flt/decimal_seq.rs @@ -0,0 +1,30 @@ +use core::num::dec2flt::decimal_seq::{DecimalSeq, parse_decimal_seq}; + +#[test] +fn test_trim() { + let mut dec = DecimalSeq::default(); + let digits = [1, 2, 3, 4]; + + dec.digits[0..4].copy_from_slice(&digits); + dec.num_digits = 8; + dec.trim(); + + assert_eq!(dec.digits[0..4], digits); + assert_eq!(dec.num_digits, 4); +} + +#[test] +fn test_parse() { + let tests = [("1.234", [1, 2, 3, 4], 1)]; + + for (s, exp_digits, decimal_point) in tests { + let actual = parse_decimal_seq(s.as_bytes()); + let mut digits = [0; DecimalSeq::MAX_DIGITS]; + digits[..exp_digits.len()].copy_from_slice(&exp_digits); + + let expected = + DecimalSeq { num_digits: exp_digits.len(), decimal_point, truncated: false, digits }; + + assert_eq!(actual, expected); + } +} diff --git a/library/coretests/tests/num/dec2flt/mod.rs b/library/coretests/tests/num/dec2flt/mod.rs index 874e0ec7093c7..51f3017ddc79c 100644 --- a/library/coretests/tests/num/dec2flt/mod.rs +++ b/library/coretests/tests/num/dec2flt/mod.rs @@ -1,5 +1,6 @@ #![allow(overflowing_literals)] +mod decimal_seq; mod float; mod lemire; mod parse; From 626d2c5eed687e2c166d25f29cdfcaaba48f5f23 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 9 Dec 2024 09:25:22 +0000 Subject: [PATCH 17/30] dec2flt: Rename `Number` to `Decimal` The previous commit renamed `Decimal` to `DecimalSeq`. Now, rename the type that represents a decimal floating point number to be `Decimal`. Additionally, add some tests for internal behavior. --- .../src/num/dec2flt/{number.rs => decimal.rs} | 6 ++-- library/core/src/num/dec2flt/mod.rs | 2 +- library/core/src/num/dec2flt/parse.rs | 10 +++---- .../coretests/tests/num/dec2flt/decimal.rs | 28 +++++++++++++++++++ library/coretests/tests/num/dec2flt/mod.rs | 1 + library/coretests/tests/num/dec2flt/parse.rs | 26 ++++++++--------- 6 files changed, 51 insertions(+), 22 deletions(-) rename library/core/src/num/dec2flt/{number.rs => decimal.rs} (96%) create mode 100644 library/coretests/tests/num/dec2flt/decimal.rs diff --git a/library/core/src/num/dec2flt/number.rs b/library/core/src/num/dec2flt/decimal.rs similarity index 96% rename from library/core/src/num/dec2flt/number.rs rename to library/core/src/num/dec2flt/decimal.rs index 2538991564ae4..1ba9150a0119d 100644 --- a/library/core/src/num/dec2flt/number.rs +++ b/library/core/src/num/dec2flt/decimal.rs @@ -3,7 +3,6 @@ use crate::num::dec2flt::float::RawFloat; use crate::num::dec2flt::fpu::set_precision; -#[rustfmt::skip] const INT_POW10: [u64; 16] = [ 1, 10, @@ -23,15 +22,16 @@ const INT_POW10: [u64; 16] = [ 1000000000000000, ]; +/// A floating point number with up to 64 bits of mantissa and an `i64` exponent. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct Number { +pub struct Decimal { pub exponent: i64, pub mantissa: u64, pub negative: bool, pub many_digits: bool, } -impl Number { +impl Decimal { /// Detect if the float can be accurately reconstructed from native floats. #[inline] fn is_fast_path(&self) -> bool { diff --git a/library/core/src/num/dec2flt/mod.rs b/library/core/src/num/dec2flt/mod.rs index 471505d249403..a82c858df4998 100644 --- a/library/core/src/num/dec2flt/mod.rs +++ b/library/core/src/num/dec2flt/mod.rs @@ -97,6 +97,7 @@ use crate::fmt; use crate::str::FromStr; mod common; +pub mod decimal; pub mod decimal_seq; mod fpu; mod slow; @@ -104,7 +105,6 @@ mod table; // float is used in flt2dec, and all are used in unit tests. pub mod float; pub mod lemire; -pub mod number; pub mod parse; macro_rules! from_str_float_impl { diff --git a/library/core/src/num/dec2flt/parse.rs b/library/core/src/num/dec2flt/parse.rs index 06ee8e95fbc47..e38fedc58bec0 100644 --- a/library/core/src/num/dec2flt/parse.rs +++ b/library/core/src/num/dec2flt/parse.rs @@ -1,8 +1,8 @@ //! Functions to parse floating-point numbers. use crate::num::dec2flt::common::{ByteSlice, is_8digits}; +use crate::num::dec2flt::decimal::Decimal; use crate::num::dec2flt::float::RawFloat; -use crate::num::dec2flt::number::Number; const MIN_19DIGIT_INT: u64 = 100_0000_0000_0000_0000; @@ -100,7 +100,7 @@ fn parse_scientific(s_ref: &mut &[u8]) -> Option { /// /// This creates a representation of the float as the /// significant digits and the decimal exponent. -fn parse_partial_number(mut s: &[u8]) -> Option<(Number, usize)> { +fn parse_partial_number(mut s: &[u8]) -> Option<(Decimal, usize)> { debug_assert!(!s.is_empty()); // parse initial digits before dot @@ -146,7 +146,7 @@ fn parse_partial_number(mut s: &[u8]) -> Option<(Number, usize)> { // handle uncommon case with many digits if n_digits <= 19 { - return Some((Number { exponent, mantissa, negative: false, many_digits: false }, len)); + return Some((Decimal { exponent, mantissa, negative: false, many_digits: false }, len)); } n_digits -= 19; @@ -179,13 +179,13 @@ fn parse_partial_number(mut s: &[u8]) -> Option<(Number, usize)> { exponent += exp_number; } - Some((Number { exponent, mantissa, negative: false, many_digits }, len)) + Some((Decimal { exponent, mantissa, negative: false, many_digits }, len)) } /// Try to parse a non-special floating point number, /// as well as two slices with integer and fractional parts /// and the parsed exponent. -pub fn parse_number(s: &[u8]) -> Option { +pub fn parse_number(s: &[u8]) -> Option { if let Some((float, rest)) = parse_partial_number(s) { if rest == s.len() { return Some(float); diff --git a/library/coretests/tests/num/dec2flt/decimal.rs b/library/coretests/tests/num/dec2flt/decimal.rs new file mode 100644 index 0000000000000..1fa06de692e07 --- /dev/null +++ b/library/coretests/tests/num/dec2flt/decimal.rs @@ -0,0 +1,28 @@ +use core::num::dec2flt::decimal::Decimal; + +type FPath = ((i64, u64, bool, bool), Option); + +const FPATHS_F32: &[FPath] = + &[((0, 0, false, false), Some(0.0)), ((0, 0, false, false), Some(0.0))]; +const FPATHS_F64: &[FPath] = + &[((0, 0, false, false), Some(0.0)), ((0, 0, false, false), Some(0.0))]; + +#[test] +fn check_fast_path_f32() { + for ((exponent, mantissa, negative, many_digits), expected) in FPATHS_F32.iter().copied() { + let dec = Decimal { exponent, mantissa, negative, many_digits }; + let actual = dec.try_fast_path::(); + + assert_eq!(actual, expected); + } +} + +#[test] +fn check_fast_path_f64() { + for ((exponent, mantissa, negative, many_digits), expected) in FPATHS_F64.iter().copied() { + let dec = Decimal { exponent, mantissa, negative, many_digits }; + let actual = dec.try_fast_path::(); + + assert_eq!(actual, expected); + } +} diff --git a/library/coretests/tests/num/dec2flt/mod.rs b/library/coretests/tests/num/dec2flt/mod.rs index 51f3017ddc79c..a9025be5ca7f1 100644 --- a/library/coretests/tests/num/dec2flt/mod.rs +++ b/library/coretests/tests/num/dec2flt/mod.rs @@ -1,5 +1,6 @@ #![allow(overflowing_literals)] +mod decimal; mod decimal_seq; mod float; mod lemire; diff --git a/library/coretests/tests/num/dec2flt/parse.rs b/library/coretests/tests/num/dec2flt/parse.rs index 4a5d24ba7d5fa..ec705a61e13ca 100644 --- a/library/coretests/tests/num/dec2flt/parse.rs +++ b/library/coretests/tests/num/dec2flt/parse.rs @@ -1,9 +1,9 @@ -use core::num::dec2flt::number::Number; +use core::num::dec2flt::decimal::Decimal; use core::num::dec2flt::parse::parse_number; use core::num::dec2flt::{dec2flt, pfe_invalid}; -fn new_number(e: i64, m: u64) -> Number { - Number { exponent: e, mantissa: m, negative: false, many_digits: false } +fn new_dec(e: i64, m: u64) -> Decimal { + Decimal { exponent: e, mantissa: m, negative: false, many_digits: false } } #[test] @@ -31,23 +31,23 @@ fn invalid_chars() { } } -fn parse_positive(s: &[u8]) -> Option { +fn parse_positive(s: &[u8]) -> Option { parse_number(s) } #[test] fn valid() { - assert_eq!(parse_positive(b"123.456e789"), Some(new_number(786, 123456))); - assert_eq!(parse_positive(b"123.456e+789"), Some(new_number(786, 123456))); - assert_eq!(parse_positive(b"123.456e-789"), Some(new_number(-792, 123456))); - assert_eq!(parse_positive(b".050"), Some(new_number(-3, 50))); - assert_eq!(parse_positive(b"999"), Some(new_number(0, 999))); - assert_eq!(parse_positive(b"1.e300"), Some(new_number(300, 1))); - assert_eq!(parse_positive(b".1e300"), Some(new_number(299, 1))); - assert_eq!(parse_positive(b"101e-33"), Some(new_number(-33, 101))); + assert_eq!(parse_positive(b"123.456e789"), Some(new_dec(786, 123456))); + assert_eq!(parse_positive(b"123.456e+789"), Some(new_dec(786, 123456))); + assert_eq!(parse_positive(b"123.456e-789"), Some(new_dec(-792, 123456))); + assert_eq!(parse_positive(b".050"), Some(new_dec(-3, 50))); + assert_eq!(parse_positive(b"999"), Some(new_dec(0, 999))); + assert_eq!(parse_positive(b"1.e300"), Some(new_dec(300, 1))); + assert_eq!(parse_positive(b".1e300"), Some(new_dec(299, 1))); + assert_eq!(parse_positive(b"101e-33"), Some(new_dec(-33, 101))); let zeros = "0".repeat(25); let s = format!("1.5e{zeros}"); - assert_eq!(parse_positive(s.as_bytes()), Some(new_number(-1, 15))); + assert_eq!(parse_positive(s.as_bytes()), Some(new_dec(-1, 15))); } macro_rules! assert_float_result_bits_eq { From 6c34daff57101922dc97e1860bf5940ea88d3906 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 9 Dec 2024 09:25:22 +0000 Subject: [PATCH 18/30] dec2flt: Rename fields to be consistent with documented notation --- library/core/src/num/dec2flt/common.rs | 13 +++++++------ library/core/src/num/dec2flt/lemire.rs | 4 ++-- library/core/src/num/dec2flt/mod.rs | 13 ++++++++----- library/core/src/num/dec2flt/slow.rs | 2 +- library/coretests/tests/num/dec2flt/float.rs | 8 ++++---- library/coretests/tests/num/dec2flt/lemire.rs | 4 ++-- 6 files changed, 24 insertions(+), 20 deletions(-) diff --git a/library/core/src/num/dec2flt/common.rs b/library/core/src/num/dec2flt/common.rs index 1646d3a95d3b1..a140a311c452f 100644 --- a/library/core/src/num/dec2flt/common.rs +++ b/library/core/src/num/dec2flt/common.rs @@ -63,19 +63,20 @@ pub(crate) fn is_8digits(v: u64) -> bool { (a | b) & 0x8080_8080_8080_8080 == 0 } -/// A custom 64-bit floating point type, representing `f * 2^e`. -/// e is biased, so it be directly shifted into the exponent bits. +/// A custom 64-bit floating point type, representing `m * 2^p`. +/// p is biased, so it be directly shifted into the exponent bits. #[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] pub struct BiasedFp { /// The significant digits. - pub f: u64, + pub m: u64, /// The biased, binary exponent. - pub e: i32, + pub p_biased: i32, } impl BiasedFp { + /// Represent `0 ^ p` #[inline] - pub const fn zero_pow2(e: i32) -> Self { - Self { f: 0, e } + pub const fn zero_pow2(p_biased: i32) -> Self { + Self { m: 0, p_biased } } } diff --git a/library/core/src/num/dec2flt/lemire.rs b/library/core/src/num/dec2flt/lemire.rs index 01642e1b1112a..0cc39cb9b6231 100644 --- a/library/core/src/num/dec2flt/lemire.rs +++ b/library/core/src/num/dec2flt/lemire.rs @@ -73,7 +73,7 @@ pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { mantissa += mantissa & 1; mantissa >>= 1; power2 = (mantissa >= (1_u64 << F::MANTISSA_EXPLICIT_BITS)) as i32; - return BiasedFp { f: mantissa, e: power2 }; + return BiasedFp { m: mantissa, p_biased: power2 }; } // Need to handle rounding ties. Normally, we need to round up, // but if we fall right in between and we have an even basis, we @@ -111,7 +111,7 @@ pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { // Exponent is above largest normal value, must be infinite. return fp_inf; } - BiasedFp { f: mantissa, e: power2 } + BiasedFp { m: mantissa, p_biased: power2 } } /// Calculate a base 2 exponent from a decimal exponent. diff --git a/library/core/src/num/dec2flt/mod.rs b/library/core/src/num/dec2flt/mod.rs index a82c858df4998..9d1989c4b817d 100644 --- a/library/core/src/num/dec2flt/mod.rs +++ b/library/core/src/num/dec2flt/mod.rs @@ -233,8 +233,8 @@ pub fn pfe_invalid() -> ParseFloatError { /// Converts a `BiasedFp` to the closest machine float type. fn biased_fp_to_float(x: BiasedFp) -> T { - let mut word = x.f; - word |= (x.e as u64) << T::MANTISSA_EXPLICIT_BITS; + let mut word = x.m; + word |= (x.p_biased as u64) << T::MANTISSA_EXPLICIT_BITS; T::from_u64_bits(word) } @@ -272,12 +272,15 @@ pub fn dec2flt(s: &str) -> Result { // redundantly using the Eisel-Lemire algorithm if it was unable to // correctly round on the first pass. let mut fp = compute_float::(num.exponent, num.mantissa); - if num.many_digits && fp.e >= 0 && fp != compute_float::(num.exponent, num.mantissa + 1) { - fp.e = -1; + if num.many_digits + && fp.p_biased >= 0 + && fp != compute_float::(num.exponent, num.mantissa + 1) + { + fp.p_biased = -1; } // Unable to correctly round the float using the Eisel-Lemire algorithm. // Fallback to a slower, but always correct algorithm. - if fp.e < 0 { + if fp.p_biased < 0 { fp = parse_long_mantissa::(s); } diff --git a/library/core/src/num/dec2flt/slow.rs b/library/core/src/num/dec2flt/slow.rs index e274e1aa0aa1b..e0c4ae117da02 100644 --- a/library/core/src/num/dec2flt/slow.rs +++ b/library/core/src/num/dec2flt/slow.rs @@ -105,5 +105,5 @@ pub(crate) fn parse_long_mantissa(s: &[u8]) -> BiasedFp { } // Zero out all the bits above the explicit mantissa bits. mantissa &= (1_u64 << F::MANTISSA_EXPLICIT_BITS) - 1; - BiasedFp { f: mantissa, e: power2 } + BiasedFp { m: mantissa, p_biased: power2 } } diff --git a/library/coretests/tests/num/dec2flt/float.rs b/library/coretests/tests/num/dec2flt/float.rs index 7a9587a18d030..40f0776e02148 100644 --- a/library/coretests/tests/num/dec2flt/float.rs +++ b/library/coretests/tests/num/dec2flt/float.rs @@ -12,8 +12,8 @@ fn test_f32_integer_decode() { // Ignore the "sign" (quiet / signalling flag) of NAN. // It can vary between runtime operations and LLVM folding. - let (nan_m, nan_e, _nan_s) = f32::NAN.integer_decode(); - assert_eq!((nan_m, nan_e), (12582912, 105)); + let (nan_m, nan_p, _nan_s) = f32::NAN.integer_decode(); + assert_eq!((nan_m, nan_p), (12582912, 105)); } #[test] @@ -28,6 +28,6 @@ fn test_f64_integer_decode() { // Ignore the "sign" (quiet / signalling flag) of NAN. // It can vary between runtime operations and LLVM folding. - let (nan_m, nan_e, _nan_s) = f64::NAN.integer_decode(); - assert_eq!((nan_m, nan_e), (6755399441055744, 972)); + let (nan_m, nan_p, _nan_s) = f64::NAN.integer_decode(); + assert_eq!((nan_m, nan_p), (6755399441055744, 972)); } diff --git a/library/coretests/tests/num/dec2flt/lemire.rs b/library/coretests/tests/num/dec2flt/lemire.rs index f71bbb7c7a318..9f228a25e46b1 100644 --- a/library/coretests/tests/num/dec2flt/lemire.rs +++ b/library/coretests/tests/num/dec2flt/lemire.rs @@ -2,12 +2,12 @@ use core::num::dec2flt::lemire::compute_float; fn compute_float32(q: i64, w: u64) -> (i32, u64) { let fp = compute_float::(q, w); - (fp.e, fp.f) + (fp.p_biased, fp.m) } fn compute_float64(q: i64, w: u64) -> (i32, u64) { let fp = compute_float::(q, w); - (fp.e, fp.f) + (fp.p_biased, fp.m) } #[test] From 19a909ae0e9f923db2da0fc10264a75b7aa5ad99 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 9 Dec 2024 09:25:22 +0000 Subject: [PATCH 19/30] dec2flt: Refactor float traits A lot of the magic constants can be turned into expressions. This reduces some code duplication. Additionally, add traits to make these operations fully generic. This will make it easier to support `f16` and `f128`. --- library/core/src/num/dec2flt/float.rs | 272 ++++++++++++------ library/core/src/num/dec2flt/lemire.rs | 18 +- library/core/src/num/dec2flt/mod.rs | 6 +- library/core/src/num/dec2flt/slow.rs | 18 +- library/coretests/tests/num/dec2flt/float.rs | 40 +++ library/coretests/tests/num/dec2flt/lemire.rs | 6 + library/coretests/tests/num/dec2flt/parse.rs | 15 + src/etc/test-float-parse/src/traits.rs | 6 +- 8 files changed, 267 insertions(+), 114 deletions(-) diff --git a/library/core/src/num/dec2flt/float.rs b/library/core/src/num/dec2flt/float.rs index da57aa9a546af..b8a28a6756917 100644 --- a/library/core/src/num/dec2flt/float.rs +++ b/library/core/src/num/dec2flt/float.rs @@ -1,14 +1,57 @@ //! Helper trait for generic float types. +use core::f64; + use crate::fmt::{Debug, LowerExp}; use crate::num::FpCategory; -use crate::ops::{Add, Div, Mul, Neg}; +use crate::ops::{self, Add, Div, Mul, Neg}; + +/// Lossy `as` casting between two types. +pub trait CastInto: Copy { + fn cast(self) -> T; +} + +/// Collection of traits that allow us to be generic over integer size. +pub trait Integer: + Sized + + Clone + + Copy + + Debug + + ops::Shr + + ops::Shl + + ops::BitAnd + + ops::BitOr + + PartialEq + + CastInto +{ + const ZERO: Self; + const ONE: Self; +} + +macro_rules! int { + ($($ty:ty),+) => { + $( + impl CastInto for $ty { + fn cast(self) -> i16 { + self as i16 + } + } + + impl Integer for $ty { + const ZERO: Self = 0; + const ONE: Self = 1; + } + )+ + } +} + +int!(u32, u64); -/// A helper trait to avoid duplicating basically all the conversion code for `f32` and `f64`. +/// A helper trait to avoid duplicating basically all the conversion code for IEEE floats. /// /// See the parent module's doc comment for why this is necessary. /// -/// Should **never ever** be implemented for other types or be used outside the dec2flt module. +/// Should **never ever** be implemented for other types or be used outside the `dec2flt` module. #[doc(hidden)] pub trait RawFloat: Sized @@ -24,62 +67,107 @@ pub trait RawFloat: + Copy + Debug { + /// The unsigned integer with the same size as the float + type Int: Integer + Into; + + /* general constants */ + const INFINITY: Self; const NEG_INFINITY: Self; const NAN: Self; const NEG_NAN: Self; - /// The number of bits in the significand, *excluding* the hidden bit. - const MANTISSA_EXPLICIT_BITS: usize; - - // Round-to-even only happens for negative values of q - // when q ≥ −4 in the 64-bit case and when q ≥ −17 in - // the 32-bitcase. - // - // When q ≥ 0,we have that 5^q ≤ 2m+1. In the 64-bit case,we - // have 5^q ≤ 2m+1 ≤ 2^54 or q ≤ 23. In the 32-bit case,we have - // 5^q ≤ 2m+1 ≤ 2^25 or q ≤ 10. - // - // When q < 0, we have w ≥ (2m+1)×5^−q. We must have that w < 2^64 - // so (2m+1)×5^−q < 2^64. We have that 2m+1 > 2^53 (64-bit case) - // or 2m+1 > 2^24 (32-bit case). Hence,we must have 2^53×5^−q < 2^64 - // (64-bit) and 2^24×5^−q < 2^64 (32-bit). Hence we have 5^−q < 2^11 - // or q ≥ −4 (64-bit case) and 5^−q < 2^40 or q ≥ −17 (32-bitcase). - // - // Thus we have that we only need to round ties to even when - // we have that q ∈ [−4,23](in the 64-bit case) or q∈[−17,10] - // (in the 32-bit case). In both cases,the power of five(5^|q|) - // fits in a 64-bit word. - const MIN_EXPONENT_ROUND_TO_EVEN: i32; - const MAX_EXPONENT_ROUND_TO_EVEN: i32; + /// Bit width of the float + const BITS: u32; - // Minimum exponent that for a fast path case, or `-⌊(MANTISSA_EXPLICIT_BITS+1)/log2(5)⌋` - const MIN_EXPONENT_FAST_PATH: i64; + /// The number of bits in the significand, *including* the hidden bit. + const SIG_TOTAL_BITS: u32; - // Maximum exponent that for a fast path case, or `⌊(MANTISSA_EXPLICIT_BITS+1)/log2(5)⌋` - const MAX_EXPONENT_FAST_PATH: i64; + const EXP_MASK: Self::Int; + const SIG_MASK: Self::Int; - // Maximum exponent that can be represented for a disguised-fast path case. - // This is `MAX_EXPONENT_FAST_PATH + ⌊(MANTISSA_EXPLICIT_BITS+1)/log2(10)⌋` - const MAX_EXPONENT_DISGUISED_FAST_PATH: i64; - - // Minimum exponent value `-(1 << (EXP_BITS - 1)) + 1`. - const MINIMUM_EXPONENT: i32; + /// The number of bits in the significand, *excluding* the hidden bit. + const SIG_BITS: u32 = Self::SIG_TOTAL_BITS - 1; + + /// Number of bits in the exponent. + const EXP_BITS: u32 = Self::BITS - Self::SIG_BITS - 1; + + /// The saturated (maximum bitpattern) value of the exponent, i.e. the infinite + /// representation. + /// + /// This shifted fully right, use `EXP_MASK` for the shifted value. + const EXP_SAT: u32 = (1 << Self::EXP_BITS) - 1; + + /// Signed version of `EXP_SAT` since we convert a lot. + const INFINITE_POWER: i32 = Self::EXP_SAT as i32; + + /// The exponent bias value. This is also the maximum value of the exponent. + const EXP_BIAS: u32 = Self::EXP_SAT >> 1; + + /// Minimum exponent value of normal values. + const EXP_MIN: i32 = -(Self::EXP_BIAS as i32 - 1); + + /// Round-to-even only happens for negative values of q + /// when q ≥ −4 in the 64-bit case and when q ≥ −17 in + /// the 32-bitcase. + /// + /// When q ≥ 0,we have that 5^q ≤ 2m+1. In the 64-bit case,we + /// have 5^q ≤ 2m+1 ≤ 2^54 or q ≤ 23. In the 32-bit case,we have + /// 5^q ≤ 2m+1 ≤ 2^25 or q ≤ 10. + /// + /// When q < 0, we have w ≥ (2m+1)×5^−q. We must have that w < 2^64 + /// so (2m+1)×5^−q < 2^64. We have that 2m+1 > 2^53 (64-bit case) + /// or 2m+1 > 2^24 (32-bit case). Hence,we must have 2^53×5^−q < 2^64 + /// (64-bit) and 2^24×5^−q < 2^64 (32-bit). Hence we have 5^−q < 2^11 + /// or q ≥ −4 (64-bit case) and 5^−q < 2^40 or q ≥ −17 (32-bitcase). + /// + /// Thus we have that we only need to round ties to even when + /// we have that q ∈ [−4,23](in the 64-bit case) or q∈[−17,10] + /// (in the 32-bit case). In both cases,the power of five(5^|q|) + /// fits in a 64-bit word. + const MIN_EXPONENT_ROUND_TO_EVEN: i32; + const MAX_EXPONENT_ROUND_TO_EVEN: i32; - // Largest exponent value `(1 << EXP_BITS) - 1`. - const INFINITE_POWER: i32; + /* limits related to Fast pathing */ + + /// Largest decimal exponent for a non-infinite value. + /// + /// This is the max exponent in binary converted to the max exponent in decimal. Allows fast + /// pathing anything larger than `10^LARGEST_POWER_OF_TEN`, which will round to infinity. + const LARGEST_POWER_OF_TEN: i32 = { + let largest_pow2 = Self::EXP_BIAS + 1; + pow2_to_pow10(largest_pow2 as i64) as i32 + }; + + /// Smallest decimal exponent for a non-zero value. This allows for fast pathing anything + /// smaller than `10^SMALLEST_POWER_OF_TEN`, which will round to zero. + /// + /// The smallest power of ten is represented by `⌊log10(2^-n / (2^64 - 1))⌋`, where `n` is + /// the smallest power of two. The `2^64 - 1)` denomenator comes from the number of values + /// that are representable by the intermediate storage format. I don't actually know _why_ + /// the storage format is relevant here. + /// + /// The values may be calculated using the formula. Unfortunately we cannot calculate them at + /// compile time since intermediates exceed the range of an `f64`. + const SMALLEST_POWER_OF_TEN: i32; - // Index (in bits) of the sign. - const SIGN_INDEX: usize; + /// Maximum exponent for a fast path case, or `⌊(SIG_BITS+1)/log2(5)⌋` + // assuming FLT_EVAL_METHOD = 0 + const MAX_EXPONENT_FAST_PATH: i64 = { + let log2_5 = f64::consts::LOG2_10 - 1.0; + (Self::SIG_TOTAL_BITS as f64 / log2_5) as i64 + }; - // Smallest decimal exponent for a non-zero value. - const SMALLEST_POWER_OF_TEN: i32; + /// Minimum exponent for a fast path case, or `-⌊(SIG_BITS+1)/log2(5)⌋` + const MIN_EXPONENT_FAST_PATH: i64 = -Self::MAX_EXPONENT_FAST_PATH; - // Largest decimal exponent for a non-infinite value. - const LARGEST_POWER_OF_TEN: i32; + /// Maximum exponent that can be represented for a disguised-fast path case. + /// This is `MAX_EXPONENT_FAST_PATH + ⌊(SIG_BITS+1)/log2(10)⌋` + const MAX_EXPONENT_DISGUISED_FAST_PATH: i64 = + Self::MAX_EXPONENT_FAST_PATH + (Self::SIG_TOTAL_BITS as f64 / f64::consts::LOG2_10) as i64; - // Maximum mantissa for the fast-path (`1 << 53` for f64). - const MAX_MANTISSA_FAST_PATH: u64 = 2_u64 << Self::MANTISSA_EXPLICIT_BITS; + /// Maximum mantissa for the fast-path (`1 << 53` for f64). + const MAX_MANTISSA_FAST_PATH: u64 = 1 << Self::SIG_TOTAL_BITS; /// Converts integer into float through an as cast. /// This is only called in the fast-path algorithm, and therefore @@ -96,27 +184,51 @@ pub trait RawFloat: /// Returns the category that this number falls into. fn classify(self) -> FpCategory; + /// Transmute to the integer representation + fn to_bits(self) -> Self::Int; + /// Returns the mantissa, exponent and sign as integers. - fn integer_decode(self) -> (u64, i16, i8); + /// + /// That is, this returns `(m, p, s)` such that `s * m * 2^p` represents the original float. + /// For 0, the exponent will be `-(EXP_BIAS + SIG_BITS`, which is the + /// minimum subnormal power. + fn integer_decode(self) -> (u64, i16, i8) { + let bits = self.to_bits(); + let sign: i8 = if bits >> (Self::BITS - 1) == Self::Int::ZERO { 1 } else { -1 }; + let mut exponent: i16 = ((bits & Self::EXP_MASK) >> Self::SIG_BITS).cast(); + let mantissa = if exponent == 0 { + (bits & Self::SIG_MASK) << 1 + } else { + (bits & Self::SIG_MASK) | (Self::Int::ONE << Self::SIG_BITS) + }; + // Exponent bias + mantissa shift + exponent -= (Self::EXP_BIAS + Self::SIG_BITS) as i16; + (mantissa.into(), exponent, sign) + } +} + +/// Solve for `b` in `10^b = 2^a` +const fn pow2_to_pow10(a: i64) -> i64 { + let res = (a as f64) / f64::consts::LOG2_10; + res as i64 } impl RawFloat for f32 { + type Int = u32; + const INFINITY: Self = f32::INFINITY; const NEG_INFINITY: Self = f32::NEG_INFINITY; const NAN: Self = f32::NAN; const NEG_NAN: Self = -f32::NAN; - const MANTISSA_EXPLICIT_BITS: usize = 23; + const BITS: u32 = 32; + const SIG_TOTAL_BITS: u32 = Self::MANTISSA_DIGITS; + const EXP_MASK: Self::Int = Self::EXP_MASK; + const SIG_MASK: Self::Int = Self::MAN_MASK; + const MIN_EXPONENT_ROUND_TO_EVEN: i32 = -17; const MAX_EXPONENT_ROUND_TO_EVEN: i32 = 10; - const MIN_EXPONENT_FAST_PATH: i64 = -10; // assuming FLT_EVAL_METHOD = 0 - const MAX_EXPONENT_FAST_PATH: i64 = 10; - const MAX_EXPONENT_DISGUISED_FAST_PATH: i64 = 17; - const MINIMUM_EXPONENT: i32 = -127; - const INFINITE_POWER: i32 = 0xFF; - const SIGN_INDEX: usize = 31; const SMALLEST_POWER_OF_TEN: i32 = -65; - const LARGEST_POWER_OF_TEN: i32 = 38; #[inline] fn from_u64(v: u64) -> Self { @@ -136,16 +248,8 @@ impl RawFloat for f32 { TABLE[exponent & 15] } - /// Returns the mantissa, exponent and sign as integers. - fn integer_decode(self) -> (u64, i16, i8) { - let bits = self.to_bits(); - let sign: i8 = if bits >> 31 == 0 { 1 } else { -1 }; - let mut exponent: i16 = ((bits >> 23) & 0xff) as i16; - let mantissa = - if exponent == 0 { (bits & 0x7fffff) << 1 } else { (bits & 0x7fffff) | 0x800000 }; - // Exponent bias + mantissa shift - exponent -= 127 + 23; - (mantissa as u64, exponent, sign) + fn to_bits(self) -> Self::Int { + self.to_bits() } fn classify(self) -> FpCategory { @@ -154,22 +258,21 @@ impl RawFloat for f32 { } impl RawFloat for f64 { - const INFINITY: Self = f64::INFINITY; - const NEG_INFINITY: Self = f64::NEG_INFINITY; - const NAN: Self = f64::NAN; - const NEG_NAN: Self = -f64::NAN; + type Int = u64; + + const INFINITY: Self = Self::INFINITY; + const NEG_INFINITY: Self = Self::NEG_INFINITY; + const NAN: Self = Self::NAN; + const NEG_NAN: Self = -Self::NAN; + + const BITS: u32 = 64; + const SIG_TOTAL_BITS: u32 = Self::MANTISSA_DIGITS; + const EXP_MASK: Self::Int = Self::EXP_MASK; + const SIG_MASK: Self::Int = Self::MAN_MASK; - const MANTISSA_EXPLICIT_BITS: usize = 52; const MIN_EXPONENT_ROUND_TO_EVEN: i32 = -4; const MAX_EXPONENT_ROUND_TO_EVEN: i32 = 23; - const MIN_EXPONENT_FAST_PATH: i64 = -22; // assuming FLT_EVAL_METHOD = 0 - const MAX_EXPONENT_FAST_PATH: i64 = 22; - const MAX_EXPONENT_DISGUISED_FAST_PATH: i64 = 37; - const MINIMUM_EXPONENT: i32 = -1023; - const INFINITE_POWER: i32 = 0x7FF; - const SIGN_INDEX: usize = 63; const SMALLEST_POWER_OF_TEN: i32 = -342; - const LARGEST_POWER_OF_TEN: i32 = 308; #[inline] fn from_u64(v: u64) -> Self { @@ -190,19 +293,8 @@ impl RawFloat for f64 { TABLE[exponent & 31] } - /// Returns the mantissa, exponent and sign as integers. - fn integer_decode(self) -> (u64, i16, i8) { - let bits = self.to_bits(); - let sign: i8 = if bits >> 63 == 0 { 1 } else { -1 }; - let mut exponent: i16 = ((bits >> 52) & 0x7ff) as i16; - let mantissa = if exponent == 0 { - (bits & 0xfffffffffffff) << 1 - } else { - (bits & 0xfffffffffffff) | 0x10000000000000 - }; - // Exponent bias + mantissa shift - exponent -= 1023 + 52; - (mantissa, exponent, sign) + fn to_bits(self) -> Self::Int { + self.to_bits() } fn classify(self) -> FpCategory { diff --git a/library/core/src/num/dec2flt/lemire.rs b/library/core/src/num/dec2flt/lemire.rs index 0cc39cb9b6231..f84929a03c172 100644 --- a/library/core/src/num/dec2flt/lemire.rs +++ b/library/core/src/num/dec2flt/lemire.rs @@ -38,7 +38,7 @@ pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { // Normalize our significant digits, so the most-significant bit is set. let lz = w.leading_zeros(); w <<= lz; - let (lo, hi) = compute_product_approx(q, w, F::MANTISSA_EXPLICIT_BITS + 3); + let (lo, hi) = compute_product_approx(q, w, F::SIG_BITS as usize + 3); if lo == 0xFFFF_FFFF_FFFF_FFFF { // If we have failed to approximate w x 5^-q with our 128-bit value. // Since the addition of 1 could lead to an overflow which could then @@ -61,8 +61,8 @@ pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { } } let upperbit = (hi >> 63) as i32; - let mut mantissa = hi >> (upperbit + 64 - F::MANTISSA_EXPLICIT_BITS as i32 - 3); - let mut power2 = power(q as i32) + upperbit - lz as i32 - F::MINIMUM_EXPONENT; + let mut mantissa = hi >> (upperbit + 64 - F::SIG_BITS as i32 - 3); + let mut power2 = power(q as i32) + upperbit - lz as i32 - F::EXP_MIN + 1; if power2 <= 0 { if -power2 + 1 >= 64 { // Have more than 64 bits below the minimum exponent, must be 0. @@ -72,7 +72,7 @@ pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { mantissa >>= -power2 + 1; mantissa += mantissa & 1; mantissa >>= 1; - power2 = (mantissa >= (1_u64 << F::MANTISSA_EXPLICIT_BITS)) as i32; + power2 = (mantissa >= (1_u64 << F::SIG_BITS)) as i32; return BiasedFp { m: mantissa, p_biased: power2 }; } // Need to handle rounding ties. Normally, we need to round up, @@ -89,8 +89,8 @@ pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { if lo <= 1 && q >= F::MIN_EXPONENT_ROUND_TO_EVEN as i64 && q <= F::MAX_EXPONENT_ROUND_TO_EVEN as i64 - && mantissa & 3 == 1 - && (mantissa << (upperbit + 64 - F::MANTISSA_EXPLICIT_BITS as i32 - 3)) == hi + && mantissa & 0b11 == 0b01 + && (mantissa << (upperbit + 64 - F::SIG_BITS as i32 - 3)) == hi { // Zero the lowest bit, so we don't round up. mantissa &= !1_u64; @@ -98,15 +98,15 @@ pub fn compute_float(q: i64, mut w: u64) -> BiasedFp { // Round-to-even, then shift the significant digits into place. mantissa += mantissa & 1; mantissa >>= 1; - if mantissa >= (2_u64 << F::MANTISSA_EXPLICIT_BITS) { + if mantissa >= (2_u64 << F::SIG_BITS) { // Rounding up overflowed, so the carry bit is set. Set the // mantissa to 1 (only the implicit, hidden bit is set) and // increase the exponent. - mantissa = 1_u64 << F::MANTISSA_EXPLICIT_BITS; + mantissa = 1_u64 << F::SIG_BITS; power2 += 1; } // Zero out the hidden bit. - mantissa &= !(1_u64 << F::MANTISSA_EXPLICIT_BITS); + mantissa &= !(1_u64 << F::SIG_BITS); if power2 >= F::INFINITE_POWER { // Exponent is above largest normal value, must be infinite. return fp_inf; diff --git a/library/core/src/num/dec2flt/mod.rs b/library/core/src/num/dec2flt/mod.rs index 9d1989c4b817d..d1a0e1db31314 100644 --- a/library/core/src/num/dec2flt/mod.rs +++ b/library/core/src/num/dec2flt/mod.rs @@ -232,10 +232,10 @@ pub fn pfe_invalid() -> ParseFloatError { } /// Converts a `BiasedFp` to the closest machine float type. -fn biased_fp_to_float(x: BiasedFp) -> T { +fn biased_fp_to_float(x: BiasedFp) -> F { let mut word = x.m; - word |= (x.p_biased as u64) << T::MANTISSA_EXPLICIT_BITS; - T::from_u64_bits(word) + word |= (x.p_biased as u64) << F::SIG_BITS; + F::from_u64_bits(word) } /// Converts a decimal string into a floating point number. diff --git a/library/core/src/num/dec2flt/slow.rs b/library/core/src/num/dec2flt/slow.rs index e0c4ae117da02..3baed42652393 100644 --- a/library/core/src/num/dec2flt/slow.rs +++ b/library/core/src/num/dec2flt/slow.rs @@ -74,36 +74,36 @@ pub(crate) fn parse_long_mantissa(s: &[u8]) -> BiasedFp { } // We are now in the range [1/2 ... 1] but the binary format uses [1 ... 2]. exp2 -= 1; - while (F::MINIMUM_EXPONENT + 1) > exp2 { - let mut n = ((F::MINIMUM_EXPONENT + 1) - exp2) as usize; + while F::EXP_MIN > exp2 { + let mut n = (F::EXP_MIN - exp2) as usize; if n > MAX_SHIFT { n = MAX_SHIFT; } d.right_shift(n); exp2 += n as i32; } - if (exp2 - F::MINIMUM_EXPONENT) >= F::INFINITE_POWER { + if (exp2 - F::EXP_MIN + 1) >= F::INFINITE_POWER { return fp_inf; } // Shift the decimal to the hidden bit, and then round the value // to get the high mantissa+1 bits. - d.left_shift(F::MANTISSA_EXPLICIT_BITS + 1); + d.left_shift(F::SIG_BITS as usize + 1); let mut mantissa = d.round(); - if mantissa >= (1_u64 << (F::MANTISSA_EXPLICIT_BITS + 1)) { + if mantissa >= (1_u64 << (F::SIG_BITS + 1)) { // Rounding up overflowed to the carry bit, need to // shift back to the hidden bit. d.right_shift(1); exp2 += 1; mantissa = d.round(); - if (exp2 - F::MINIMUM_EXPONENT) >= F::INFINITE_POWER { + if (exp2 - F::EXP_MIN + 1) >= F::INFINITE_POWER { return fp_inf; } } - let mut power2 = exp2 - F::MINIMUM_EXPONENT; - if mantissa < (1_u64 << F::MANTISSA_EXPLICIT_BITS) { + let mut power2 = exp2 - F::EXP_MIN + 1; + if mantissa < (1_u64 << F::SIG_BITS) { power2 -= 1; } // Zero out all the bits above the explicit mantissa bits. - mantissa &= (1_u64 << F::MANTISSA_EXPLICIT_BITS) - 1; + mantissa &= (1_u64 << F::SIG_BITS) - 1; BiasedFp { m: mantissa, p_biased: power2 } } diff --git a/library/coretests/tests/num/dec2flt/float.rs b/library/coretests/tests/num/dec2flt/float.rs index 40f0776e02148..b5afd3e3b2436 100644 --- a/library/coretests/tests/num/dec2flt/float.rs +++ b/library/coretests/tests/num/dec2flt/float.rs @@ -31,3 +31,43 @@ fn test_f64_integer_decode() { let (nan_m, nan_p, _nan_s) = f64::NAN.integer_decode(); assert_eq!((nan_m, nan_p), (6755399441055744, 972)); } + +/* Sanity checks of computed magic numbers */ + +#[test] +fn test_f32_consts() { + assert_eq!(::INFINITY, f32::INFINITY); + assert_eq!(::NEG_INFINITY, -f32::INFINITY); + assert_eq!(::NAN.to_bits(), f32::NAN.to_bits()); + assert_eq!(::NEG_NAN.to_bits(), (-f32::NAN).to_bits()); + assert_eq!(::SIG_BITS, 23); + assert_eq!(::MIN_EXPONENT_ROUND_TO_EVEN, -17); + assert_eq!(::MAX_EXPONENT_ROUND_TO_EVEN, 10); + assert_eq!(::MIN_EXPONENT_FAST_PATH, -10); + assert_eq!(::MAX_EXPONENT_FAST_PATH, 10); + assert_eq!(::MAX_EXPONENT_DISGUISED_FAST_PATH, 17); + assert_eq!(::EXP_MIN, -126); + assert_eq!(::EXP_SAT, 0xff); + assert_eq!(::SMALLEST_POWER_OF_TEN, -65); + assert_eq!(::LARGEST_POWER_OF_TEN, 38); + assert_eq!(::MAX_MANTISSA_FAST_PATH, 16777216); +} + +#[test] +fn test_f64_consts() { + assert_eq!(::INFINITY, f64::INFINITY); + assert_eq!(::NEG_INFINITY, -f64::INFINITY); + assert_eq!(::NAN.to_bits(), f64::NAN.to_bits()); + assert_eq!(::NEG_NAN.to_bits(), (-f64::NAN).to_bits()); + assert_eq!(::SIG_BITS, 52); + assert_eq!(::MIN_EXPONENT_ROUND_TO_EVEN, -4); + assert_eq!(::MAX_EXPONENT_ROUND_TO_EVEN, 23); + assert_eq!(::MIN_EXPONENT_FAST_PATH, -22); + assert_eq!(::MAX_EXPONENT_FAST_PATH, 22); + assert_eq!(::MAX_EXPONENT_DISGUISED_FAST_PATH, 37); + assert_eq!(::EXP_MIN, -1022); + assert_eq!(::EXP_SAT, 0x7ff); + assert_eq!(::SMALLEST_POWER_OF_TEN, -342); + assert_eq!(::LARGEST_POWER_OF_TEN, 308); + assert_eq!(::MAX_MANTISSA_FAST_PATH, 9007199254740992); +} diff --git a/library/coretests/tests/num/dec2flt/lemire.rs b/library/coretests/tests/num/dec2flt/lemire.rs index 9f228a25e46b1..0db80fbd52506 100644 --- a/library/coretests/tests/num/dec2flt/lemire.rs +++ b/library/coretests/tests/num/dec2flt/lemire.rs @@ -1,3 +1,4 @@ +use core::num::dec2flt::float::RawFloat; use core::num::dec2flt::lemire::compute_float; fn compute_float32(q: i64, w: u64) -> (i32, u64) { @@ -27,6 +28,11 @@ fn compute_float_f32_rounding() { // Let's check the lines to see if anything is different in table... assert_eq!(compute_float32(-10, 167772190000000000), (151, 2)); assert_eq!(compute_float32(-10, 167772200000000000), (151, 2)); + + // Check the rounding point between infinity and the next representable number down + assert_eq!(compute_float32(38, 3), (f32::INFINITE_POWER - 1, 6402534)); + assert_eq!(compute_float32(38, 4), (f32::INFINITE_POWER, 0)); // infinity + assert_eq!(compute_float32(20, 3402823470000000000), (f32::INFINITE_POWER - 1, 8388607)); } #[test] diff --git a/library/coretests/tests/num/dec2flt/parse.rs b/library/coretests/tests/num/dec2flt/parse.rs index ec705a61e13ca..59be3915052d8 100644 --- a/library/coretests/tests/num/dec2flt/parse.rs +++ b/library/coretests/tests/num/dec2flt/parse.rs @@ -57,6 +57,21 @@ macro_rules! assert_float_result_bits_eq { }}; } +#[test] +fn regression() { + // These showed up in fuzz tests when the minimum exponent was incorrect. + assert_float_result_bits_eq!( + 0x0, + f64, + "3313756768023998018398807867233977556112078681253148176737587500333136120852692315608454494981109839693784033457129423181787087843504060087613228932431e-475" + ); + assert_float_result_bits_eq!( + 0x0, + f64, + "5298127456259331337220.92759278003098321644501973966679724599271041396379712108366679824365568578569680024083293475291869842408884554511641179110778276695274832779269225510492006696321279587846006535230380114430977056662212751544508159333199129106162019382177820713609e-346" + ); +} + #[test] fn issue31109() { // Regression test for #31109. diff --git a/src/etc/test-float-parse/src/traits.rs b/src/etc/test-float-parse/src/traits.rs index f5333d63b3693..f7f5f5dd35b70 100644 --- a/src/etc/test-float-parse/src/traits.rs +++ b/src/etc/test-float-parse/src/traits.rs @@ -147,12 +147,12 @@ pub trait Float: } macro_rules! impl_float { - ($($fty:ty, $ity:ty, $bits:literal);+) => { + ($($fty:ty, $ity:ty);+) => { $( impl Float for $fty { type Int = $ity; type SInt = ::Signed; - const BITS: u32 = $bits; + const BITS: u32 = <$ity>::BITS; const MAN_BITS: u32 = Self::MANTISSA_DIGITS - 1; const MAN_MASK: Self::Int = (Self::Int::ONE << Self::MAN_BITS) - Self::Int::ONE; const SIGN_MASK: Self::Int = Self::Int::ONE << (Self::BITS-1); @@ -168,7 +168,7 @@ macro_rules! impl_float { } } -impl_float!(f32, u32, 32; f64, u64, 64); +impl_float!(f32, u32; f64, u64); /// A test generator. Should provide an iterator that produces unique patterns to parse. /// From 37e223ccaa1d0036540e6e3de5d4ed4b721afd60 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 9 Dec 2024 09:25:22 +0000 Subject: [PATCH 20/30] dec2flt: Refactor the fast path This is just a bit of code cleanup to make use of returning early. --- library/core/src/num/dec2flt/decimal.rs | 47 ++++++++++++------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/library/core/src/num/dec2flt/decimal.rs b/library/core/src/num/dec2flt/decimal.rs index 1ba9150a0119d..db7176c124318 100644 --- a/library/core/src/num/dec2flt/decimal.rs +++ b/library/core/src/num/dec2flt/decimal.rs @@ -34,14 +34,15 @@ pub struct Decimal { impl Decimal { /// Detect if the float can be accurately reconstructed from native floats. #[inline] - fn is_fast_path(&self) -> bool { + fn can_use_fast_path(&self) -> bool { F::MIN_EXPONENT_FAST_PATH <= self.exponent && self.exponent <= F::MAX_EXPONENT_DISGUISED_FAST_PATH && self.mantissa <= F::MAX_MANTISSA_FAST_PATH && !self.many_digits } - /// The fast path algorithm using machine-sized integers and floats. + /// Try turning the decimal into an exact float representation, using machine-sized integers + /// and floats. /// /// This is extracted into a separate function so that it can be attempted before constructing /// a Decimal. This only works if both the mantissa and the exponent @@ -59,30 +60,28 @@ impl Decimal { // require setting it by changing the global state (like the control word of the x87 FPU). let _cw = set_precision::(); - if self.is_fast_path::() { - let mut value = if self.exponent <= F::MAX_EXPONENT_FAST_PATH { - // normal fast path - let value = F::from_u64(self.mantissa); - if self.exponent < 0 { - value / F::pow10_fast_path((-self.exponent) as _) - } else { - value * F::pow10_fast_path(self.exponent as _) - } + if !self.can_use_fast_path::() { + return None; + } + + let value = if self.exponent <= F::MAX_EXPONENT_FAST_PATH { + // normal fast path + let value = F::from_u64(self.mantissa); + if self.exponent < 0 { + value / F::pow10_fast_path((-self.exponent) as _) } else { - // disguised fast path - let shift = self.exponent - F::MAX_EXPONENT_FAST_PATH; - let mantissa = self.mantissa.checked_mul(INT_POW10[shift as usize])?; - if mantissa > F::MAX_MANTISSA_FAST_PATH { - return None; - } - F::from_u64(mantissa) * F::pow10_fast_path(F::MAX_EXPONENT_FAST_PATH as _) - }; - if self.negative { - value = -value; + value * F::pow10_fast_path(self.exponent as _) } - Some(value) } else { - None - } + // disguised fast path + let shift = self.exponent - F::MAX_EXPONENT_FAST_PATH; + let mantissa = self.mantissa.checked_mul(INT_POW10[shift as usize])?; + if mantissa > F::MAX_MANTISSA_FAST_PATH { + return None; + } + F::from_u64(mantissa) * F::pow10_fast_path(F::MAX_EXPONENT_FAST_PATH as _) + }; + + if self.negative { Some(-value) } else { Some(value) } } } From 6324b398735961c4636fd41d5044a196063d5efa Mon Sep 17 00:00:00 2001 From: LuuuXXX Date: Wed, 12 Feb 2025 10:24:57 +0800 Subject: [PATCH 21/30] promote ohos targets to tier to with host tools --- Cargo.lock | 21 +++++++++++++++---- compiler/rustc_codegen_llvm/Cargo.toml | 3 ++- compiler/rustc_data_structures/Cargo.toml | 3 ++- compiler/rustc_llvm/build.rs | 3 ++- compiler/rustc_query_impl/Cargo.toml | 4 ++-- src/bootstrap/src/core/build_steps/llvm.rs | 4 ++++ .../docker/host-x86_64/dist-ohos/Dockerfile | 13 +++++++++++- src/ci/docker/scripts/ohos-openssl.sh | 7 +++++++ src/ci/docker/scripts/ohos-sdk.sh | 6 +++--- src/doc/rustc/src/platform-support.md | 6 +++--- 10 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 src/ci/docker/scripts/ohos-openssl.sh diff --git a/Cargo.lock b/Cargo.lock index 72f2d4f6cd390..ea5d9bffefbfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2178,6 +2178,20 @@ dependencies = [ "smallvec", ] +[[package]] +name = "measureme-mirror" +version = "12.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe110855993552cfa51a5018e8cdf2acf7f948c46136322017a9a8484ffc5ea8" +dependencies = [ + "log", + "memmap2", + "parking_lot", + "perf-event-open-sys", + "rustc-hash 1.1.0", + "smallvec", +] + [[package]] name = "memchr" version = "2.7.4" @@ -3365,7 +3379,7 @@ dependencies = [ "gimli 0.30.0", "itertools", "libc", - "measureme", + "measureme-mirror", "object 0.36.7", "rustc-demangle", "rustc_abi", @@ -3483,7 +3497,7 @@ dependencies = [ "indexmap", "jobserver", "libc", - "measureme", + "measureme-mirror", "memmap2", "parking_lot", "portable-atomic", @@ -4249,8 +4263,7 @@ dependencies = [ name = "rustc_query_impl" version = "0.0.0" dependencies = [ - "measureme", - "rustc_attr_data_structures", + "measureme-mirror", "rustc_data_structures", "rustc_errors", "rustc_hashes", diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index d3ce7c5a1130f..1122883914d83 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -14,7 +14,8 @@ bitflags = "2.4.1" gimli = "0.30" itertools = "0.12" libc = "0.2" -measureme = "11" +# FIXME: waiting for the new version of measureme. (https://github.com/rust-lang/measureme/pull/240) +measureme = { package = "measureme-mirror", version = "12.0.1" } object = { version = "0.36.3", default-features = false, features = ["std", "read"] } rustc-demangle = "0.1.21" rustc_abi = { path = "../rustc_abi" } diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index bdf5494f2107b..6629f2a751653 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -12,7 +12,8 @@ elsa = "1.11.0" ena = "0.14.3" indexmap = "2.4.0" jobserver_crate = { version = "0.1.28", package = "jobserver" } -measureme = "11" +# FIXME: waiting for the new version of measureme. (https://github.com/rust-lang/measureme/pull/240) +measureme = { package = "measureme-mirror", version = "12.0.1" } rustc-hash = "2.0.0" rustc-rayon = { version = "0.5.1", features = ["indexmap"] } rustc-stable-hash = { version = "0.1.0", features = ["nightly"] } diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index 3d1f3b2cd4dd4..6692ea735401a 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -241,7 +241,7 @@ fn main() { println!("cargo:rustc-link-lib=kstat"); } - if (target.starts_with("arm") && !target.contains("freebsd")) + if (target.starts_with("arm") && !target.contains("freebsd")) && !target.contains("ohos") || target.starts_with("mips-") || target.starts_with("mipsel-") || target.starts_with("powerpc-") @@ -371,6 +371,7 @@ fn main() { || target.contains("freebsd") || target.contains("windows-gnullvm") || target.contains("aix") + || target.contains("ohos") { "c++" } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() { diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml index c85156e059e5a..14a7391f10890 100644 --- a/compiler/rustc_query_impl/Cargo.toml +++ b/compiler/rustc_query_impl/Cargo.toml @@ -5,8 +5,8 @@ edition = "2024" [dependencies] # tidy-alphabetical-start -measureme = "11" -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } +# FIXME: waiting for the new version of measureme. (https://github.com/rust-lang/measureme/pull/240) +measureme = { package = "measureme-mirror", version = "12.0.1" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_hashes = { path = "../rustc_hashes" } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 40d701f22c137..5919e989b3413 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -471,6 +471,10 @@ impl Step for Llvm { cfg.define("LLVM_BUILD_32_BITS", "ON"); } + if target.starts_with("x86_64") && target.contains("ohos") { + cfg.define("LLVM_TOOL_LLVM_RTDYLD_BUILD", "OFF"); + } + let mut enabled_llvm_projects = Vec::new(); if helpers::forcing_clang_based_tests() { diff --git a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile index bbbf0b3adf2a2..23c0c8b49d2b3 100644 --- a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile @@ -22,6 +22,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/ohos-sdk.sh /scripts/ RUN sh /scripts/ohos-sdk.sh +COPY scripts/ohos-openssl.sh /scripts/ +RUN sh /scripts/ohos-openssl.sh + COPY scripts/ohos/aarch64-unknown-linux-ohos-clang.sh /usr/local/bin/ COPY scripts/ohos/aarch64-unknown-linux-ohos-clang++.sh /usr/local/bin/ COPY scripts/ohos/armv7-unknown-linux-ohos-clang.sh /usr/local/bin/ @@ -30,6 +33,14 @@ COPY scripts/ohos/x86_64-unknown-linux-ohos-clang.sh /usr/local/bin/ COPY scripts/ohos/x86_64-unknown-linux-ohos-clang++.sh /usr/local/bin/ # env +ENV AARCH64_UNKNOWN_LINUX_OHOS_OPENSSL_DIR=/opt/ohos-openssl/prelude/arm64-v8a +ENV ARMV7_UNKNOWN_LINUX_OHOS_OPENSSL_DIR=/opt/ohos-openssl/prelude/armeabi-v7a +ENV X86_64_UNKNOWN_LINUX_OHOS_OPENSSL_DIR=/opt/ohos-openssl/prelude/x86_64 + +ENV AARCH64_UNKNOWN_LINUX_OHOS_OPENSSL_NO_VENDOR=1 +ENV ARMV7_UNKNOWN_LINUX_OHOS_OPENSSL_NO_VENDOR=1 +ENV X86_64_UNKNOWN_LINUX_OHOS_OPENSSL_NO_VENDOR=1 + ENV TARGETS=aarch64-unknown-linux-ohos ENV TARGETS=$TARGETS,armv7-unknown-linux-ohos ENV TARGETS=$TARGETS,x86_64-unknown-linux-ohos @@ -51,7 +62,7 @@ ENV RUST_CONFIGURE_ARGS \ --enable-profiler \ --disable-docs -ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS +ENV SCRIPT python3 ../x.py dist --host=$TARGETS --target $TARGETS COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/scripts/ohos-openssl.sh b/src/ci/docker/scripts/ohos-openssl.sh new file mode 100644 index 0000000000000..713ab6131e31a --- /dev/null +++ b/src/ci/docker/scripts/ohos-openssl.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -ex + +URL=https://github.com/ohos-rs/ohos-openssl/archive/refs/tags/0.1.0.tar.gz + +mkdir -p /opt/ohos-openssl +curl -fL $URL | tar xz -C /opt/ohos-openssl --strip-components=1 diff --git a/src/ci/docker/scripts/ohos-sdk.sh b/src/ci/docker/scripts/ohos-sdk.sh index 321be2b8697b0..0b62e49f7ca20 100755 --- a/src/ci/docker/scripts/ohos-sdk.sh +++ b/src/ci/docker/scripts/ohos-sdk.sh @@ -1,9 +1,9 @@ #!/bin/sh set -ex -URL=https://repo.huaweicloud.com/openharmony/os/4.0-Release/ohos-sdk-windows_linux-public.tar.gz +URL=https://repo.huaweicloud.com/openharmony/os/5.0.0-Release/ohos-sdk-windows_linux-public.tar.gz -curl $URL | tar xz -C /tmp ohos-sdk/linux/native-linux-x64-4.0.10.13-Release.zip +curl $URL | tar xz -C /tmp linux/native-linux-x64-5.0.0.71-Release.zip mkdir /opt/ohos-sdk cd /opt/ohos-sdk -unzip -qq /tmp/ohos-sdk/linux/native-linux-x64-4.0.10.13-Release.zip +unzip -qq /tmp/linux/native-linux-x64-5.0.0.71-Release.zip diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index c4e5c1aac2f5d..a98c6f8d86111 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -89,9 +89,11 @@ target | notes -------|------- `aarch64-pc-windows-msvc` | ARM64 Windows MSVC `aarch64-unknown-linux-musl` | ARM64 Linux with musl 1.2.3 +[`aarch64-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | ARM64 OpenHarmony `arm-unknown-linux-gnueabi` | Armv6 Linux (kernel 3.2, glibc 2.17) `arm-unknown-linux-gnueabihf` | Armv6 Linux, hardfloat (kernel 3.2, glibc 2.17) `armv7-unknown-linux-gnueabihf` | Armv7-A Linux, hardfloat (kernel 3.2, glibc 2.17) +[`armv7-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | Armv7-A OpenHarmony [`loongarch64-unknown-linux-gnu`](platform-support/loongarch-linux.md) | LoongArch64 Linux, LP64D ABI (kernel 5.19, glibc 2.36) [`loongarch64-unknown-linux-musl`](platform-support/loongarch-linux.md) | LoongArch64 Linux, LP64D ABI (kernel 5.19, musl 1.2.5) `powerpc-unknown-linux-gnu` | PowerPC Linux (kernel 3.2, glibc 2.17) @@ -104,6 +106,7 @@ target | notes [`x86_64-unknown-freebsd`](platform-support/freebsd.md) | 64-bit x86 FreeBSD [`x86_64-unknown-illumos`](platform-support/illumos.md) | illumos `x86_64-unknown-linux-musl` | 64-bit Linux with musl 1.2.3 +[`x86_64-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | x86_64 OpenHarmony [`x86_64-unknown-netbsd`](platform-support/netbsd.md) | NetBSD/amd64 ## Tier 2 without Host Tools @@ -142,7 +145,6 @@ target | std | notes [`aarch64-linux-android`](platform-support/android.md) | ✓ | ARM64 Android [`aarch64-pc-windows-gnullvm`](platform-support/pc-windows-gnullvm.md) | ✓ | ARM64 MinGW (Windows 10+), LLVM ABI [`aarch64-unknown-fuchsia`](platform-support/fuchsia.md) | ✓ | ARM64 Fuchsia -[`aarch64-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | ARM64 OpenHarmony `aarch64-unknown-none` | * | Bare ARM64, hardfloat `aarch64-unknown-none-softfloat` | * | Bare ARM64, softfloat [`aarch64-unknown-uefi`](platform-support/unknown-uefi.md) | ? | ARM64 UEFI @@ -158,7 +160,6 @@ target | std | notes `armv7-unknown-linux-gnueabi` | ✓ | Armv7-A Linux (kernel 4.15, glibc 2.27) `armv7-unknown-linux-musleabi` | ✓ | Armv7-A Linux with musl 1.2.3 `armv7-unknown-linux-musleabihf` | ✓ | Armv7-A Linux with musl 1.2.3, hardfloat -[`armv7-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | Armv7-A OpenHarmony [`armv7a-none-eabi`](platform-support/arm-none-eabi.md) | * | Bare Armv7-A [`armv7r-none-eabi`](platform-support/armv7r-none-eabi.md) | * | Bare Armv7-R [`armv7r-none-eabihf`](platform-support/armv7r-none-eabi.md) | * | Bare Armv7-R, hardfloat @@ -205,7 +206,6 @@ target | std | notes [`x86_64-pc-windows-gnullvm`](platform-support/pc-windows-gnullvm.md) | ✓ | 64-bit x86 MinGW (Windows 10+), LLVM ABI [`x86_64-unknown-fuchsia`](platform-support/fuchsia.md) | ✓ | 64-bit x86 Fuchsia `x86_64-unknown-linux-gnux32` | ✓ | 64-bit Linux (x32 ABI) (kernel 4.15, glibc 2.27) -[`x86_64-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | x86_64 OpenHarmony [`x86_64-unknown-none`](platform-support/x86_64-unknown-none.md) | * | Freestanding/bare-metal x86_64, softfloat [`x86_64-unknown-redox`](platform-support/redox.md) | ✓ | Redox OS [`x86_64-unknown-uefi`](platform-support/unknown-uefi.md) | ? | 64-bit UEFI From 7279acf2025a5affd51862d126624ed0f49f701b Mon Sep 17 00:00:00 2001 From: LuuuXXX Date: Sat, 15 Feb 2025 09:19:02 +0800 Subject: [PATCH 22/30] use measureme-12.0.1 --- Cargo.lock | 12 ++++++------ compiler/rustc_codegen_llvm/Cargo.toml | 3 +-- compiler/rustc_data_structures/Cargo.toml | 3 +-- compiler/rustc_query_impl/Cargo.toml | 3 +-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea5d9bffefbfb..d4ca5ce5d59dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2179,10 +2179,10 @@ dependencies = [ ] [[package]] -name = "measureme-mirror" +name = "measureme" version = "12.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe110855993552cfa51a5018e8cdf2acf7f948c46136322017a9a8484ffc5ea8" +checksum = "570a507d8948a66a97f42cbbaf8a6bb9516a51017d4ee949502ad7a10a864395" dependencies = [ "log", "memmap2", @@ -2275,7 +2275,7 @@ dependencies = [ "libc", "libffi", "libloading", - "measureme", + "measureme 11.0.1", "rand 0.9.0", "regex", "rustc_version", @@ -3379,7 +3379,7 @@ dependencies = [ "gimli 0.30.0", "itertools", "libc", - "measureme-mirror", + "measureme 12.0.1", "object 0.36.7", "rustc-demangle", "rustc_abi", @@ -3497,7 +3497,7 @@ dependencies = [ "indexmap", "jobserver", "libc", - "measureme-mirror", + "measureme 12.0.1", "memmap2", "parking_lot", "portable-atomic", @@ -4263,7 +4263,7 @@ dependencies = [ name = "rustc_query_impl" version = "0.0.0" dependencies = [ - "measureme-mirror", + "measureme 12.0.1", "rustc_data_structures", "rustc_errors", "rustc_hashes", diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 1122883914d83..ec1fd4b641a27 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -14,8 +14,7 @@ bitflags = "2.4.1" gimli = "0.30" itertools = "0.12" libc = "0.2" -# FIXME: waiting for the new version of measureme. (https://github.com/rust-lang/measureme/pull/240) -measureme = { package = "measureme-mirror", version = "12.0.1" } +measureme = "12.0.1" object = { version = "0.36.3", default-features = false, features = ["std", "read"] } rustc-demangle = "0.1.21" rustc_abi = { path = "../rustc_abi" } diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index 6629f2a751653..df3bee6ee9cc8 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -12,8 +12,7 @@ elsa = "1.11.0" ena = "0.14.3" indexmap = "2.4.0" jobserver_crate = { version = "0.1.28", package = "jobserver" } -# FIXME: waiting for the new version of measureme. (https://github.com/rust-lang/measureme/pull/240) -measureme = { package = "measureme-mirror", version = "12.0.1" } +measureme = "12.0.1" rustc-hash = "2.0.0" rustc-rayon = { version = "0.5.1", features = ["indexmap"] } rustc-stable-hash = { version = "0.1.0", features = ["nightly"] } diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml index 14a7391f10890..b6773fb460fd4 100644 --- a/compiler/rustc_query_impl/Cargo.toml +++ b/compiler/rustc_query_impl/Cargo.toml @@ -5,8 +5,7 @@ edition = "2024" [dependencies] # tidy-alphabetical-start -# FIXME: waiting for the new version of measureme. (https://github.com/rust-lang/measureme/pull/240) -measureme = { package = "measureme-mirror", version = "12.0.1" } +measureme = "12.0.1" rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_hashes = { path = "../rustc_hashes" } From 6efacfb7a59ebde2620398861713fae136060a04 Mon Sep 17 00:00:00 2001 From: LuuuXXX Date: Mon, 24 Feb 2025 11:42:07 +0800 Subject: [PATCH 23/30] add fix for full tools and sanitizer --- Cargo.lock | 20 +++++++++---------- src/bootstrap/src/core/build_steps/llvm.rs | 13 ++++++++++++ .../docker/host-x86_64/dist-ohos/Dockerfile | 5 ++++- .../rustc/src/platform-support/openharmony.md | 13 +++++++++++- src/gcc | 2 +- src/llvm-project | 2 +- src/tools/cargo | 2 +- src/tools/enzyme | 2 +- 8 files changed, 43 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4ca5ce5d59dc..05a2534c85737 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1997,22 +1997,22 @@ dependencies = [ ] [[package]] -name = "libffi" -version = "3.2.0" +name = "libffi-sys2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce826c243048e3d5cec441799724de52e2d42f820468431fc3fceee2341871e2" +checksum = "47aedd9774ffb3dcab5c96f593cb5a0caf421a5e38b16bab3b8cdef5facb6ea2" dependencies = [ - "libc", - "libffi-sys", + "cc", ] [[package]] -name = "libffi-sys" -version = "2.3.0" +name = "libffi2" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36115160c57e8529781b4183c2bb51fdc1f6d6d1ed345591d84be7703befb3c" +checksum = "c88a3402cad8ff58216ec6d07e5ecfa9e15fc36613c0a832db541e4e1a718a7b" dependencies = [ - "cc", + "libc", + "libffi-sys2", ] [[package]] @@ -2273,7 +2273,7 @@ dependencies = [ "directories", "getrandom 0.3.1", "libc", - "libffi", + "libffi2", "libloading", "measureme 11.0.1", "rand 0.9.0", diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 5919e989b3413..6a392b6bd7c1e 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -814,6 +814,10 @@ fn configure_cmake( cflags.push(s); } + if target.contains("ohos") { + cflags.push(" -D_LINUX_SYSINFO_H"); + } + if builder.config.llvm_clang_cl.is_some() { cflags.push(format!(" --target={target}")); } @@ -834,6 +838,11 @@ fn configure_cmake( cxxflags.push(" "); cxxflags.push(s); } + + if target.contains("ohos") { + cxxflags.push(" -D_LINUX_SYSINFO_H"); + } + if builder.config.llvm_clang_cl.is_some() { cxxflags.push(format!(" --target={target}")); } @@ -1220,6 +1229,10 @@ impl Step for Sanitizers { cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); cfg.define("LLVM_CONFIG_PATH", &llvm_config); + if self.target.contains("ohos") { + cfg.define("COMPILER_RT_USE_BUILTINS_LIBRARY", "ON"); + } + // On Darwin targets the sanitizer runtimes are build as universal binaries. // Unfortunately sccache currently lacks support to build them successfully. // Disable compiler launcher on Darwin targets to avoid potential issues. diff --git a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile index 23c0c8b49d2b3..2c514fa0d4ddd 100644 --- a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile @@ -60,7 +60,10 @@ ENV \ ENV RUST_CONFIGURE_ARGS \ --enable-profiler \ - --disable-docs + --disable-docs \ + --tools=cargo,clippy,rustdocs,rustfmt,rust-analyzer,rust-analyzer-proc-macro-srv,analysis,src,wasm-component-ld \ + --enable-extended \ + --enable-sanitizers ENV SCRIPT python3 ../x.py dist --host=$TARGETS --target $TARGETS diff --git a/src/doc/rustc/src/platform-support/openharmony.md b/src/doc/rustc/src/platform-support/openharmony.md index 1632f44aeecc0..5988c468a8ba8 100644 --- a/src/doc/rustc/src/platform-support/openharmony.md +++ b/src/doc/rustc/src/platform-support/openharmony.md @@ -1,6 +1,6 @@ # `*-unknown-linux-ohos` -**Tier: 2** +**Tier: 2（with Host Tools）** * aarch64-unknown-linux-ohos * armv7-unknown-linux-ohos @@ -18,6 +18,17 @@ system. - Amanieu d'Antras ([@Amanieu](https://github.com/Amanieu)) - Lu Binglun ([@lubinglun](https://github.com/lubinglun)) +## Requirements + +All the ohos targets of Tier 2 with host tools support all extended rust tools. +(exclude `miri`, the support of `miri` will be added soon) + +### Host toolchain + +The targets require a reasonably up-to-date OpenHarmony SDK on the host. + +The targets support `cargo`, which require [ohos-openssl](https://github.com/ohos-rs/ohos-openssl). + ## Setup The OpenHarmony SDK doesn't currently support Rust compilation directly, so diff --git a/src/gcc b/src/gcc index 48664a6cab29d..fd3498bff0b93 160000 --- a/src/gcc +++ b/src/gcc @@ -1 +1 @@ -Subproject commit 48664a6cab29d48138ffa004b7978d52ef73e3ac +Subproject commit fd3498bff0b939dda91d56960acc33d55f2f9cdf diff --git a/src/llvm-project b/src/llvm-project index 1c3bb96fdb6db..92e80685d0d5d 160000 --- a/src/llvm-project +++ b/src/llvm-project @@ -1 +1 @@ -Subproject commit 1c3bb96fdb6db7b8e8f24edb016099c223fdd27e +Subproject commit 92e80685d0d5dcea3ccf321995c43b72338639c6 diff --git a/src/tools/cargo b/src/tools/cargo index 2622e844bc1e2..ce948f4616e3d 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit 2622e844bc1e2e6123e54e94e4706f7b6195ce3d +Subproject commit ce948f4616e3d4277e30c75c8bb01e094910df39 diff --git a/src/tools/enzyme b/src/tools/enzyme index a35f4f773118c..5004a8f6f5d84 160000 --- a/src/tools/enzyme +++ b/src/tools/enzyme @@ -1 +1 @@ -Subproject commit a35f4f773118ccfbd8d05102eb12a34097b1ee55 +Subproject commit 5004a8f6f5d8468b64fae457afb7d96e1784c783 From 4dab55bcaa3f1a60f11b3ff36159c199bc210616 Mon Sep 17 00:00:00 2001 From: LuuuXXX Date: Tue, 4 Mar 2025 17:38:06 +0800 Subject: [PATCH 24/30] Revert "add fix for full tools and sanitizer" This reverts commit 6efacfb7a59ebde2620398861713fae136060a04. --- Cargo.lock | 20 +++++++++---------- src/bootstrap/src/core/build_steps/llvm.rs | 13 ------------ .../docker/host-x86_64/dist-ohos/Dockerfile | 5 +---- .../rustc/src/platform-support/openharmony.md | 13 +----------- src/gcc | 2 +- src/llvm-project | 2 +- src/tools/cargo | 2 +- src/tools/enzyme | 2 +- 8 files changed, 16 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05a2534c85737..d4ca5ce5d59dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1997,22 +1997,22 @@ dependencies = [ ] [[package]] -name = "libffi-sys2" -version = "2.4.0" +name = "libffi" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47aedd9774ffb3dcab5c96f593cb5a0caf421a5e38b16bab3b8cdef5facb6ea2" +checksum = "ce826c243048e3d5cec441799724de52e2d42f820468431fc3fceee2341871e2" dependencies = [ - "cc", + "libc", + "libffi-sys", ] [[package]] -name = "libffi2" -version = "3.3.0" +name = "libffi-sys" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c88a3402cad8ff58216ec6d07e5ecfa9e15fc36613c0a832db541e4e1a718a7b" +checksum = "f36115160c57e8529781b4183c2bb51fdc1f6d6d1ed345591d84be7703befb3c" dependencies = [ - "libc", - "libffi-sys2", + "cc", ] [[package]] @@ -2273,7 +2273,7 @@ dependencies = [ "directories", "getrandom 0.3.1", "libc", - "libffi2", + "libffi", "libloading", "measureme 11.0.1", "rand 0.9.0", diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 6a392b6bd7c1e..5919e989b3413 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -814,10 +814,6 @@ fn configure_cmake( cflags.push(s); } - if target.contains("ohos") { - cflags.push(" -D_LINUX_SYSINFO_H"); - } - if builder.config.llvm_clang_cl.is_some() { cflags.push(format!(" --target={target}")); } @@ -838,11 +834,6 @@ fn configure_cmake( cxxflags.push(" "); cxxflags.push(s); } - - if target.contains("ohos") { - cxxflags.push(" -D_LINUX_SYSINFO_H"); - } - if builder.config.llvm_clang_cl.is_some() { cxxflags.push(format!(" --target={target}")); } @@ -1229,10 +1220,6 @@ impl Step for Sanitizers { cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); cfg.define("LLVM_CONFIG_PATH", &llvm_config); - if self.target.contains("ohos") { - cfg.define("COMPILER_RT_USE_BUILTINS_LIBRARY", "ON"); - } - // On Darwin targets the sanitizer runtimes are build as universal binaries. // Unfortunately sccache currently lacks support to build them successfully. // Disable compiler launcher on Darwin targets to avoid potential issues. diff --git a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile index 2c514fa0d4ddd..23c0c8b49d2b3 100644 --- a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile @@ -60,10 +60,7 @@ ENV \ ENV RUST_CONFIGURE_ARGS \ --enable-profiler \ - --disable-docs \ - --tools=cargo,clippy,rustdocs,rustfmt,rust-analyzer,rust-analyzer-proc-macro-srv,analysis,src,wasm-component-ld \ - --enable-extended \ - --enable-sanitizers + --disable-docs ENV SCRIPT python3 ../x.py dist --host=$TARGETS --target $TARGETS diff --git a/src/doc/rustc/src/platform-support/openharmony.md b/src/doc/rustc/src/platform-support/openharmony.md index 5988c468a8ba8..1632f44aeecc0 100644 --- a/src/doc/rustc/src/platform-support/openharmony.md +++ b/src/doc/rustc/src/platform-support/openharmony.md @@ -1,6 +1,6 @@ # `*-unknown-linux-ohos` -**Tier: 2（with Host Tools）** +**Tier: 2** * aarch64-unknown-linux-ohos * armv7-unknown-linux-ohos @@ -18,17 +18,6 @@ system. - Amanieu d'Antras ([@Amanieu](https://github.com/Amanieu)) - Lu Binglun ([@lubinglun](https://github.com/lubinglun)) -## Requirements - -All the ohos targets of Tier 2 with host tools support all extended rust tools. -(exclude `miri`, the support of `miri` will be added soon) - -### Host toolchain - -The targets require a reasonably up-to-date OpenHarmony SDK on the host. - -The targets support `cargo`, which require [ohos-openssl](https://github.com/ohos-rs/ohos-openssl). - ## Setup The OpenHarmony SDK doesn't currently support Rust compilation directly, so diff --git a/src/gcc b/src/gcc index fd3498bff0b93..48664a6cab29d 160000 --- a/src/gcc +++ b/src/gcc @@ -1 +1 @@ -Subproject commit fd3498bff0b939dda91d56960acc33d55f2f9cdf +Subproject commit 48664a6cab29d48138ffa004b7978d52ef73e3ac diff --git a/src/llvm-project b/src/llvm-project index 92e80685d0d5d..1c3bb96fdb6db 160000 --- a/src/llvm-project +++ b/src/llvm-project @@ -1 +1 @@ -Subproject commit 92e80685d0d5dcea3ccf321995c43b72338639c6 +Subproject commit 1c3bb96fdb6db7b8e8f24edb016099c223fdd27e diff --git a/src/tools/cargo b/src/tools/cargo index ce948f4616e3d..2622e844bc1e2 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit ce948f4616e3d4277e30c75c8bb01e094910df39 +Subproject commit 2622e844bc1e2e6123e54e94e4706f7b6195ce3d diff --git a/src/tools/enzyme b/src/tools/enzyme index 5004a8f6f5d84..a35f4f773118c 160000 --- a/src/tools/enzyme +++ b/src/tools/enzyme @@ -1 +1 @@ -Subproject commit 5004a8f6f5d8468b64fae457afb7d96e1784c783 +Subproject commit a35f4f773118ccfbd8d05102eb12a34097b1ee55 From 3eb04fd590527a103a70027a9da5ed9b20b76238 Mon Sep 17 00:00:00 2001 From: LuuuXXX Date: Tue, 4 Mar 2025 17:55:06 +0800 Subject: [PATCH 25/30] add support for extend rust tools and sanitizer --- src/bootstrap/src/core/build_steps/llvm.rs | 11 ++++++++++- src/ci/docker/host-x86_64/dist-ohos/Dockerfile | 5 ++++- src/doc/rustc/src/platform-support/openharmony.md | 13 ++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 5919e989b3413..3a32487a17cf4 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -813,7 +813,9 @@ fn configure_cmake( cflags.push(" "); cflags.push(s); } - + if target.contains("ohos") { + cflags.push(" -D_LINUX_SYSINFO_H"); + } if builder.config.llvm_clang_cl.is_some() { cflags.push(format!(" --target={target}")); } @@ -834,6 +836,9 @@ fn configure_cmake( cxxflags.push(" "); cxxflags.push(s); } + if target.contains("ohos") { + cxxflags.push(" -D_LINUX_SYSINFO_H"); + } if builder.config.llvm_clang_cl.is_some() { cxxflags.push(format!(" --target={target}")); } @@ -1220,6 +1225,10 @@ impl Step for Sanitizers { cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); cfg.define("LLVM_CONFIG_PATH", &llvm_config); + if self.target.contains("ohos") { + cfg.define("COMPILER_RT_USE_BUILTINS_LIBRARY", "ON"); + } + // On Darwin targets the sanitizer runtimes are build as universal binaries. // Unfortunately sccache currently lacks support to build them successfully. // Disable compiler launcher on Darwin targets to avoid potential issues. diff --git a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile index 23c0c8b49d2b3..2c514fa0d4ddd 100644 --- a/src/ci/docker/host-x86_64/dist-ohos/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-ohos/Dockerfile @@ -60,7 +60,10 @@ ENV \ ENV RUST_CONFIGURE_ARGS \ --enable-profiler \ - --disable-docs + --disable-docs \ + --tools=cargo,clippy,rustdocs,rustfmt,rust-analyzer,rust-analyzer-proc-macro-srv,analysis,src,wasm-component-ld \ + --enable-extended \ + --enable-sanitizers ENV SCRIPT python3 ../x.py dist --host=$TARGETS --target $TARGETS diff --git a/src/doc/rustc/src/platform-support/openharmony.md b/src/doc/rustc/src/platform-support/openharmony.md index 1632f44aeecc0..a2107b48a8671 100644 --- a/src/doc/rustc/src/platform-support/openharmony.md +++ b/src/doc/rustc/src/platform-support/openharmony.md @@ -1,6 +1,6 @@ # `*-unknown-linux-ohos` -**Tier: 2** +**Tier: 2 (with Host Tools)** * aarch64-unknown-linux-ohos * armv7-unknown-linux-ohos @@ -18,6 +18,17 @@ system. - Amanieu d'Antras ([@Amanieu](https://github.com/Amanieu)) - Lu Binglun ([@lubinglun](https://github.com/lubinglun)) +## Requirements + +All the ohos targets of Tier 2 with host tools support all extended rust tools. +(exclude `miri`, the support of `miri` will be added soon) + +### Host toolchain + +The targets require a reasonably up-to-date OpenHarmony SDK on the host. + +The targets support `cargo`, which require [ohos-openssl](https://github.com/ohos-rs/ohos-openssl). + ## Setup The OpenHarmony SDK doesn't currently support Rust compilation directly, so From 6463590f0ce9ae9113bba171d8f9ea0d64c7e47a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 15 Feb 2025 14:25:43 +0100 Subject: [PATCH 26/30] Postprocess test suite metrics into GitHub summary --- .github/workflows/ci.yml | 17 +++++ src/ci/citool/Cargo.lock | 9 +++ src/ci/citool/Cargo.toml | 2 + src/ci/citool/src/main.rs | 15 ++++ src/ci/citool/src/metrics.rs | 138 +++++++++++++++++++++++++++++++++++ 5 files changed, 181 insertions(+) create mode 100644 src/ci/citool/src/metrics.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cfbb4e77c465..a91598586b9fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,6 +182,13 @@ jobs: - name: show the current environment run: src/ci/scripts/dump-environment.sh + # Pre-build citool before the following step uninstalls rustup + # Build is into the build directory, to avoid modifying sources + - name: build citool + run: | + cd src/ci/citool + CARGO_TARGET_DIR=../../../build/citool cargo build + - name: run the build # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs. run: src/ci/scripts/run-build-from-ci.sh 2>&1 @@ -218,6 +225,16 @@ jobs: # erroring about invalid credentials instead. if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1' + - name: postprocess metrics into the summary + run: | + if [ -f build/metrics.json ]; then + ./build/citool/debug/citool postprocess-metrics build/metrics.json ${GITHUB_STEP_SUMMARY} + elif [ -f obj/build/metrics.json ]; then + ./build/citool/debug/citool postprocess-metrics obj/build/metrics.json ${GITHUB_STEP_SUMMARY} + else + echo "No metrics.json found" + fi + - name: upload job metrics to DataDog if: needs.calculate_matrix.outputs.run_type != 'pr' env: diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock index 39b6b44da643b..e5be2d42472b7 100644 --- a/src/ci/citool/Cargo.lock +++ b/src/ci/citool/Cargo.lock @@ -58,11 +58,20 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "build_helper" +version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "citool" version = "0.1.0" dependencies = [ "anyhow", + "build_helper", "clap", "insta", "serde", diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml index e77c67c71477a..11172832cb818 100644 --- a/src/ci/citool/Cargo.toml +++ b/src/ci/citool/Cargo.toml @@ -10,6 +10,8 @@ serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" +build_helper = { path = "../../build_helper" } + [dev-dependencies] insta = "1" diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index ad9cc8b82a6f9..cef92e998da1a 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -1,3 +1,5 @@ +mod metrics; + use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::Command; @@ -6,6 +8,8 @@ use anyhow::Context; use clap::Parser; use serde_yaml::Value; +use crate::metrics::postprocess_metrics; + const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker"); const JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../github-actions/jobs.yml"); @@ -338,6 +342,14 @@ enum Args { #[clap(long = "type", default_value = "auto")] job_type: JobType, }, + /// Postprocess the metrics.json file generated by bootstrap. + PostprocessMetrics { + /// Path to the metrics.json file + metrics_path: PathBuf, + /// Path to a file where the postprocessed metrics summary will be stored. + /// Usually, this will be GITHUB_STEP_SUMMARY on CI. + summary_path: PathBuf, + }, } #[derive(clap::ValueEnum, Clone)] @@ -369,6 +381,9 @@ fn main() -> anyhow::Result<()> { Args::RunJobLocally { job_type, name } => { run_workflow_locally(load_db(default_jobs_file)?, job_type, name)? } + Args::PostprocessMetrics { metrics_path, summary_path } => { + postprocess_metrics(&metrics_path, &summary_path)?; + } } Ok(()) diff --git a/src/ci/citool/src/metrics.rs b/src/ci/citool/src/metrics.rs new file mode 100644 index 0000000000000..9a1c7c4d910ad --- /dev/null +++ b/src/ci/citool/src/metrics.rs @@ -0,0 +1,138 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Write; +use std::path::Path; + +use anyhow::Context; +use build_helper::metrics::{JsonNode, JsonRoot, TestOutcome, TestSuite, TestSuiteMetadata}; + +pub fn postprocess_metrics(metrics_path: &Path, summary_path: &Path) -> anyhow::Result<()> { + let metrics = load_metrics(metrics_path)?; + + let mut file = File::options() + .append(true) + .create(true) + .open(summary_path) + .with_context(|| format!("Cannot open summary file at {summary_path:?}"))?; + + record_test_suites(&metrics, &mut file)?; + + Ok(()) +} + +fn record_test_suites(metrics: &JsonRoot, file: &mut File) -> anyhow::Result<()> { + let suites = get_test_suites(&metrics); + + if !suites.is_empty() { + let aggregated = aggregate_test_suites(&suites); + let table = render_table(aggregated); + writeln!(file, "\n# Test results\n")?; + writeln!(file, "{table}")?; + } else { + eprintln!("No test suites found in metrics"); + } + + Ok(()) +} + +fn render_table(suites: BTreeMap) -> String { + use std::fmt::Write; + + let mut table = "| Test suite | Passed ✅ | Ignored 🚫 | Failed ❌ |\n".to_string(); + writeln!(table, "|:------|------:|------:|------:|").unwrap(); + + fn write_row( + buffer: &mut String, + name: &str, + record: &TestSuiteRecord, + surround: &str, + ) -> std::fmt::Result { + let TestSuiteRecord { passed, ignored, failed } = record; + let total = (record.passed + record.ignored + record.failed) as f64; + let passed_pct = ((*passed as f64) / total) * 100.0; + let ignored_pct = ((*ignored as f64) / total) * 100.0; + let failed_pct = ((*failed as f64) / total) * 100.0; + + write!(buffer, "| {surround}{name}{surround} |")?; + write!(buffer, " {surround}{passed} ({passed_pct:.0}%){surround} |")?; + write!(buffer, " {surround}{ignored} ({ignored_pct:.0}%){surround} |")?; + writeln!(buffer, " {surround}{failed} ({failed_pct:.0}%){surround} |")?; + + Ok(()) + } + + let mut total = TestSuiteRecord::default(); + for (name, record) in suites { + write_row(&mut table, &name, &record, "").unwrap(); + total.passed += record.passed; + total.ignored += record.ignored; + total.failed += record.failed; + } + write_row(&mut table, "Total", &total, "**").unwrap(); + table +} + +#[derive(Default)] +struct TestSuiteRecord { + passed: u64, + ignored: u64, + failed: u64, +} + +fn aggregate_test_suites(suites: &[&TestSuite]) -> BTreeMap { + let mut records: BTreeMap = BTreeMap::new(); + for suite in suites { + let name = match &suite.metadata { + TestSuiteMetadata::CargoPackage { crates, stage, .. } => { + format!("{} (stage {stage})", crates.join(", ")) + } + TestSuiteMetadata::Compiletest { suite, stage, .. } => { + format!("{suite} (stage {stage})") + } + }; + let record = records.entry(name).or_default(); + for test in &suite.tests { + match test.outcome { + TestOutcome::Passed => { + record.passed += 1; + } + TestOutcome::Failed => { + record.failed += 1; + } + TestOutcome::Ignored { .. } => { + record.ignored += 1; + } + } + } + } + records +} + +fn get_test_suites(metrics: &JsonRoot) -> Vec<&TestSuite> { + fn visit_test_suites<'a>(nodes: &'a [JsonNode], suites: &mut Vec<&'a TestSuite>) { + for node in nodes { + match node { + JsonNode::RustbuildStep { children, .. } => { + visit_test_suites(&children, suites); + } + JsonNode::TestSuite(suite) => { + suites.push(&suite); + } + } + } + } + + let mut suites = vec![]; + for invocation in &metrics.invocations { + visit_test_suites(&invocation.children, &mut suites); + } + suites +} + +fn load_metrics(path: &Path) -> anyhow::Result { + let metrics = std::fs::read_to_string(path) + .with_context(|| format!("Cannot read JSON metrics from {path:?}"))?; + let metrics: JsonRoot = serde_json::from_str(&metrics) + .with_context(|| format!("Cannot deserialize JSON metrics from {path:?}"))?; + Ok(metrics) +} From 2e5ab4e6a07a9ebdb6e593ca19112148250e6906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 19 Feb 2025 16:06:47 +0100 Subject: [PATCH 27/30] Store bootstrap command-line into metrics --- src/bootstrap/src/utils/metrics.rs | 8 ++++++++ src/build_helper/src/metrics.rs | 2 ++ 2 files changed, 10 insertions(+) diff --git a/src/bootstrap/src/utils/metrics.rs b/src/bootstrap/src/utils/metrics.rs index b51fd490535a3..57766fd63fb11 100644 --- a/src/bootstrap/src/utils/metrics.rs +++ b/src/bootstrap/src/utils/metrics.rs @@ -200,6 +200,14 @@ impl BuildMetrics { } }; invocations.push(JsonInvocation { + // The command-line invocation with which bootstrap was invoked. + // Skip the first argument, as it is a potentially long absolute + // path that is not interesting. + cmdline: std::env::args_os() + .skip(1) + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>() + .join(" "), start_time: state .invocation_start .duration_since(SystemTime::UNIX_EPOCH) diff --git a/src/build_helper/src/metrics.rs b/src/build_helper/src/metrics.rs index 538c33e9b1591..c9be63ed534b0 100644 --- a/src/build_helper/src/metrics.rs +++ b/src/build_helper/src/metrics.rs @@ -12,6 +12,8 @@ pub struct JsonRoot { #[derive(Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub struct JsonInvocation { + // Remembers the command-line invocation with which bootstrap was invoked. + pub cmdline: String, // Unix timestamp in seconds // // This is necessary to easily correlate this invocation with logs or other data. From ead58ea6f8bf860459fe5351a83717a8e591b0af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 4 Mar 2025 12:29:51 +0100 Subject: [PATCH 28/30] Move `BuildStep` and metric logging into `build_helper` --- src/build_helper/src/metrics.rs | 80 +++++++++++++++++++++++++++++++ src/tools/opt-dist/src/metrics.rs | 73 ++-------------------------- 2 files changed, 83 insertions(+), 70 deletions(-) diff --git a/src/build_helper/src/metrics.rs b/src/build_helper/src/metrics.rs index c9be63ed534b0..4b585c1518246 100644 --- a/src/build_helper/src/metrics.rs +++ b/src/build_helper/src/metrics.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use serde_derive::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] @@ -100,3 +102,81 @@ fn null_as_f64_nan<'de, D: serde::Deserializer<'de>>(d: D) -> Result::deserialize(d).map(|f| f.unwrap_or(f64::NAN)) } + +/// Represents a single bootstrap step, with the accumulated duration of all its children. +#[derive(Clone, Debug)] +pub struct BuildStep { + pub r#type: String, + pub children: Vec, + pub duration: Duration, +} + +impl BuildStep { + /// Create a `BuildStep` representing a single invocation of bootstrap. + /// The most important thing is that the build step aggregates the + /// durations of all children, so that it can be easily accessed. + pub fn from_invocation(invocation: &JsonInvocation) -> Self { + fn parse(node: &JsonNode) -> Option { + match node { + JsonNode::RustbuildStep { + type_: kind, + children, + duration_excluding_children_sec, + .. + } => { + let children: Vec<_> = children.into_iter().filter_map(parse).collect(); + let children_duration = children.iter().map(|c| c.duration).sum::(); + Some(BuildStep { + r#type: kind.to_string(), + children, + duration: children_duration + + Duration::from_secs_f64(*duration_excluding_children_sec), + }) + } + JsonNode::TestSuite(_) => None, + } + } + + let duration = Duration::from_secs_f64(invocation.duration_including_children_sec); + let children: Vec<_> = invocation.children.iter().filter_map(parse).collect(); + Self { r#type: "total".to_string(), children, duration } + } + + pub fn find_all_by_type(&self, r#type: &str) -> Vec<&Self> { + let mut result = Vec::new(); + self.find_by_type(r#type, &mut result); + result + } + + fn find_by_type<'a>(&'a self, r#type: &str, result: &mut Vec<&'a Self>) { + if self.r#type == r#type { + result.push(self); + } + for child in &self.children { + child.find_by_type(r#type, result); + } + } +} + +/// Writes build steps into a nice indented table. +pub fn format_build_steps(root: &BuildStep) -> String { + use std::fmt::Write; + + let mut substeps: Vec<(u32, &BuildStep)> = Vec::new(); + + fn visit<'a>(step: &'a BuildStep, level: u32, substeps: &mut Vec<(u32, &'a BuildStep)>) { + substeps.push((level, step)); + for child in &step.children { + visit(child, level + 1, substeps); + } + } + + visit(root, 0, &mut substeps); + + let mut output = String::new(); + for (level, step) in substeps { + let label = format!("{}{}", ".".repeat(level as usize), step.r#type); + writeln!(output, "{label:<65}{:>8.2}s", step.duration.as_secs_f64()).unwrap(); + } + output +} diff --git a/src/tools/opt-dist/src/metrics.rs b/src/tools/opt-dist/src/metrics.rs index 89c4cb12d4c3f..0a745566eb514 100644 --- a/src/tools/opt-dist/src/metrics.rs +++ b/src/tools/opt-dist/src/metrics.rs @@ -1,33 +1,10 @@ use std::time::Duration; -use build_helper::metrics::{JsonNode, JsonRoot}; +use build_helper::metrics::{BuildStep, JsonRoot, format_build_steps}; use camino::Utf8Path; use crate::timer::TimerSection; -#[derive(Clone, Debug)] -pub struct BuildStep { - r#type: String, - children: Vec, - duration: Duration, -} - -impl BuildStep { - pub fn find_all_by_type(&self, r#type: &str) -> Vec<&BuildStep> { - let mut result = Vec::new(); - self.find_by_type(r#type, &mut result); - result - } - fn find_by_type<'a>(&'a self, r#type: &str, result: &mut Vec<&'a BuildStep>) { - if self.r#type == r#type { - result.push(self); - } - for child in &self.children { - child.find_by_type(r#type, result); - } - } -} - /// Loads the metrics of the most recent bootstrap execution from a metrics.json file. pub fn load_metrics(path: &Utf8Path) -> anyhow::Result { let content = std::fs::read(path.as_std_path())?; @@ -37,30 +14,7 @@ pub fn load_metrics(path: &Utf8Path) -> anyhow::Result { .pop() .ok_or_else(|| anyhow::anyhow!("No bootstrap invocation found in metrics file"))?; - fn parse(node: JsonNode) -> Option { - match node { - JsonNode::RustbuildStep { - type_: kind, - children, - duration_excluding_children_sec, - .. - } => { - let children: Vec<_> = children.into_iter().filter_map(parse).collect(); - let children_duration = children.iter().map(|c| c.duration).sum::(); - Some(BuildStep { - r#type: kind.to_string(), - children, - duration: children_duration - + Duration::from_secs_f64(duration_excluding_children_sec), - }) - } - JsonNode::TestSuite(_) => None, - } - } - - let duration = Duration::from_secs_f64(invocation.duration_including_children_sec); - let children: Vec<_> = invocation.children.into_iter().filter_map(parse).collect(); - Ok(BuildStep { r#type: "root".to_string(), children, duration }) + Ok(BuildStep::from_invocation(&invocation)) } /// Logs the individual metrics in a table and add Rustc and LLVM durations to the passed @@ -82,27 +36,6 @@ pub fn record_metrics(metrics: &BuildStep, timer: &mut TimerSection) { timer.add_duration("Rustc", rustc_duration); } - log_metrics(metrics); -} - -fn log_metrics(metrics: &BuildStep) { - use std::fmt::Write; - - let mut substeps: Vec<(u32, &BuildStep)> = Vec::new(); - - fn visit<'a>(step: &'a BuildStep, level: u32, substeps: &mut Vec<(u32, &'a BuildStep)>) { - substeps.push((level, step)); - for child in &step.children { - visit(child, level + 1, substeps); - } - } - - visit(metrics, 0, &mut substeps); - - let mut output = String::new(); - for (level, step) in substeps { - let label = format!("{}{}", ".".repeat(level as usize), step.r#type); - writeln!(output, "{label:<65}{:>8.2}s", step.duration.as_secs_f64()).unwrap(); - } + let output = format_build_steps(metrics); log::info!("Build step durations\n{output}"); } From 7b53ac7ee67fbd62cadd883c860a6df263b6a481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sun, 2 Mar 2025 18:23:11 +0100 Subject: [PATCH 29/30] Record bootstrap step durations into GitHub summary in citool --- src/build_helper/src/metrics.rs | 10 ++++++++-- src/ci/citool/src/metrics.rs | 30 ++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/build_helper/src/metrics.rs b/src/build_helper/src/metrics.rs index 4b585c1518246..eb306550fc4a6 100644 --- a/src/build_helper/src/metrics.rs +++ b/src/build_helper/src/metrics.rs @@ -175,8 +175,14 @@ pub fn format_build_steps(root: &BuildStep) -> String { let mut output = String::new(); for (level, step) in substeps { - let label = format!("{}{}", ".".repeat(level as usize), step.r#type); - writeln!(output, "{label:<65}{:>8.2}s", step.duration.as_secs_f64()).unwrap(); + let label = format!( + "{}{}", + ".".repeat(level as usize), + // Bootstrap steps can be generic and thus contain angle brackets (<...>). + // However, Markdown interprets these as HTML, so we need to escap ethem. + step.r#type.replace('<', "<").replace('>', ">") + ); + writeln!(output, "{label:.<65}{:>8.2}s", step.duration.as_secs_f64()).unwrap(); } output } diff --git a/src/ci/citool/src/metrics.rs b/src/ci/citool/src/metrics.rs index 9a1c7c4d910ad..2386413ed6b1d 100644 --- a/src/ci/citool/src/metrics.rs +++ b/src/ci/citool/src/metrics.rs @@ -4,7 +4,9 @@ use std::io::Write; use std::path::Path; use anyhow::Context; -use build_helper::metrics::{JsonNode, JsonRoot, TestOutcome, TestSuite, TestSuiteMetadata}; +use build_helper::metrics::{ + BuildStep, JsonNode, JsonRoot, TestOutcome, TestSuite, TestSuiteMetadata, format_build_steps, +}; pub fn postprocess_metrics(metrics_path: &Path, summary_path: &Path) -> anyhow::Result<()> { let metrics = load_metrics(metrics_path)?; @@ -15,7 +17,31 @@ pub fn postprocess_metrics(metrics_path: &Path, summary_path: &Path) -> anyhow:: .open(summary_path) .with_context(|| format!("Cannot open summary file at {summary_path:?}"))?; - record_test_suites(&metrics, &mut file)?; + if !metrics.invocations.is_empty() { + writeln!(file, "# Bootstrap steps")?; + record_bootstrap_step_durations(&metrics, &mut file)?; + record_test_suites(&metrics, &mut file)?; + } + + Ok(()) +} + +fn record_bootstrap_step_durations(metrics: &JsonRoot, file: &mut File) -> anyhow::Result<()> { + for invocation in &metrics.invocations { + let step = BuildStep::from_invocation(invocation); + let table = format_build_steps(&step); + eprintln!("Step `{}`\n{table}\n", invocation.cmdline); + writeln!( + file, + r"
+{} +
{table}
+
+", + invocation.cmdline + )?; + } + eprintln!("Recorded {} bootstrap invocation(s)", metrics.invocations.len()); Ok(()) } From f3312609f7a9db95047196387fdd8c69edaa898e Mon Sep 17 00:00:00 2001 From: LuuuXXX Date: Tue, 4 Mar 2025 20:30:57 +0800 Subject: [PATCH 30/30] add note for miri --- src/doc/rustc/src/platform-support/openharmony.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/doc/rustc/src/platform-support/openharmony.md b/src/doc/rustc/src/platform-support/openharmony.md index a2107b48a8671..e772a3d09f3d3 100644 --- a/src/doc/rustc/src/platform-support/openharmony.md +++ b/src/doc/rustc/src/platform-support/openharmony.md @@ -29,6 +29,9 @@ The targets require a reasonably up-to-date OpenHarmony SDK on the host. The targets support `cargo`, which require [ohos-openssl](https://github.com/ohos-rs/ohos-openssl). +`miri` isn't supported yet, since its dependencies (`libffi` and `tikv-jemalloc-sys`) don't support +compiling for the OHOS targets. + ## Setup The OpenHarmony SDK doesn't currently support Rust compilation directly, so