Rollup merge of rust-lang#153801 - ywxt:parallel-test, r=petrochenkov

Add the option to run UI tests with the parallel frontend

This PR adds two arguments for tests:

1. `--parallel-frontend-threads`: specify `-Zthread` to compile test case (currently UI tests only)
2. `--iteration-count`: the number of times to run each test

Also, due to the non-deterministic diagnostic orders and cycle errors, this PR adds the directive `//@ ignore-parallel-frontend` to ignore tests with cycle error when the parallel-frontend is enabled (by `--parallel-frontend-threads`) and enables `//@ compare-output-by-lines` by default.

Context: [#t-compiler/parallel-rustc > Add the parallel front-end test suite @ 💬](https://rust-lang.zulipchat.com/#narrow/channel/187679-t-compiler.2Fparallel-rustc/topic/Add.20the.20parallel.20front-end.20test.20suite/near/578781369)

This PR should work with rust-lang#153797 together.

Author: Zalathar

src/doc/rustc-dev-guide/src/tests/compiletest.md

@@ -836,3 +836,20 @@ In CI, compare modes are only used in one Linux builder, and only with the follo
 Note that compare modes are separate to [revisions](#revisions).
 All revisions are tested when running `./x test tests/ui`, however compare-modes must be
 manually run individually via the `--compare-mode` flag.
+
+## Parallel frontend
+
+Compiletest can be run with the `--parallel-frontend-threads` flag to run the compiler in parallel mode.
+This can be used to check that the compiler produces the same output in parallel mode as in non-parallel mode, and to check for any issues that might arise in parallel mode.
+
+To run the tests in parallel mode, you need to pass the `--parallel-frontend-threads` CLI flag:
+
+```bash
+./x test tests/ui -- --parallel-frontend-threads=N --iteration-count=M
+```
+
+Where `N` is the number of threads to use for the parallel frontend, and `M` is the number of times to run each test in parallel mode (to increase the chances of catching any non-determinism).
+
+Also, when running with `--parallel-frontend-threads`, the `compare-output-by-lines` directive would be implied for all tests, since the output from the parallel frontend can be non-deterministic in terms of the order of lines.
+
+The parallel frontend is available in UI tests only at the moment, and is not currently supported in other test suites.

src/doc/rustc-dev-guide/src/tests/directives.md

@@ -148,6 +148,7 @@ Some examples of `X` in `ignore-X` or `only-X`:
 - When [remote testing] is used: `remote`
 - When particular debuggers are being tested: `cdb`, `gdb`, `lldb`
 - When particular debugger versions are matched: `ignore-gdb-version`
+- When the [parallel frontend] is enabled: `ignore-parallel-frontend`
 - Specific [compare modes]: `compare-mode-polonius`, `compare-mode-chalk`,
   `compare-mode-split-dwarf`, `compare-mode-split-dwarf-single`
 - The two different test modes used by coverage tests:
@@ -233,6 +234,7 @@ The following directives will check LLVM support:
 See also [Debuginfo tests](compiletest.md#debuginfo-tests) for directives for ignoring debuggers.
 
 [remote testing]: running.md#running-tests-on-a-remote-machine
+[parallel frontend]: compiletest.md#parallel-frontend
 [compare modes]: ui.md#compare-modes
 [`x86_64-gnu-debug`]: https://github.com/rust-lang/rust/blob/ab3dba92db355b8d97db915a2dca161a117e959c/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile#L32
 [`aarch64-gnu-debug`]: https://github.com/rust-lang/rust/blob/20c909ff9cdd88d33768a4ddb8952927a675b0ad/src/ci/docker/host-aarch64/aarch64-gnu-debug/Dockerfile#L32

src/tools/compiletest/src/common.rs

@@ -721,9 +721,17 @@ pub struct Config {
     ///
     /// This is forwarded from bootstrap's `jobs` configuration.
     pub jobs: u32,
+
+    /// Number of parallel threads to use for the frontend when building test artifacts.
+    pub parallel_frontend_threads: u32,
+    /// Number of times to execute each test.
+    pub iteration_count: u32,
 }
 
 impl Config {
+    pub const DEFAULT_PARALLEL_FRONTEND_THREADS: u32 = 1;
+    pub const DEFAULT_ITERATION_COUNT: u32 = 1;
+
     /// FIXME: this run scheme is... confusing.
     pub fn run_enabled(&self) -> bool {
         self.run.unwrap_or_else(|| {
@@ -834,6 +842,17 @@ impl Config {
             || self.target_cfg().os == "emscripten";
         !unsupported_target
     }
+
+    /// Whether the parallel frontend is enabled,
+    /// which is the case when `parallel_frontend_threads` is not set to `1`.
+    ///
+    /// - `0` means auto-detect: use the number of available hardware threads on the host.
+    ///   But we treat it as the parallel frontend being enabled in this case.
+    /// - `1` means single-threaded (parallel frontend disabled).
+    /// - `>1` means an explicitly configured thread count.
+    pub fn parallel_frontend_enabled(&self) -> bool {
+        self.parallel_frontend_threads != 1
+    }
 }
 
 /// Known widths of `target_has_atomic`.

src/tools/compiletest/src/directives.rs

@@ -67,7 +67,7 @@ impl EarlyProps {
         let mut props = EarlyProps::default();
 
         iter_directives(
-            config.mode,
+            config,
             file_directives,
             // (dummy comment to force args into vertical layout)
             &mut |ln: &DirectiveLine<'_>| {
@@ -362,7 +362,7 @@ impl TestProps {
             let file_directives = FileDirectives::from_file_contents(testfile, &file_contents);
 
             iter_directives(
-                config.mode,
+                config,
                 &file_directives,
                 // (dummy comment to force args into vertical layout)
                 &mut |ln: &DirectiveLine<'_>| {
@@ -574,43 +574,51 @@ fn check_directive<'a>(
 }
 
 fn iter_directives(
-    mode: TestMode,
+    config: &Config,
     file_directives: &FileDirectives<'_>,
     it: &mut dyn FnMut(&DirectiveLine<'_>),
 ) {
     let testfile = file_directives.path;
 
-    // Coverage tests in coverage-run mode always have these extra directives, without needing to
-    // specify them manually in every test file.
-    //
-    // FIXME(jieyouxu): I feel like there's a better way to do this, leaving for later.
-    if mode == TestMode::CoverageRun {
-        let extra_directives: &[&str] = &[
-            "//@ needs-profiler-runtime",
-            // FIXME(pietroalbini): this test currently does not work on cross-compiled targets
-            // because remote-test is not capable of sending back the *.profraw files generated by
-            // the LLVM instrumentation.
-            "//@ ignore-cross-compile",
-        ];
-        // Process the extra implied directives, with a dummy line number of 0.
-        for directive_str in extra_directives {
-            let directive_line = line_directive(testfile, LineNumber::ZERO, directive_str)
-                .unwrap_or_else(|| panic!("bad extra-directive line: {directive_str:?}"));
-            it(&directive_line);
+    let extra_directives = match config.mode {
+        TestMode::CoverageRun => {
+            // Coverage tests in coverage-run mode always have these extra directives, without needing to
+            // specify them manually in every test file.
+            //
+            // FIXME(jieyouxu): I feel like there's a better way to do this, leaving for later.
+            vec![
+                "//@ needs-profiler-runtime",
+                // FIXME(pietroalbini): this test currently does not work on cross-compiled targets
+                // because remote-test is not capable of sending back the *.profraw files generated by
+                // the LLVM instrumentation.
+                "//@ ignore-cross-compile",
+            ]
+        }
+        TestMode::Codegen if !file_directives.has_explicit_no_std_core_attribute => {
+            // Note: affects all codegen test suites under test mode `codegen`, e.g. `codegen-llvm`.
+            //
+            // Codegen tests automatically receive implied `//@ needs-target-std`, unless
+            // `#![no_std]`/`#![no_core]` attribute was explicitly seen. The rationale is basically to avoid
+            // having to manually maintain a bunch of `//@ needs-target-std` directives esp. for targets
+            // tested/built out-of-tree.
+            vec!["//@ needs-target-std"]
+        }
+        TestMode::Ui if config.parallel_frontend_enabled() => {
+            // UI tests in parallel-frontend mode always have this extra directive, without needing to
+            // specify it manually in every test file.
+            vec!["//@ compare-output-by-lines"]
         }
-    }
 
-    // Note: affects all codegen test suites under test mode `codegen`, e.g. `codegen-llvm`.
-    //
-    // Codegen tests automatically receive implied `//@ needs-target-std`, unless
-    // `#![no_std]`/`#![no_core]` attribute was explicitly seen. The rationale is basically to avoid
-    // having to manually maintain a bunch of `//@ needs-target-std` directives esp. for targets
-    // tested/built out-of-tree.
-    if mode == TestMode::Codegen && !file_directives.has_explicit_no_std_core_attribute {
-        let implied_needs_target_std_line =
-            line_directive(testfile, LineNumber::ZERO, "//@ needs-target-std")
-                .expect("valid `needs-target-std` directive line");
-        it(&implied_needs_target_std_line);
+        _ => {
+            // No extra directives for other test modes.
+            vec![]
+        }
+    };
+
+    for directive_str in extra_directives {
+        let directive_line = line_directive(testfile, LineNumber::ZERO, directive_str)
+            .unwrap_or_else(|| panic!("bad extra-directive line: {directive_str:?}"));
+        it(&directive_line);
     }
 
     for directive_line in &file_directives.lines {
@@ -951,55 +959,52 @@ pub(crate) fn make_test_description(
     let mut should_fail = false;
 
     // Scan through the test file to handle `ignore-*`, `only-*`, and `needs-*` directives.
-    iter_directives(
-        config.mode,
-        file_directives,
-        &mut |ln @ &DirectiveLine { line_number, .. }| {
-            if !ln.applies_to_test_revision(test_revision) {
-                return;
-            }
+    iter_directives(config, file_directives, &mut |ln @ &DirectiveLine { line_number, .. }| {
+        if !ln.applies_to_test_revision(test_revision) {
+            return;
+        }
+
+        // Parse `aux-*` directives, for use by up-to-date checks.
+        parse_and_update_aux(config, ln, aux_props);
 
-            // Parse `aux-*` directives, for use by up-to-date checks.
-            parse_and_update_aux(config, ln, aux_props);
-
-            macro_rules! decision {
-                ($e:expr) => {
-                    match $e {
-                        IgnoreDecision::Ignore { reason } => {
-                            ignore = true;
-                            ignore_message = Some(reason.into());
-                        }
-                        IgnoreDecision::Error { message } => {
-                            error!("{path}:{line_number}: {message}");
-                            *poisoned = true;
-                            return;
-                        }
-                        IgnoreDecision::Continue => {}
+        macro_rules! decision {
+            ($e:expr) => {
+                match $e {
+                    IgnoreDecision::Ignore { reason } => {
+                        ignore = true;
+                        ignore_message = Some(reason.into());
                     }
-                };
-            }
+                    IgnoreDecision::Error { message } => {
+                        error!("{path}:{line_number}: {message}");
+                        *poisoned = true;
+                        return;
+                    }
+                    IgnoreDecision::Continue => {}
+                }
+            };
+        }
 
-            decision!(cfg::handle_ignore(&cache.cfg_conditions, ln));
-            decision!(cfg::handle_only(&cache.cfg_conditions, ln));
-            decision!(needs::handle_needs(&cache.needs, config, ln));
-            decision!(ignore_llvm(config, ln));
-            decision!(ignore_backends(config, ln));
-            decision!(needs_backends(config, ln));
-            decision!(ignore_cdb(config, ln));
-            decision!(ignore_gdb(config, ln));
-            decision!(ignore_lldb(config, ln));
-
-            if config.target == "wasm32-unknown-unknown"
-                && config.parse_name_directive(ln, directives::CHECK_RUN_RESULTS)
-            {
-                decision!(IgnoreDecision::Ignore {
-                    reason: "ignored on WASM as the run results cannot be checked there".into(),
-                });
-            }
+        decision!(cfg::handle_ignore(&cache.cfg_conditions, ln));
+        decision!(cfg::handle_only(&cache.cfg_conditions, ln));
+        decision!(needs::handle_needs(&cache.needs, config, ln));
+        decision!(ignore_llvm(config, ln));
+        decision!(ignore_backends(config, ln));
+        decision!(needs_backends(config, ln));
+        decision!(ignore_cdb(config, ln));
+        decision!(ignore_gdb(config, ln));
+        decision!(ignore_lldb(config, ln));
+        decision!(ignore_parallel_frontend(config, ln));
+
+        if config.target == "wasm32-unknown-unknown"
+            && config.parse_name_directive(ln, directives::CHECK_RUN_RESULTS)
+        {
+            decision!(IgnoreDecision::Ignore {
+                reason: "ignored on WASM as the run results cannot be checked there".into(),
+            });
+        }
 
-            should_fail |= config.parse_name_directive(ln, "should-fail");
-        },
-    );
+        should_fail |= config.parse_name_directive(ln, "should-fail");
+    });
 
     // The `should-fail` annotation doesn't apply to pretty tests,
     // since we run the pretty printer across all tests by default.
@@ -1270,6 +1275,17 @@ fn ignore_llvm(config: &Config, line: &DirectiveLine<'_>) -> IgnoreDecision {
     IgnoreDecision::Continue
 }
 
+fn ignore_parallel_frontend(config: &Config, line: &DirectiveLine<'_>) -> IgnoreDecision {
+    if config.parallel_frontend_enabled()
+        && config.parse_name_directive(line, "ignore-parallel-frontend")
+    {
+        return IgnoreDecision::Ignore {
+            reason: "ignored when the parallel frontend is enabled".into(),
+        };
+    }
+    IgnoreDecision::Continue
+}
+
 enum IgnoreDecision {
     Ignore { reason: String },
     Continue,

src/tools/compiletest/src/directives/cfg.rs

@@ -11,6 +11,7 @@ const EXTERNAL_IGNORES_LIST: &[&str] = &[
     "ignore-backends",
     "ignore-gdb-version",
     "ignore-llvm-version",
+    "ignore-parallel-frontend",
     "ignore-pass",
     // tidy-alphabetical-end
 ];

src/tools/compiletest/src/directives/directive_names.rs

@@ -101,6 +101,7 @@ pub(crate) const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "ignore-nvptx64",
     "ignore-nvptx64-nvidia-cuda",
     "ignore-openbsd",
+    "ignore-parallel-frontend",
     "ignore-pass",
     "ignore-powerpc",
     "ignore-powerpc64",

src/tools/compiletest/src/lib.rs

@@ -219,7 +219,14 @@ fn parse_config(args: Vec<String>) -> Config {
             "CODEGEN BACKEND [NAME | PATH]",
         )
         .optflag("", "bypass-ignore-backends", "ignore `//@ ignore-backends` directives")
-        .reqopt("", "jobs", "number of parallel jobs bootstrap was configured with", "JOBS");
+        .reqopt("", "jobs", "number of parallel jobs bootstrap was configured with", "JOBS")
+        .optopt(
+            "",
+            "parallel-frontend-threads",
+            "number of parallel threads to use for the frontend when building test artifacts",
+            "THREADS_COUNT",
+        )
+        .optopt("", "iteration-count", "number of times to execute each test", "COUNT");
 
     let (argv0, args_) = args.split_first().unwrap();
     if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
@@ -369,6 +376,20 @@ fn parse_config(args: Vec<String>) -> Config {
         None => panic!("`--jobs` is required"),
     };
 
+    let parallel_frontend_threads = match matches.opt_str("parallel-frontend-threads") {
+        Some(threads) => {
+            threads.parse::<u32>().expect("expected `--parallel-frontend-threads` to be an `u32`")
+        }
+        None => Config::DEFAULT_PARALLEL_FRONTEND_THREADS,
+    };
+    let iteration_count = match matches.opt_str("iteration-count") {
+        Some(count) => {
+            count.parse::<u32>().expect("expected `--iteration-count` to be a positive integer")
+        }
+        None => Config::DEFAULT_ITERATION_COUNT,
+    };
+    assert!(iteration_count > 0, "`--iteration-count` must be a positive integer");
+
     Config {
         bless: matches.opt_present("bless"),
         fail_fast: matches.opt_present("fail-fast")
@@ -489,6 +510,9 @@ fn parse_config(args: Vec<String>) -> Config {
         bypass_ignore_backends: matches.opt_present("bypass-ignore-backends"),
 
         jobs,
+
+        parallel_frontend_threads,
+        iteration_count,
     }
 }