Auto merge of #62766 - alexcrichton:stabilize-pipelined-compilation, r=oli-obk

rustc: Stabilize options for pipelined compilation

This commit stabilizes options in the compiler necessary for Cargo to
enable "pipelined compilation" by default. The concept of pipelined
compilation, how it's implemented, and what it means for rustc are
documented in #60988. This PR is coupled with a PR against Cargo
(rust-lang/cargo#7143) which updates Cargo's support for pipelined
compliation to rustc, and also enables support by default in Cargo.
(note that the Cargo PR cannot land until this one against rustc lands).

The technical changes performed here were to stabilize the functionality
proposed in #60419 and #60987, the underlying pieces to enable pipelined
compilation support in Cargo. The issues have had some discussion during
stabilization, but the newly stabilized surface area here is:

* A new `--json` flag was added to the compiler.
* The `--json` flag can be passed multiple times.
* The value of the `--json` flag is a comma-separated list of
  directives.
* The `--json` flag cannot be combined with `--color`
* The `--json` flag must be combined with `--error-format=json`
* The acceptable list of directives to `--json` are:
  * `diagnostic-short` - the `rendered` field of diagnostics will have a
    "short" rendering matching `--error-format=short`
  * `diagnostic-rendered-ansi` - the `rendered` field of diagnostics
    will be colorized with ansi color codes embedded in the string field
  * `artifacts` - JSON blobs will be emitted for artifacts being emitted
    by the compiler

The unstable `-Z emit-artifact-notifications` and `--json-rendered`
flags have also been removed during this commit as well.

Closes #60419
Closes #60987
Closes #60988

Author: bors

src/bootstrap/test.rs

@@ -229,6 +229,9 @@ impl Step for Cargo {
         cargo.env("CFG_DISABLE_CROSS_TESTS", "1");
         // Disable a test that has issues with mingw.
         cargo.env("CARGO_TEST_DISABLE_GIT_CLI", "1");
+        // Forcibly disable tests using nightly features since any changes to
+        // those features won't be able to land.
+        cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1");
 
         try_run(
             builder,

src/doc/rustc/src/command-line-arguments.md

@@ -271,3 +271,36 @@ current directory out of pathnames emitted into the object files. The
 replacement is purely textual, with no consideration of the current system's
 pathname syntax. For example `--remap-path-prefix foo=bar` will match
 `foo/lib.rs` but not `./foo/lib.rs`.
+
+## `--json`: configure json messages printed by the compiler
+
+When the `--error-format=json` option is passed to rustc then all of the
+compiler's diagnostic output will be emitted in the form of JSON blobs. The
+`--json` argument can be used in conjunction with `--error-format=json` to
+configure what the JSON blobs contain as well as which ones are emitted.
+
+With `--error-format=json` the compiler will always emit any compiler errors as
+a JSON blob, but the following options are also available to the `--json` flag
+to customize the output:
+
+- `diagnostic-short` - json blobs for diagnostic messages should use the "short"
+  rendering instead of the normal "human" default. This means that the output of
+  `--error-format=short` will be embedded into the JSON diagnostics instead of
+  the default `--error-format=human`.
+
+- `diagnostic-rendered-ansi` - by default JSON blobs in their `rendered` field
+  will contain a plain text rendering of the diagnostic. This option instead
+  indicates that the diagnostic should have embedded ANSI color codes intended
+  to be used to colorize the message in the manner rustc typically already does
+  for terminal outputs. Note that this is usefully combined with crates like
+  `fwdansi` to translate these ANSI codes on Windows to console commands or
+  `strip-ansi-escapes` if you'd like to optionally remove the ansi colors
+  afterwards.
+
+- `artifacts` - this instructs rustc to emit a JSON blob for each artifact that
+  is emitted. An artifact corresponds to a request from the `--emit` CLI
+  argument, and as soon as the artifact is available on the filesystem a
+  notification will be emitted.
+
+Note that it is invalid to combine the `--json` argument with the `--color`
+argument, and it is required to combine `--json` with `--error-format=json`.

src/librustc/session/config.rs

@@ -438,6 +438,10 @@ top_level_options!(
         remap_path_prefix: Vec<(PathBuf, PathBuf)> [UNTRACKED],
 
         edition: Edition [TRACKED],
+
+        // Whether or not we're emitting JSON blobs about each artifact produced
+        // by the compiler.
+        json_artifact_notifications: bool [TRACKED],
     }
 );
 
@@ -625,6 +629,7 @@ impl Default for Options {
             cli_forced_thinlto_off: false,
             remap_path_prefix: Vec::new(),
             edition: DEFAULT_EDITION,
+            json_artifact_notifications: false,
         }
     }
 }
@@ -1459,8 +1464,6 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
          the same values as the target option of the same name"),
     allow_features: Option<Vec<String>> = (None, parse_opt_comma_list, [TRACKED],
         "only allow the listed language features to be enabled in code (space separated)"),
-    emit_artifact_notifications: bool = (false, parse_bool, [UNTRACKED],
-        "emit notifications after each artifact has been output (only in the JSON format)"),
     symbol_mangling_version: SymbolManglingVersion = (SymbolManglingVersion::Legacy,
         parse_symbol_mangling_version, [TRACKED],
         "which mangling version to use for symbol names"),
@@ -1818,11 +1821,11 @@ pub fn rustc_optgroups() -> Vec<RustcOptGroup> {
             "How errors and other messages are produced",
             "human|json|short",
         ),
-        opt::opt(
+        opt::multi_s(
             "",
-            "json-rendered",
-            "Choose `rendered` field of json diagnostics render scheme",
-            "plain|termcolor",
+            "json",
+            "Configure the JSON output of the compiler",
+            "CONFIG",
         ),
         opt::opt_s(
             "",
@@ -1918,10 +1921,9 @@ pub fn get_cmd_lint_options(matches: &getopts::Matches,
     (lint_opts, describe_lints, lint_cap)
 }
 
-pub fn build_session_options_and_crate_config(
-    matches: &getopts::Matches,
-) -> (Options, FxHashSet<(String, Option<String>)>) {
-    let color = match matches.opt_str("color").as_ref().map(|s| &s[..]) {
+/// Parse the `--color` flag
+pub fn parse_color(matches: &getopts::Matches) -> ColorConfig {
+    match matches.opt_str("color").as_ref().map(|s| &s[..]) {
         Some("auto") => ColorConfig::Auto,
         Some("always") => ColorConfig::Always,
         Some("never") => ColorConfig::Never,
@@ -1936,46 +1938,52 @@ pub fn build_session_options_and_crate_config(
                 arg
             ),
         ),
-    };
+    }
+}
 
-    let edition = match matches.opt_str("edition") {
-        Some(arg) => Edition::from_str(&arg).unwrap_or_else(|_|
+/// Parse the `--json` flag.
+///
+/// The first value returned is how to render JSON diagnostics, and the second
+/// is whether or not artifact notifications are enabled.
+pub fn parse_json(matches: &getopts::Matches) -> (HumanReadableErrorType, bool) {
+    let mut json_rendered: fn(ColorConfig) -> HumanReadableErrorType =
+        HumanReadableErrorType::Default;
+    let mut json_color = ColorConfig::Never;
+    let mut json_artifact_notifications = false;
+    for option in matches.opt_strs("json") {
+        // For now conservatively forbid `--color` with `--json` since `--json`
+        // won't actually be emitting any colors and anything colorized is
+        // embedded in a diagnostic message anyway.
+        if matches.opt_str("color").is_some() {
             early_error(
                 ErrorOutputType::default(),
-                &format!(
-                    "argument for --edition must be one of: \
-                     {}. (instead was `{}`)",
-                    EDITION_NAME_LIST,
-                    arg
-                ),
-            ),
-        ),
-        None => DEFAULT_EDITION,
-    };
+                "cannot specify the `--color` option with `--json`",
+            );
+        }
 
-    if !edition.is_stable() && !nightly_options::is_nightly_build() {
-        early_error(
-                ErrorOutputType::default(),
-                &format!(
-                    "Edition {} is unstable and only \
-                     available for nightly builds of rustc.",
-                    edition,
-                )
-        )
+        for sub_option in option.split(',') {
+            match sub_option {
+                "diagnostic-short" => json_rendered = HumanReadableErrorType::Short,
+                "diagnostic-rendered-ansi" => json_color = ColorConfig::Always,
+                "artifacts" => json_artifact_notifications = true,
+                s => {
+                    early_error(
+                        ErrorOutputType::default(),
+                        &format!("unknown `--json` option `{}`", s),
+                    )
+                }
+            }
+        }
     }
+    (json_rendered(json_color), json_artifact_notifications)
+}
 
-    let json_rendered = matches.opt_str("json-rendered").and_then(|s| match s.as_str() {
-        "plain" => None,
-        "termcolor" => Some(HumanReadableErrorType::Default(ColorConfig::Always)),
-        _ => early_error(
-            ErrorOutputType::default(),
-            &format!(
-                "argument for --json-rendered must be `plain` or `termcolor` (instead was `{}`)",
-                s,
-            ),
-        ),
-    }).unwrap_or(HumanReadableErrorType::Default(ColorConfig::Never));
-
+/// Parse the `--error-format` flag
+pub fn parse_error_format(
+    matches: &getopts::Matches,
+    color: ColorConfig,
+    json_rendered: HumanReadableErrorType,
+) -> ErrorOutputType {
     // We need the opts_present check because the driver will send us Matches
     // with only stable options if no unstable options are used. Since error-format
     // is unstable, it will not be present. We have to use opts_present not
@@ -2004,6 +2012,60 @@ pub fn build_session_options_and_crate_config(
         ErrorOutputType::HumanReadable(HumanReadableErrorType::Default(color))
     };
 
+    match error_format {
+        ErrorOutputType::Json { .. } => {}
+
+        // Conservatively require that the `--json` argument is coupled with
+        // `--error-format=json`. This means that `--json` is specified we
+        // should actually be emitting JSON blobs.
+        _ if matches.opt_strs("json").len() > 0 => {
+            early_error(
+                ErrorOutputType::default(),
+                "using `--json` requires also using `--error-format=json`",
+            );
+        }
+
+        _ => {}
+    }
+
+    return error_format;
+}
+
+pub fn build_session_options_and_crate_config(
+    matches: &getopts::Matches,
+) -> (Options, FxHashSet<(String, Option<String>)>) {
+    let color = parse_color(matches);
+
+    let edition = match matches.opt_str("edition") {
+        Some(arg) => Edition::from_str(&arg).unwrap_or_else(|_|
+            early_error(
+                ErrorOutputType::default(),
+                &format!(
+                    "argument for --edition must be one of: \
+                     {}. (instead was `{}`)",
+                    EDITION_NAME_LIST,
+                    arg
+                ),
+            ),
+        ),
+        None => DEFAULT_EDITION,
+    };
+
+    if !edition.is_stable() && !nightly_options::is_nightly_build() {
+        early_error(
+                ErrorOutputType::default(),
+                &format!(
+                    "Edition {} is unstable and only \
+                     available for nightly builds of rustc.",
+                    edition,
+                )
+        )
+    }
+
+    let (json_rendered, json_artifact_notifications) = parse_json(matches);
+
+    let error_format = parse_error_format(matches, color, json_rendered);
+
     let unparsed_crate_types = matches.opt_strs("crate-type");
     let crate_types = parse_crate_types_from_list(unparsed_crate_types)
         .unwrap_or_else(|e| early_error(error_format, &e[..]));
@@ -2014,9 +2076,6 @@ pub fn build_session_options_and_crate_config(
     let mut debugging_opts = build_debugging_options(matches, error_format);
 
     if !debugging_opts.unstable_options {
-        if matches.opt_str("json-rendered").is_some() {
-            early_error(error_format, "`--json-rendered=x` is unstable");
-        }
         if let ErrorOutputType::Json { pretty: true, json_rendered } = error_format {
             early_error(
                 ErrorOutputType::Json { pretty: false, json_rendered },
@@ -2441,6 +2500,7 @@ pub fn build_session_options_and_crate_config(
             cli_forced_thinlto_off: disable_thinlto,
             remap_path_prefix,
             edition,
+            json_artifact_notifications,
         },
         cfg,
     )

src/librustc_codegen_ssa/back/link.rs

@@ -95,7 +95,7 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
                     );
                 }
             }
-            if sess.opts.debugging_opts.emit_artifact_notifications {
+            if sess.opts.json_artifact_notifications {
                 sess.parse_sess.span_diagnostic.emit_artifact_notification(&out_filename, "link");
             }
         }

src/librustc_interface/passes.rs

@@ -696,7 +696,7 @@ fn write_out_deps(compiler: &Compiler, outputs: &OutputFilenames, out_filenames:
 
     match result {
         Ok(_) => {
-            if sess.opts.debugging_opts.emit_artifact_notifications {
+            if sess.opts.json_artifact_notifications {
                  sess.parse_sess.span_diagnostic
                     .emit_artifact_notification(&deps_filename, "dep-info");
             }
@@ -1059,7 +1059,7 @@ fn encode_and_write_metadata(
         if let Err(e) = fs::rename(&metadata_filename, &out_filename) {
             tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
         }
-        if tcx.sess.opts.debugging_opts.emit_artifact_notifications {
+        if tcx.sess.opts.json_artifact_notifications {
             tcx.sess.parse_sess.span_diagnostic
                 .emit_artifact_notification(&out_filename, "metadata");
         }

src/librustc_save_analysis/lib.rs

@@ -1071,7 +1071,7 @@ impl SaveHandler for DumpHandler<'_> {
             error!("Can't serialize save-analysis: {:?}", e);
         }
 
-        if sess.opts.debugging_opts.emit_artifact_notifications {
+        if sess.opts.json_artifact_notifications {
             sess.parse_sess.span_diagnostic
                 .emit_artifact_notification(&file_name, "save-analysis");
         }

src/librustdoc/config.rs

@@ -3,10 +3,9 @@ use std::fmt;
 use std::path::PathBuf;
 
 use errors;
-use errors::emitter::{ColorConfig, HumanReadableErrorType};
 use getopts;
 use rustc::lint::Level;
-use rustc::session::early_error;
+use rustc::session;
 use rustc::session::config::{CodegenOptions, DebuggingOptions, ErrorOutputType, Externs};
 use rustc::session::config::{nightly_options, build_codegen_options, build_debugging_options,
                              get_cmd_lint_options, ExternEntry};
@@ -243,36 +242,9 @@ impl Options {
             return Err(0);
         }
 
-        let color = match matches.opt_str("color").as_ref().map(|s| &s[..]) {
-            Some("auto") => ColorConfig::Auto,
-            Some("always") => ColorConfig::Always,
-            Some("never") => ColorConfig::Never,
-            None => ColorConfig::Auto,
-            Some(arg) => {
-                early_error(ErrorOutputType::default(),
-                            &format!("argument for `--color` must be `auto`, `always` or `never` \
-                                      (instead was `{}`)", arg));
-            }
-        };
-        // FIXME: deduplicate this code from the identical code in librustc/session/config.rs
-        let error_format = match matches.opt_str("error-format").as_ref().map(|s| &s[..]) {
-            None |
-            Some("human") => ErrorOutputType::HumanReadable(HumanReadableErrorType::Default(color)),
-            Some("json") => ErrorOutputType::Json {
-                pretty: false,
-                json_rendered: HumanReadableErrorType::Default(ColorConfig::Never),
-            },
-            Some("pretty-json") => ErrorOutputType::Json {
-                pretty: true,
-                json_rendered: HumanReadableErrorType::Default(ColorConfig::Never),
-            },
-            Some("short") => ErrorOutputType::HumanReadable(HumanReadableErrorType::Short(color)),
-            Some(arg) => {
-                early_error(ErrorOutputType::default(),
-                            &format!("argument for `--error-format` must be `human`, `json` or \
-                                      `short` (instead was `{}`)", arg));
-            }
-        };
+        let color = session::config::parse_color(&matches);
+        let (json_rendered, _artifacts) = session::config::parse_json(&matches);
+        let error_format = session::config::parse_error_format(&matches, color, json_rendered);
 
         let codegen_options = build_codegen_options(matches, error_format);
         let debugging_options = build_debugging_options(matches, error_format);

src/librustdoc/lib.rs

@@ -286,6 +286,12 @@ fn opts() -> Vec<RustcOptGroup> {
                      "How errors and other messages are produced",
                      "human|json|short")
         }),
+        stable("json", |o| {
+            o.optopt("",
+                     "json",
+                     "Configure the structure of JSON diagnostics",
+                     "CONFIG")
+        }),
         unstable("disable-minification", |o| {
              o.optflag("",
                        "disable-minification",

src/test/ui/emit-artifact-notifications.rs

@@ -1,4 +1,4 @@
-// compile-flags:--emit=metadata --error-format=json -Z emit-artifact-notifications
+// compile-flags:--emit=metadata --error-format=json --json artifacts
 // build-pass (FIXME(62277): could be check-pass?)
 // ignore-pass
 // ^-- needed because `--pass check` does not emit the output needed.

src/test/ui/json-and-color.rs

@@ -0,0 +1,4 @@
+// build-fail
+// compile-flags: --json=artifacts --error-format=json --color never
+
+fn main() {}

src/test/ui/json-and-color.stderr

@@ -0,0 +1,2 @@
+error: cannot specify the `--color` option with `--json`
+

src/test/ui/json-and-error-format.rs

@@ -0,0 +1,4 @@
+// build-fail
+// compile-flags: --json=artifacts --error-format=short
+
+fn main() {}

src/test/ui/json-and-error-format.stderr

@@ -0,0 +1,2 @@
+error: using `--json` requires also using `--error-format=json`
+

src/test/ui/json-invalid.rs

@@ -0,0 +1,4 @@
+// build-fail
+// compile-flags: --json=foo --error-format=json
+
+fn main() {}