refactor(compile): Log all ignored unused externs (#16920)

### What does this PR try to resolve?

Shoring up the logging we do for unused dependency lints to help with
debugging and profiling.

This could also be useful for users to get more insight on things we
can't or won't show, like helping hunt down features to deactivate to
reduce unused dependencies (e.g. `cli` on `pulldown-cmark`) or removing
transitively unused dependencies.

### How to test and review this PR?

Running this on `cargo` itself:
```console
$ CARGO_LOG=cargo::core::compiler::unused_deps=debug nargo check
...
   2.537113285s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg crypto-common v0.1.6 (normal): ignoring unused extern `typenum`, untracked dependent
   2.537569241s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg heapless v0.8.0 (normal): ignoring unused extern `stable_deref_trait`, untracked dependent
   2.537758659s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg faster-hex v0.10.0 (normal): ignoring unused extern `heapless`, untracked dependent
   2.537984665s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-date v0.15.1 (normal): ignoring unused extern `bstr`, untracked dependent
   2.538312867s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-object v0.58.0 (normal): ignoring unused extern `gix_hashtable`, untracked dependent
   2.538704975s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-tempfile v21.0.2 (normal): ignoring unused extern `libc`, untracked dependent
   2.538762374s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-tempfile v21.0.2 (normal): ignoring unused extern `parking_lot`, untracked dependent
   2.539010269s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-commitgraph v0.35.0 (normal): ignoring unused extern `bstr`, untracked dependent
   2.539205527s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg libnghttp2-sys v0.1.13+1.68.1 (normal): ignoring unused extern `libc`, untracked dependent
   2.539597903s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg curl-sys v0.4.87+curl-8.19.0 (normal): ignoring unused extern `libnghttp2_sys`, untracked dependent
   2.539654548s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg curl-sys v0.4.87+curl-8.19.0 (normal): ignoring unused extern `libz_sys`, untracked dependent
   2.539711192s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg curl-sys v0.4.87+curl-8.19.0 (normal): ignoring unused extern `openssl_sys`, untracked dependent
   2.540319246s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg curl v0.4.49 (normal): ignoring unused extern `openssl_probe`, untracked dependent
   2.540373049s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg curl v0.4.49 (normal): ignoring unused extern `openssl_sys`, untracked dependent
   2.540811522s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-index v0.49.0 (normal): ignoring unused extern `gix_fs`, untracked dependent
   2.541504178s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-revision v0.43.0 (normal): ignoring unused extern `gix_commitgraph`, untracked dependent
   2.541560742s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-revision v0.43.0 (normal): ignoring unused extern `nonempty`, untracked dependent
   2.541873702s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg nix v0.30.1 (normal): ignoring unused extern `bitflags`, lint is allowed
   2.542548816s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg nom v7.1.3 (normal): ignoring unused extern `minimal_lexical`, lint is allowed
   2.542935941s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-negotiate v0.29.0 (normal): ignoring unused extern `gix_commitgraph`, untracked dependent
   2.542989018s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix-negotiate v0.29.0 (normal): ignoring unused extern `gix_object`, untracked dependent
   2.543198046s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg tracing-subscriber v0.3.23 (normal): ignoring unused extern `once_cell`, lint is allowed
   2.543249959s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg tracing-subscriber v0.3.23 (normal): ignoring unused extern `regex_automata`, lint is allowed
   2.543640436s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg git2 v0.20.4 (normal): ignoring unused extern `openssl_sys`, untracked dependent
   2.544004330s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg regex v1.12.3 (normal): ignoring unused extern `aho_corasick`, lint is allowed
   2.544419766s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg futures v0.3.32 (normal): ignoring unused extern `futures_task`, lint is allowed
   2.544564859s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix v0.81.0 (normal): ignoring unused extern `gix_revision`, untracked dependent
   2.544617049s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg gix v0.81.0 (normal): ignoring unused extern `prodash`, untracked dependent
   2.544779907s debug emit_unused_warnings: cargo::core::compiler::unused_deps: pkg rustfix v0.9.6 (normal): ignoring unused extern `tracing`, untracked dependent
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 2.54s

```

Doesn't work too well for `--all-targets` because we don't support dev
targets at the moment, so we don't track which externs came from
dependencies vs dev-dependencies.

Also, of note that I found while doing this that confused me at first is
that we don't fingerprint `-Zcargo-lints`, so users can have stale
caches with no unused externs "reported" by rustc to cargo.

Author: weihanglo

src/cargo/core/compiler/unused_deps.rs

@@ -7,7 +7,7 @@ use cargo_util_terminal::report::Patch;
 use cargo_util_terminal::report::Snippet;
 use indexmap::IndexMap;
 use indexmap::IndexSet;
-use tracing::trace;
+use tracing::{debug, instrument, trace};
 
 use super::BuildRunner;
 use super::unit::Unit;
@@ -30,13 +30,11 @@ pub struct UnusedDepState {
 }
 
 impl UnusedDepState {
+    #[instrument(name = "UnusedDepState::new", skip_all)]
     pub fn new(build_runner: &mut BuildRunner<'_, '_>) -> Self {
-        let mut states = IndexMap::<_, IndexMap<_, DependenciesState>>::new();
-
-        let roots = &build_runner.bcx.roots;
-
         // Find all units for a package that can report unused externs
         let mut root_build_script_builds = IndexSet::new();
+        let roots = &build_runner.bcx.roots;
         for root in roots.iter() {
             for build_script_run in build_runner.unit_deps(root).iter() {
                 if !build_script_run.unit.target.is_custom_build()
@@ -62,6 +60,7 @@ impl UnusedDepState {
             "selected dep kinds: {:?}",
             build_runner.bcx.selected_dep_kinds
         );
+        let mut states = IndexMap::<_, IndexMap<_, DependenciesState>>::new();
         for root in roots.iter().chain(root_build_script_builds.iter()) {
             let pkg_id = root.pkg.package_id();
             let dep_kind = dep_kind_of(root);
@@ -85,7 +84,7 @@ impl UnusedDepState {
                 .or_default()
                 .entry(dep_kind)
                 .or_default();
-            state.needed_units += 1;
+            *state.needed_units.get_or_insert_default() += 1;
             for dep in build_runner.unit_deps(root).iter() {
                 trace!(
                     "    => {} (deps={})",
@@ -108,16 +107,26 @@ impl UnusedDepState {
 
     pub fn record_unused_externs_for_unit(&mut self, unit: &Unit, unused_externs: Vec<String>) {
         let pkg_id = unit.pkg.package_id();
-        let kind = dep_kind_of(unit);
-        if let Some(state) = self.states.get_mut(&pkg_id).and_then(|s| s.get_mut(&kind)) {
-            state
-                .unused_externs
-                .entry(unit.clone())
-                .or_default()
-                .extend(unused_externs.into_iter().map(|s| InternedString::new(&s)));
-        }
+        let dep_kind = dep_kind_of(unit);
+        trace!(
+            "pkg {} v{} ({dep_kind:?}): unused externs {unused_externs:?}",
+            pkg_id.name(),
+            pkg_id.version(),
+        );
+        let state = self
+            .states
+            .entry(pkg_id)
+            .or_default()
+            .entry(dep_kind)
+            .or_default();
+        state
+            .unused_externs
+            .entry(unit.clone())
+            .or_default()
+            .extend(unused_externs.into_iter().map(|s| InternedString::new(&s)));
     }
 
+    #[instrument(skip_all)]
     pub fn emit_unused_warnings(
         &self,
         warn_count: &mut usize,
@@ -146,6 +155,15 @@ impl UnusedDepState {
             );
 
             if lint_level == LintLevel::Allow {
+                for (dep_kind, state) in states.iter() {
+                    for ext in state.unused_externs.values().flatten() {
+                        debug!(
+                            "pkg {} v{} ({dep_kind:?}): ignoring unused extern `{ext}`, lint is allowed",
+                            pkg_id.name(),
+                            pkg_id.version(),
+                        );
+                    }
+                }
                 continue;
             }
 
@@ -169,16 +187,29 @@ impl UnusedDepState {
             let manifest_path = rel_cwd_manifest_path(pkg.manifest_path(), build_runner.bcx.gctx);
             let mut lint_count = 0;
             for (dep_kind, state) in states.iter() {
-                if state.unused_externs.len() != state.needed_units {
+                let Some(needed_units) = state.needed_units else {
+                    // not one we care to report
+                    for ext in state.unused_externs.values().flatten() {
+                        debug!(
+                            "pkg {} v{} ({dep_kind:?}): ignoring unused extern `{ext}`, untracked dependent",
+                            pkg_id.name(),
+                            pkg_id.version(),
+                        );
+                    }
+                    continue;
+                };
+                if state.unused_externs.len() != needed_units {
                     // Some compilations errored without printing the unused externs.
                     // Don't print the warning in order to reduce false positive
                     // spam during errors.
-                    trace!(
-                        "pkg {} v{} ({dep_kind:?}): ignoring unused deps due to {} outstanding units",
-                        pkg_id.name(),
-                        pkg_id.version(),
-                        state.needed_units
-                    );
+                    for ext in state.unused_externs.values().flatten() {
+                        debug!(
+                            "pkg {} v{} ({dep_kind:?}): ignoring unused extern `{ext}`, {} outstanding units",
+                            pkg_id.name(),
+                            pkg_id.version(),
+                            needed_units - state.unused_externs.len()
+                        );
+                    }
                     continue;
                 }
 
@@ -205,11 +236,10 @@ impl UnusedDepState {
                     };
                     for dependency in dependency {
                         if ignore.contains(&dependency.name_in_toml()) {
-                            trace!(
-                                "pkg {} v{} ({dep_kind:?}): extern {} is ignored",
+                            debug!(
+                                "pkg {} v{} ({dep_kind:?}): ignoring unused extern `{ext}`, requested in manifest",
                                 pkg_id.name(),
                                 pkg_id.version(),
-                                ext
                             );
                             continue;
                         }
@@ -293,7 +323,7 @@ struct DependenciesState {
     ///
     /// To avoid warning in cases where we didn't,
     /// e.g. if a [`Unit`] errored and didn't report unused externs.
-    needed_units: usize,
+    needed_units: Option<usize>,
     /// As reported by rustc
     unused_externs: IndexMap<Unit, Vec<InternedString>>,
 }