Auto merge of #9510 - Alexendoo:lintcheck-server2, r=matthiaskrgr

Add `cargo lintcheck --recursive` to check dependencies of crates

r? `@matthiaskrgr`

This just adds the mode without changing how a regular run works

Takes a fair bit longer to run than a `-j4` or regular run

```
cargo lintcheck -j4  468.33s user 43.78s system 290% cpu 2:56.42 total
cargo lintcheck  428.81s user 41.85s system 199% cpu 3:55.37 total
cargo lintcheck --recursive  679.83s user 45.01s system 210% cpu 5:43.72 total
```

But finds more results:

<details>
<summary>Stats</summary>
<pre><code>clippy::explicit_counter_loop 0 =&gt; 1
clippy::needless_question_mark 0 =&gt; 3
clippy::unnecessary_cast 0 =&gt; 2
clippy::to_string_in_format_args 0 =&gt; 4
clippy::deprecated_cfg_attr 0 =&gt; 23
clippy::redundant_closure 0 =&gt; 9
clippy::drop_copy 0 =&gt; 4
clippy::double_must_use 0 =&gt; 1
clippy::transmute_num_to_bytes 0 =&gt; 9
clippy::bind_instead_of_map 0 =&gt; 14
clippy::float_cmp 0 =&gt; 16
clippy::is_digit_ascii_radix 0 =&gt; 16
clippy::manual_swap 0 =&gt; 1
clippy::needless_match 0 =&gt; 2
clippy::vec_init_then_push 0 =&gt; 1
clippy::never_loop 0 =&gt; 1
clippy::option_map_or_none 0 =&gt; 4
clippy::tabs_in_doc_comments 0 =&gt; 1
clippy::naive_bytecount 0 =&gt; 1
clippy::collapsible_if 0 =&gt; 24
clippy::copy_iterator 0 =&gt; 5
clippy::unused_io_amount 0 =&gt; 2
clippy::result_large_err 0 =&gt; 141
clippy::useless_conversion 0 =&gt; 24
clippy::flat_map_option 0 =&gt; 8
clippy::useless_format 0 =&gt; 2
clippy::module_inception 0 =&gt; 1
clippy::drop_ref 0 =&gt; 2
clippy::unnecessary_fold 0 =&gt; 2
clippy::neg_multiply 0 =&gt; 1
clippy::while_let_loop 0 =&gt; 6
clippy::missing_inline_in_public_items 0 =&gt; 37
clippy::unnecessary_mut_passed 0 =&gt; 1
unknown_lints 0 =&gt; 15
clippy::wildcard_dependencies 0 =&gt; 3
clippy::same_item_push 0 =&gt; 2
clippy::useless_asref 0 =&gt; 1
clippy::unnecessary_unwrap 0 =&gt; 4
clippy::iter_not_returning_iterator 0 =&gt; 5
clippy::comparison_to_empty 0 =&gt; 10
clippy::ref_option_ref 0 =&gt; 4
clippy::unused_peekable 0 =&gt; 1
clippy::needless_range_loop 0 =&gt; 8
clippy::absurd_extreme_comparisons 0 =&gt; 2
clippy::unnecessary_operation 0 =&gt; 2
clippy::for_kv_map 0 =&gt; 5
clippy::unnecessary_owned_empty_strings 0 =&gt; 3
clippy::transmutes_expressible_as_ptr_casts 0 =&gt; 1
clippy::toplevel_ref_arg 0 =&gt; 2
clippy::uninit_vec 0 =&gt; 3
clippy::filter_next 0 =&gt; 1
clippy::wildcard_in_or_patterns 0 =&gt; 6
clippy::cast_ptr_alignment 0 =&gt; 48
clippy::manual_memcpy 0 =&gt; 1
clippy::assign_op_pattern 0 =&gt; 313
clippy::unnecessary_lazy_evaluations 0 =&gt; 14
clippy::println_empty_string 0 =&gt; 2
clippy::redundant_pattern 0 =&gt; 2
clippy::declare_interior_mutable_const 0 =&gt; 8
clippy::large_stack_arrays 0 =&gt; 4
clippy::match_bool 0 =&gt; 4
clippy::unicode_not_nfc 0 =&gt; 2075
clippy::inconsistent_digit_grouping 0 =&gt; 4
clippy::no_effect_underscore_binding 0 =&gt; 2
clippy::let_and_return 0 =&gt; 5
clippy::transmute_ptr_to_ref 0 =&gt; 12
clippy::op_ref 0 =&gt; 13
clippy::unnecessary_join 0 =&gt; 4
clippy::into_iter_on_ref 0 =&gt; 13
clippy::from_str_radix_10 0 =&gt; 7
clippy::ptr_offset_with_cast 0 =&gt; 48
clippy::erasing_op 0 =&gt; 1
clippy::swap_ptr_to_ref 0 =&gt; 3
clippy::needless_bitwise_bool 0 =&gt; 2
clippy::extend_with_drain 0 =&gt; 19
clippy::only_used_in_recursion 0 =&gt; 4
clippy::needless_late_init 0 =&gt; 8
clippy::excessive_precision 0 =&gt; 1959
clippy::match_ref_pats 0 =&gt; 10
clippy::unit_arg 0 =&gt; 20
clippy::bool_comparison 0 =&gt; 4
clippy::bool_assert_comparison 0 =&gt; 1
clippy::eq_op 0 =&gt; 6
clippy::cast_abs_to_unsigned 0 =&gt; 6
clippy::format_in_format_args 0 =&gt; 1
clippy::iter_cloned_collect 0 =&gt; 4
clippy::ptr_eq 0 =&gt; 3
clippy::needless_bool 0 =&gt; 5
clippy::transmute_ptr_to_ptr 0 =&gt; 16
clippy::needless_option_take 0 =&gt; 2
clippy::flat_map_identity 0 =&gt; 1
clippy::needless_splitn 0 =&gt; 2
clippy::blocks_in_if_conditions 0 =&gt; 1
clippy::write_literal 0 =&gt; 1
clippy::manual_split_once 0 =&gt; 1
clippy::result_unit_err 0 =&gt; 36
clippy::unused_unit 0 =&gt; 11
clippy::single_match 0 =&gt; 22
clippy::manual_find 0 =&gt; 3
clippy::derive_ord_xor_partial_ord 0 =&gt; 6
clippy::char_lit_as_u8 0 =&gt; 2
clippy::let_unit_value 0 =&gt; 2
clippy::needless_continue 0 =&gt; 19
clippy::zero_sized_map_values 0 =&gt; 4
clippy::needless_arbitrary_self_type 0 =&gt; 6
clippy::partialeq_to_none 0 =&gt; 11
clippy::partialeq_ne_impl 0 =&gt; 1
clippy::invalid_upcast_comparisons 0 =&gt; 1
clippy::mut_range_bound 0 =&gt; 4
clippy::match_result_ok 0 =&gt; 2
clippy::ptr_arg 0 =&gt; 8
clippy::iter_nth_zero 0 =&gt; 18
clippy::needless_for_each 0 =&gt; 1
clippy::manual_unwrap_or 0 =&gt; 1
clippy::transmute_int_to_float 0 =&gt; 6
clippy::cast_slice_from_raw_parts 0 =&gt; 1
clippy::match_wild_err_arm 0 =&gt; 2
clippy::match_like_matches_macro 4 =&gt; 116
clippy::enum_glob_use 50 =&gt; 380
clippy::get_first 3 =&gt; 33
clippy::needless_doctest_main 10 =&gt; 26
clippy::struct_excessive_bools 19 =&gt; 51
clippy::cast_possible_wrap 46 =&gt; 538
clippy::manual_string_new 10 =&gt; 27
clippy::match_same_arms 53 =&gt; 1039
clippy::manual_non_exhaustive 1 =&gt; 33
clippy::redundant_pattern_matching 2 =&gt; 13
clippy::new_without_default 5 =&gt; 73
clippy::option_as_ref_deref 2 =&gt; 9
clippy::unwrap_or_else_default 2 =&gt; 4
clippy::case_sensitive_file_extension_comparisons 6 =&gt; 9
clippy::cast_precision_loss 45 =&gt; 110
clippy::needless_pass_by_value 26 =&gt; 187
clippy::redundant_closure_for_method_calls 170 =&gt; 539
clippy::let_underscore_drop 33 =&gt; 133
clippy::single_match_else 51 =&gt; 138
clippy::needless_borrow 24 =&gt; 382
clippy::redundant_else 37 =&gt; 151
clippy::type_complexity 2 =&gt; 22
clippy::ptr_as_ptr 93 =&gt; 1135
clippy::needless_lifetimes 7 =&gt; 100
clippy::single_char_add_str 2 =&gt; 22
clippy::similar_names 99 =&gt; 352
clippy::cargo_common_metadata 25 =&gt; 276
clippy::int_plus_one 1 =&gt; 2
clippy::missing_safety_doc 9 =&gt; 152
clippy::redundant_slicing 2 =&gt; 13
clippy::mut_mut 2 =&gt; 17
clippy::derive_partial_eq_without_eq 8 =&gt; 141
clippy::derive_hash_xor_eq 2 =&gt; 20
clippy::from_iter_instead_of_collect 2 =&gt; 17
clippy::verbose_bit_mask 1 =&gt; 8
clippy::too_many_lines 58 =&gt; 162
clippy::module_name_repetitions 178 =&gt; 1104
clippy::explicit_into_iter_loop 12 =&gt; 32
clippy::cast_lossless 45 =&gt; 478
clippy::many_single_char_names 9 =&gt; 23
clippy::unnested_or_patterns 27 =&gt; 127
clippy::upper_case_acronyms 5 =&gt; 29
clippy::needless_return 5 =&gt; 97
clippy::precedence 1 =&gt; 11
clippy::len_zero 2 =&gt; 70
clippy::manual_strip 2 =&gt; 30
clippy::derivable_impls 2 =&gt; 12
clippy::unused_self 20 =&gt; 187
clippy::enum_variant_names 1 =&gt; 6
clippy::self_named_constructors 1 =&gt; 3
clippy::explicit_auto_deref 19 =&gt; 314
clippy::semicolon_if_nothing_returned 137 =&gt; 1861
clippy::should_implement_trait 1 =&gt; 7
clippy::expl_impl_clone_on_copy 159 =&gt; 1318
clippy::stable_sort_primitive 4 =&gt; 12
clippy::mem_replace_with_default 7 =&gt; 46
clippy::borrow_deref_ref 5 =&gt; 140
clippy::large_enum_variant 1 =&gt; 4
clippy::map_unwrap_or 30 =&gt; 203
clippy::zero_ptr 3 =&gt; 25
clippy::filter_map_next 2 =&gt; 6
clippy::identity_op 5 =&gt; 76
clippy::checked_conversions 1 =&gt; 8
clippy::len_without_is_empty 9 =&gt; 47
clippy::missing_errors_doc 372 =&gt; 2333
clippy::fn_params_excessive_bools 3 =&gt; 7
clippy::single_component_path_imports 6 =&gt; 28
clippy::unreadable_literal 366 =&gt; 9814
clippy::field_reassign_with_default 1 =&gt; 5
clippy::redundant_clone 1 =&gt; 8
clippy::cloned_instead_of_copied 36 =&gt; 78
clippy::too_many_arguments 4 =&gt; 22
clippy::option_map_unit_fn 7 =&gt; 9
clippy::extra_unused_lifetimes 1 =&gt; 24
clippy::unnecessary_wraps 26 =&gt; 128
clippy::used_underscore_binding 1 =&gt; 50
clippy::inconsistent_struct_constructor 2 =&gt; 7
clippy::manual_range_contains 9 =&gt; 120
clippy::map_clone 7 =&gt; 46
clippy::cast_slice_different_sizes 1 =&gt; 4
clippy::missing_panics_doc 114 =&gt; 603
renamed_and_removed_lints 3 =&gt; 9
clippy::items_after_statements 155 =&gt; 309
clippy::inefficient_to_string 5 =&gt; 6
clippy::comparison_chain 1 =&gt; 19
clippy::crate_in_macro_def 3 =&gt; 6
clippy::write_with_newline 2 =&gt; 36
clippy::manual_saturating_arithmetic 1 =&gt; 2
clippy::clone_on_copy 1 =&gt; 86
clippy::negative_feature_names 3 =&gt; 16
clippy::redundant_field_names 112 =&gt; 1013
clippy::from_over_into 2 =&gt; 28
clippy::wildcard_imports 178 =&gt; 376
clippy::unusual_byte_groupings 19 =&gt; 65
clippy::option_option 2 =&gt; 10
clippy::nonminimal_bool 1 =&gt; 17
clippy::borrow_as_ptr 2 =&gt; 172
clippy::redundant_static_lifetimes 24 =&gt; 1701
clippy::or_fun_call 1 =&gt; 63
clippy::single_char_pattern 3 =&gt; 79
clippy::explicit_iter_loop 72 =&gt; 148
clippy::collapsible_else_if 2 =&gt; 27
clippy::manual_str_repeat 1 =&gt; 6
clippy::if_same_then_else 3 =&gt; 31
clippy::while_let_on_iterator 4 =&gt; 28
clippy::multiple_crate_versions 5 =&gt; 19
clippy::cast_possible_truncation 115 =&gt; 1172
clippy::explicit_deref_methods 1 =&gt; 38
clippy::default_trait_access 48 =&gt; 130
clippy::question_mark 2 =&gt; 28
clippy::must_use_candidate 612 =&gt; 5369
clippy::manual_map 1 =&gt; 12
clippy::bool_to_int_with_if 2 =&gt; 15
clippy::doc_markdown 202 =&gt; 1709
clippy::cast_sign_loss 60 =&gt; 477
clippy::wrong_self_convention 11 =&gt; 45
clippy::transmute_float_to_int 6 =&gt; 18
clippy::return_self_not_must_use 66 =&gt; 736
clippy::range_plus_one 1 =&gt; 36
clippy::manual_assert 11 =&gt; 62
clippy::trivially_copy_pass_by_ref 40 =&gt; 189
clippy::match_on_vec_items 2 =&gt; 7
clippy::inline_always 59 =&gt; 1079
clippy::if_not_else 31 =&gt; 205
clippy::implicit_clone 10 =&gt; 32
clippy::match_wildcard_for_single_variants 16 =&gt; 101
clippy::doc_link_with_quotes 7 =&gt; 35
clippy::redundant_feature_names 4 =&gt; 41
</code></pre></details>

changelog: none

Author: bors

lintcheck/Cargo.toml

@@ -12,9 +12,11 @@ publish = false
 [dependencies]
 cargo_metadata = "0.14"
 clap = "3.2"
+crossbeam-channel = "0.5.6"
 flate2 = "1.0"
 rayon = "1.5.1"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0.85"
 tar = "0.4"
 toml = "0.5"
 ureq = "2.2"

lintcheck/README.md

@@ -69,9 +69,27 @@ is checked.
 is explicitly specified in the options.
 
 ### Fix mode
-You can run `./lintcheck/target/debug/lintcheck --fix` which will run Clippy with `--fix` and
+You can run `cargo lintcheck --fix` which will run Clippy with `--fix` and
 print a warning if Clippy's suggestions fail to apply (if the resulting code does not build).  
 This lets us spot bad suggestions or false positives automatically in some cases.  
 
 Please note that the target dir should be cleaned afterwards since clippy will modify
 the downloaded sources which can lead to unexpected results when running lintcheck again afterwards.
+
+### Recursive mode
+You can run `cargo lintcheck --recursive` to also run Clippy on the dependencies
+of the crates listed in the crates source `.toml`. e.g. adding `rand 0.8.5`
+would also lint `rand_core`, `rand_chacha`, etc.
+
+Particularly slow crates in the dependency graph can be ignored using
+`recursive.ignore`:
+
+```toml
+[crates]
+cargo = {name = "cargo", versions = ['0.64.0']}
+
+[recursive]
+ignore = [
+    "unicode-normalization",
+]
+```

lintcheck/lintcheck_crates.toml

@@ -33,3 +33,11 @@ cfg-expr = {name = "cfg-expr", versions = ['0.7.1']}
 puffin = {name = "puffin", git_url = "https://github.com/EmbarkStudios/puffin", git_hash = "02dd4a3"}
 rpmalloc = {name = "rpmalloc", versions = ['0.2.0']}
 tame-oidc = {name = "tame-oidc", versions = ['0.1.0']}
+
+[recursive]
+ignore = [
+    # Takes ~30s to lint
+    "combine",
+    # Has 1.2 million `clippy::match_same_arms`s
+    "unicode-normalization",
+]

lintcheck/src/config.rs

@@ -34,11 +34,16 @@ fn get_clap_config() -> ArgMatches {
             Arg::new("markdown")
                 .long("markdown")
                 .help("Change the reports table to use markdown links"),
+            Arg::new("recursive")
+                .long("--recursive")
+                .help("Run clippy on the dependencies of crates specified in crates-toml")
+                .conflicts_with("threads")
+                .conflicts_with("fix"),
         ])
         .get_matches()
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub(crate) struct LintcheckConfig {
     /// max number of jobs to spawn (default 1)
     pub max_jobs: usize,
@@ -54,6 +59,8 @@ pub(crate) struct LintcheckConfig {
     pub lint_filter: Vec<String>,
     /// Indicate if the output should support markdown syntax
     pub markdown: bool,
+    /// Run clippy on the dependencies of crates
+    pub recursive: bool,
 }
 
 impl LintcheckConfig {
@@ -119,6 +126,7 @@ impl LintcheckConfig {
             fix: clap_config.contains_id("fix"),
             lint_filter,
             markdown,
+            recursive: clap_config.contains_id("recursive"),
         }
     }
 }

lintcheck/src/driver.rs

@@ -0,0 +1,67 @@
+use crate::recursive::{deserialize_line, serialize_line, DriverInfo};
+
+use std::io::{self, BufReader, Write};
+use std::net::TcpStream;
+use std::process::{self, Command, Stdio};
+use std::{env, mem};
+
+/// 1. Sends [DriverInfo] to the [crate::recursive::LintcheckServer] running on `addr`
+/// 2. Receives [bool] from the server, if `false` returns `None`
+/// 3. Otherwise sends the stderr of running `clippy-driver` to the server
+fn run_clippy(addr: &str) -> Option<i32> {
+    let driver_info = DriverInfo {
+        package_name: env::var("CARGO_PKG_NAME").ok()?,
+        crate_name: env::var("CARGO_CRATE_NAME").ok()?,
+        version: env::var("CARGO_PKG_VERSION").ok()?,
+    };
+
+    let mut stream = BufReader::new(TcpStream::connect(addr).unwrap());
+
+    serialize_line(&driver_info, stream.get_mut());
+
+    let should_run = deserialize_line::<bool, _>(&mut stream);
+    if !should_run {
+        return None;
+    }
+
+    // Remove --cap-lints allow so that clippy runs and lints are emitted
+    let mut include_next = true;
+    let args = env::args().skip(1).filter(|arg| match arg.as_str() {
+        "--cap-lints=allow" => false,
+        "--cap-lints" => {
+            include_next = false;
+            false
+        },
+        _ => mem::replace(&mut include_next, true),
+    });
+
+    let output = Command::new(env::var("CLIPPY_DRIVER").expect("missing env CLIPPY_DRIVER"))
+        .args(args)
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("failed to run clippy-driver");
+
+    stream
+        .get_mut()
+        .write_all(&output.stderr)
+        .unwrap_or_else(|e| panic!("{e:?} in {driver_info:?}"));
+
+    match output.status.code() {
+        Some(0) => Some(0),
+        code => {
+            io::stderr().write_all(&output.stderr).unwrap();
+            Some(code.expect("killed by signal"))
+        },
+    }
+}
+
+pub fn drive(addr: &str) {
+    process::exit(run_clippy(addr).unwrap_or_else(|| {
+        Command::new("rustc")
+            .args(env::args_os().skip(2))
+            .status()
+            .unwrap()
+            .code()
+            .unwrap()
+    }))
+}