Skip to content

Commit 56b8d56

Browse files
flip1995flip1995
committed
Implement sync subcommand in clippy_dev
Now that JOSH is used to sync, it is much easier to script the sync process. This introduces the two commands `sync pull` and `sync push`. The first one will pull changes from the Rust repo, the second one will push the changes to the Rust repo. For details, see the documentation in the book.
1 parent 74ee288 commit 56b8d56

File tree

4 files changed

+283
-1
lines changed

4 files changed

+283
-1
lines changed

‎clippy_dev/Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@ edition = "2021"
66

77
[dependencies]
88
aho-corasick = "1.0"
9+
chrono = { version = "0.4.38", default-features = false, features = ["clock"] }
910
clap = { version = "4.4", features = ["derive"] }
11+
directories = "5"
1012
indoc = "1.0"
1113
itertools = "0.12"
1214
opener = "0.6"
1315
shell-escape = "0.1"
1416
walkdir = "2.3"
17+
xshell = "0.2"
1518

1619
[features]
1720
deny-warnings = []

‎clippy_dev/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,6 @@ pub mod lint;
2121
pub mod new_lint;
2222
pub mod serve;
2323
pub mod setup;
24+
pub mod sync;
2425
pub mod update_lints;
2526
pub mod utils;

‎clippy_dev/src/main.rs

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#![warn(rust_2018_idioms, unused_lifetimes)]
44

55
use clap::{Args, Parser, Subcommand};
6-
use clippy_dev::{dogfood, fmt, lint, new_lint, serve, setup, update_lints, utils};
6+
use clippy_dev::{dogfood, fmt, lint, new_lint, serve, setup, sync, update_lints, utils};
77
use std::convert::Infallible;
88

99
fn main() {
@@ -75,6 +75,15 @@ fn main() {
7575
uplift,
7676
} => update_lints::rename(&old_name, new_name.as_ref().unwrap_or(&old_name), uplift),
7777
DevCommand::Deprecate { name, reason } => update_lints::deprecate(&name, reason.as_deref()),
78+
DevCommand::Sync(SyncCommand { subcommand }) => match subcommand {
79+
SyncSubcommand::Pull => sync::rustc_pull(),
80+
SyncSubcommand::Push {
81+
repo_path,
82+
user,
83+
branch,
84+
force,
85+
} => sync::rustc_push(repo_path, &user, &branch, force),
86+
},
7887
}
7988
}
8089

@@ -225,6 +234,8 @@ enum DevCommand {
225234
/// The reason for deprecation
226235
reason: Option<String>,
227236
},
237+
/// Sync between the rust repo and the Clippy repo
238+
Sync(SyncCommand),
228239
}
229240

230241
#[derive(Args)]
@@ -291,3 +302,31 @@ enum RemoveSubcommand {
291302
/// Remove the tasks added with 'cargo dev setup vscode-tasks'
292303
VscodeTasks,
293304
}
305+
306+
#[derive(Args)]
307+
struct SyncCommand {
308+
#[command(subcommand)]
309+
subcommand: SyncSubcommand,
310+
}
311+
312+
#[derive(Subcommand)]
313+
enum SyncSubcommand {
314+
/// Pull changes from rustc and update the toolchain
315+
Pull,
316+
/// Push changes to rustc
317+
Push {
318+
/// The path to a rustc repo that will be used for pushing changes
319+
repo_path: String,
320+
#[arg(long)]
321+
/// The GitHub username to use for pushing changes
322+
user: String,
323+
#[arg(long, short, default_value = "clippy-subtree-update")]
324+
/// The branch to push to
325+
///
326+
/// This is mostly for experimentation and usually the default should be used.
327+
branch: String,
328+
#[arg(long, short)]
329+
/// Force push changes
330+
force: bool,
331+
},
332+
}

‎clippy_dev/src/sync.rs

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
use std::fmt::Write;
2+
use std::path::Path;
3+
use std::process;
4+
use std::process::exit;
5+
6+
use chrono::offset::Utc;
7+
use xshell::{cmd, Shell};
8+
9+
use crate::utils::{clippy_project_root, replace_region_in_file, UpdateMode};
10+
11+
const JOSH_FILTER: &str = ":rev(2efebd2f0c03dabbe5c3ad7b4ebfbd99238d1fb2:prefix=src/tools/clippy):/src/tools/clippy";
12+
const JOSH_PORT: &str = "42042";
13+
14+
fn start_josh() -> impl Drop {
15+
// Create a wrapper that stops it on drop.
16+
struct Josh(process::Child);
17+
impl Drop for Josh {
18+
fn drop(&mut self) {
19+
#[cfg(unix)]
20+
{
21+
// Try to gracefully shut it down.
22+
process::Command::new("kill")
23+
.args(["-s", "INT", &self.0.id().to_string()])
24+
.output()
25+
.expect("failed to SIGINT josh-proxy");
26+
// Sadly there is no "wait with timeout"... so we just give it some time to finish.
27+
std::thread::sleep(std::time::Duration::from_secs(1));
28+
// Now hopefully it is gone.
29+
if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() {
30+
return;
31+
}
32+
}
33+
// If that didn't work (or we're not on Unix), kill it hard.
34+
eprintln!("I have to kill josh-proxy the hard way, let's hope this does not break anything.");
35+
self.0.kill().expect("failed to SIGKILL josh-proxy");
36+
}
37+
}
38+
39+
// Determine cache directory.
40+
let local_dir = {
41+
let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "clippy-josh").unwrap();
42+
user_dirs.cache_dir().to_owned()
43+
};
44+
println!("Using local cache directory: {}", local_dir.display());
45+
46+
// Start josh, silencing its output.
47+
let mut cmd = process::Command::new("josh-proxy");
48+
cmd.arg("--local").arg(local_dir);
49+
cmd.arg("--remote").arg("https://github.com");
50+
cmd.arg("--port").arg(JOSH_PORT);
51+
cmd.arg("--no-background");
52+
cmd.stdout(process::Stdio::null());
53+
cmd.stderr(process::Stdio::null());
54+
let josh = cmd
55+
.spawn()
56+
.expect("failed to start josh-proxy, make sure it is installed");
57+
// Give it some time so hopefully the port is open.
58+
std::thread::sleep(std::time::Duration::from_secs(1));
59+
60+
Josh(josh)
61+
}
62+
63+
fn rustc_hash() -> String {
64+
let sh = Shell::new().expect("failed to create shell");
65+
// Make sure we pick up the updated toolchain (usually rustup pins the toolchain
66+
// inside a single cargo/rustc invocation via this env var).
67+
sh.set_var("RUSTUP_TOOLCHAIN", "");
68+
cmd!(sh, "rustc --version --verbose")
69+
.read()
70+
.expect("failed to run `rustc -vV`")
71+
.lines()
72+
.find(|line| line.starts_with("commit-hash:"))
73+
.expect("failed to parse `rustc -vV`")
74+
.split_whitespace()
75+
.last()
76+
.expect("failed to get commit from `rustc -vV`")
77+
.to_string()
78+
}
79+
80+
fn assert_clean_repo(sh: &Shell) {
81+
if !cmd!(sh, "git status --untracked-files=no --porcelain")
82+
.read()
83+
.expect("failed to run git status")
84+
.is_empty()
85+
{
86+
eprintln!("working directory must be clean before running `cargo dev sync pull`");
87+
exit(1);
88+
}
89+
}
90+
91+
pub fn rustc_pull() {
92+
const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc";
93+
94+
let sh = Shell::new().expect("failed to create shell");
95+
sh.change_dir(clippy_project_root());
96+
97+
assert_clean_repo(&sh);
98+
99+
// Update rust-toolchain file
100+
let date = Utc::now().format("%Y-%m-%d").to_string();
101+
replace_region_in_file(
102+
UpdateMode::Change,
103+
Path::new("rust-toolchain"),
104+
"# begin autogenerated version\n",
105+
"# end autogenerated version",
106+
|res| {
107+
writeln!(res, "channel = \"nightly-{date}\"").unwrap();
108+
},
109+
);
110+
111+
let message = format!("Bump nightly version -> {date}");
112+
cmd!(sh, "git commit rust-toolchain --no-verify -m {message}")
113+
.run()
114+
.expect("FAILED to commit rust-toolchain file, something went wrong");
115+
116+
let commit = rustc_hash();
117+
118+
// Make sure josh is running in this scope
119+
{
120+
let _josh = start_josh();
121+
122+
// Fetch given rustc commit.
123+
cmd!(
124+
sh,
125+
"git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git"
126+
)
127+
.run()
128+
.inspect_err(|_| {
129+
// Try to un-do the previous `git commit`, to leave the repo in the state we found it.
130+
cmd!(sh, "git reset --hard HEAD^")
131+
.run()
132+
.expect("FAILED to clean up again after failed `git fetch`, sorry for that");
133+
})
134+
.expect("FAILED to fetch new commits, something went wrong");
135+
}
136+
137+
// This should not add any new root commits. So count those before and after merging.
138+
let num_roots = || -> u32 {
139+
cmd!(sh, "git rev-list HEAD --max-parents=0 --count")
140+
.read()
141+
.expect("failed to determine the number of root commits")
142+
.parse::<u32>()
143+
.unwrap()
144+
};
145+
let num_roots_before = num_roots();
146+
147+
// Merge the fetched commit.
148+
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
149+
.run()
150+
.expect("FAILED to merge new commits, something went wrong");
151+
152+
// Check that the number of roots did not increase.
153+
if num_roots() != num_roots_before {
154+
eprintln!("Josh created a new root commit. This is probably not the history you want.");
155+
exit(1);
156+
}
157+
}
158+
159+
pub(crate) const PUSH_PR_DESCRIPTION: &str = "Sync from Clippy commit:";
160+
161+
pub fn rustc_push(rustc_path: String, github_user: &str, branch: &str, force: bool) {
162+
let sh = Shell::new().expect("failed to create shell");
163+
sh.change_dir(clippy_project_root());
164+
165+
assert_clean_repo(&sh);
166+
167+
// Prepare the branch. Pushing works much better if we use as base exactly
168+
// the commit that we pulled from last time, so we use the `rustc --version`
169+
// to find out which commit that would be.
170+
let base = rustc_hash();
171+
172+
println!("Preparing {github_user}/rust (base: {base})...");
173+
sh.change_dir(rustc_path);
174+
if !force
175+
&& cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}")
176+
.ignore_stderr()
177+
.read()
178+
.is_ok()
179+
{
180+
eprintln!(
181+
"The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again."
182+
);
183+
exit(1);
184+
}
185+
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}")
186+
.run()
187+
.expect("failed to fetch base commit");
188+
let force_flag = if force { "--force" } else { "" };
189+
cmd!(
190+
sh,
191+
"git push https://github.com/{github_user}/rust {base}:refs/heads/{branch} {force_flag}"
192+
)
193+
.ignore_stdout()
194+
.ignore_stderr() // silence the "create GitHub PR" message
195+
.run()
196+
.expect("failed to push base commit to the new branch");
197+
198+
// Make sure josh is running in this scope
199+
{
200+
let _josh = start_josh();
201+
202+
// Do the actual push.
203+
sh.change_dir(clippy_project_root());
204+
println!("Pushing Clippy changes...");
205+
cmd!(
206+
sh,
207+
"git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
208+
)
209+
.run()
210+
.expect("failed to push changes to Josh");
211+
212+
// Do a round-trip check to make sure the push worked as expected.
213+
cmd!(
214+
sh,
215+
"git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
216+
)
217+
.ignore_stderr()
218+
.read()
219+
.expect("failed to fetch the branch from Josh");
220+
}
221+
222+
let head = cmd!(sh, "git rev-parse HEAD")
223+
.read()
224+
.expect("failed to get HEAD commit");
225+
let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD")
226+
.read()
227+
.expect("failed to get FETCH_HEAD");
228+
if head != fetch_head {
229+
eprintln!("Josh created a non-roundtrip push! Do NOT merge this into rustc!");
230+
exit(1);
231+
}
232+
println!("Confirmed that the push round-trips back to Clippy properly. Please create a rustc PR:");
233+
let description = format!("{}+rust-lang/rust-clippy@{head}", PUSH_PR_DESCRIPTION.replace(' ', "+"));
234+
println!(
235+
// Open PR with `subtree update` title to silence the `no-merges` triagebot check
236+
// See https://github.com/rust-lang/rust/pull/114157
237+
" https://github.com/rust-lang/rust/compare/{github_user}:{branch}?quick_pull=1&title=Clippy+subtree+update&body=r?+@ghost%0A%0A{description}"
238+
);
239+
}

0 commit comments

Comments
 (0)