From ea7fef1ccf4b44effbc3bbc902f9b0b9dbd251a4 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Mon, 17 Dec 2018 21:10:59 -0800
Subject: [PATCH 1/5] bootstrap: Link LLVM as a dylib with ThinLTO
When building a distributed compiler on Linux where we use ThinLTO to
create the LLVM shared object this commit switches the compiler to
dynamically linking that LLVM artifact instead of statically linking to
LLVM. The primary goal here is to reduce CI compile times, avoiding two+
ThinLTO builds of all of LLVM. By linking dynamically to LLVM we'll
reuse the one ThinLTO step done by LLVM's build itself.
Lots of discussion about this change can be found [here] and down. A
perf run will show whether this is worth it or not!
[here]: https://github.com/rust-lang/rust/pull/53245#issuecomment-417015334
---
src/bootstrap/check.rs | 5 ----
src/bootstrap/compile.rs | 49 +++++++-------------------------------
src/bootstrap/dist.rs | 8 +++----
src/bootstrap/tool.rs | 2 +-
src/librustc_llvm/build.rs | 4 ++++
5 files changed, 17 insertions(+), 51 deletions(-)
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 050db936daab1..cc539d4c89571 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -38,7 +38,6 @@ impl Step for Std {
builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target));
run_cargo(builder,
&mut cargo,
- vec![],
&libstd_stamp(builder, compiler, target),
true);
@@ -85,7 +84,6 @@ impl Step for Rustc {
builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target));
run_cargo(builder,
&mut cargo,
- vec![],
&librustc_stamp(builder, compiler, target),
true);
@@ -136,7 +134,6 @@ impl Step for CodegenBackend {
let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
run_cargo(builder,
&mut cargo,
- vec![],
&codegen_backend_stamp(builder, compiler, target, backend),
true);
}
@@ -174,7 +171,6 @@ impl Step for Test {
builder.info(&format!("Checking test artifacts ({} -> {})", &compiler.host, target));
run_cargo(builder,
&mut cargo,
- vec![],
&libtest_stamp(builder, compiler, target),
true);
@@ -222,7 +218,6 @@ impl Step for Rustdoc {
println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
run_cargo(builder,
&mut cargo,
- vec![],
&rustdoc_stamp(builder, compiler, target),
true);
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 821bd002e95b3..0d2546a0e9ca9 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -19,6 +19,7 @@ use build_helper::{output, mtime, up_to_date};
use filetime::FileTime;
use serde_json;
+use crate::dist;
use crate::util::{exe, libdir, is_dylib};
use crate::{Compiler, Mode, GitRepo};
use crate::native;
@@ -104,7 +105,6 @@ impl Step for Std {
&compiler.host, target));
run_cargo(builder,
&mut cargo,
- vec![],
&libstd_stamp(builder, compiler, target),
false);
@@ -365,7 +365,6 @@ impl Step for Test {
&compiler.host, target));
run_cargo(builder,
&mut cargo,
- vec![],
&libtest_stamp(builder, compiler, target),
false);
@@ -493,7 +492,6 @@ impl Step for Rustc {
compiler.stage, &compiler.host, target));
run_cargo(builder,
&mut cargo,
- vec![],
&librustc_stamp(builder, compiler, target),
false);
@@ -636,47 +634,18 @@ impl Step for CodegenBackend {
let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
- let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc");
+ let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "build");
cargo.arg("--manifest-path")
.arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
rustc_cargo_env(builder, &mut cargo);
let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
- let mut cargo_tails_args = vec![];
-
- if builder.config.llvm_thin_lto {
- cargo_tails_args.push("--".to_string());
-
- let num_jobs = builder.jobs();
-
- if !target.contains("msvc") {
- // Here we assume that the linker is clang. If it's not, there'll
- // be linker errors.
- cargo_tails_args.push("-Clink-arg=-fuse-ld=lld".to_string());
- cargo_tails_args.push("-Clink-arg=-flto=thin".to_string());
-
- if builder.config.llvm_optimize {
- cargo_tails_args.push("-Clink-arg=-O2".to_string());
- }
-
- // Let's make LLD respect the `-j` option.
- let num_jobs_arg = format!("-Clink-arg=-Wl,--thinlto-jobs={}", num_jobs);
- cargo_tails_args.push(num_jobs_arg);
- } else {
- // Here we assume that the linker is lld-link.exe. lld-link.exe
- // does not need the extra arguments except for num_jobs
- let num_jobs_arg = format!("-Clink-arg=/opt:lldltojobs={}", num_jobs);
- cargo_tails_args.push(num_jobs_arg);
- }
- }
-
let tmp_stamp = out_dir.join(".tmp.stamp");
let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
let files = run_cargo(builder,
cargo.arg("--features").arg(features),
- cargo_tails_args,
&tmp_stamp,
false);
if builder.config.dry_run {
@@ -749,7 +718,9 @@ pub fn build_codegen_backend(builder: &Builder,
"libstdc++.a");
cargo.env("LLVM_STATIC_STDCPP", file);
}
- if builder.config.llvm_link_shared {
+ if builder.config.llvm_link_shared ||
+ (builder.config.llvm_thin_lto && backend != "emscripten")
+ {
cargo.env("LLVM_LINK_SHARED", "1");
}
}
@@ -989,6 +960,8 @@ impl Step for Assemble {
copy_lld_to_sysroot(builder, target_compiler, &lld_install);
}
+ dist::maybe_install_llvm_dylib(builder, target_compiler.host, &sysroot);
+
// Link the compiler binary itself into place
let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host);
let rustc = out_dir.join(exe("rustc_binary", &*host));
@@ -1015,7 +988,6 @@ pub fn add_to_sysroot(builder: &Builder, sysroot_dst: &Path, stamp: &Path) {
pub fn run_cargo(builder: &Builder,
cargo: &mut Command,
- tail_args: Vec<String>,
stamp: &Path,
is_check: bool)
-> Vec<PathBuf>
@@ -1038,7 +1010,7 @@ pub fn run_cargo(builder: &Builder,
// files we need to probe for later.
let mut deps = Vec::new();
let mut toplevel = Vec::new();
- let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| {
+ let ok = stream_cargo(builder, cargo, &mut |msg| {
let filenames = match msg {
CargoMessage::CompilerArtifact { filenames, .. } => filenames,
_ => return,
@@ -1163,7 +1135,6 @@ pub fn run_cargo(builder: &Builder,
pub fn stream_cargo(
builder: &Builder,
cargo: &mut Command,
- tail_args: Vec<String>,
cb: &mut dyn FnMut(CargoMessage),
) -> bool {
if builder.config.dry_run {
@@ -1174,10 +1145,6 @@ pub fn stream_cargo(
cargo.arg("--message-format").arg("json")
.stdout(Stdio::piped());
- for arg in tail_args {
- cargo.arg(arg);
- }
-
builder.verbose(&format!("running: {:?}", cargo));
let mut child = match cargo.spawn() {
Ok(child) => child,
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 9f97e57b4562e..97a359639cb83 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -1877,13 +1877,13 @@ impl Step for HashSign {
// LLVM tools are linked dynamically.
// Note: This function does no yet support Windows but we also don't support
// linking LLVM tools dynamically on Windows yet.
-fn maybe_install_llvm_dylib(builder: &Builder,
- target: Interned<String>,
- image: &Path) {
+pub fn maybe_install_llvm_dylib(builder: &Builder,
+ target: Interned<String>,
+ sysroot: &Path) {
let src_libdir = builder
.llvm_out(target)
.join("lib");
- let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
+ let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
t!(fs::create_dir_all(&dst_libdir));
if target.contains("apple-darwin") {
diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
index 7782351a552d4..d31ea0f845873 100644
--- a/src/bootstrap/tool.rs
+++ b/src/bootstrap/tool.rs
@@ -77,7 +77,7 @@ impl Step for ToolBuild {
let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
let mut duplicates = Vec::new();
- let is_expected = compile::stream_cargo(builder, &mut cargo, vec![], &mut |msg| {
+ let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| {
// Only care about big things like the RLS/Cargo for now
match tool {
| "rls"
diff --git a/src/librustc_llvm/build.rs b/src/librustc_llvm/build.rs
index 238e79dfd22e1..98cf20a7ba7f4 100644
--- a/src/librustc_llvm/build.rs
+++ b/src/librustc_llvm/build.rs
@@ -132,6 +132,10 @@ fn main() {
continue;
}
+ if flag.starts_with("-flto") {
+ continue;
+ }
+
// -Wdate-time is not supported by the netbsd cross compiler
if is_crossed && target.contains("netbsd") && flag.contains("date-time") {
continue;
From a175969b293ccb9db3aefbe2cc1656a816295025 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 18 Dec 2018 13:01:23 -0800
Subject: [PATCH 2/5] Avoid using open_global_now
---
src/librustc_driver/lib.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs
index e9f309373008b..62f4895354fd0 100644
--- a/src/librustc_driver/lib.rs
+++ b/src/librustc_driver/lib.rs
@@ -207,7 +207,7 @@ fn load_backend_from_dylib(path: &Path) -> fn() -> Box<dyn CodegenBackend> {
// available for future dynamic libraries opened. This is currently used by
// loading LLVM and then making its symbols available for other dynamic
// libraries.
- let lib = DynamicLibrary::open_global_now(path).unwrap_or_else(|err| {
+ let lib = DynamicLibrary::open(Some(path)).unwrap_or_else(|err| {
let err = format!("couldn't load codegen backend {:?}: {:?}", path, err);
early_error(ErrorOutputType::default(), &err);
});
From 82a1bb331438f0a1432f13a08cb431475dab9907 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 18 Dec 2018 19:03:12 -0800
Subject: [PATCH 3/5] Remove no longer working test
---
src/test/run-make-fulldeps/llvm-pass/Makefile | 28 -----------
.../llvm-pass/llvm-function-pass.so.cc | 46 -------------------
.../llvm-pass/llvm-module-pass.so.cc | 45 ------------------
src/test/run-make-fulldeps/llvm-pass/main.rs | 4 --
.../run-make-fulldeps/llvm-pass/plugin.rs | 18 --------
5 files changed, 141 deletions(-)
delete mode 100644 src/test/run-make-fulldeps/llvm-pass/Makefile
delete mode 100644 src/test/run-make-fulldeps/llvm-pass/llvm-function-pass.so.cc
delete mode 100644 src/test/run-make-fulldeps/llvm-pass/llvm-module-pass.so.cc
delete mode 100644 src/test/run-make-fulldeps/llvm-pass/main.rs
delete mode 100644 src/test/run-make-fulldeps/llvm-pass/plugin.rs
diff --git a/src/test/run-make-fulldeps/llvm-pass/Makefile b/src/test/run-make-fulldeps/llvm-pass/Makefile
deleted file mode 100644
index 8a18aadf36a8b..0000000000000
--- a/src/test/run-make-fulldeps/llvm-pass/Makefile
+++ /dev/null
@@ -1,28 +0,0 @@
--include ../tools.mk
-
-ifeq ($(UNAME),Darwin)
-PLUGIN_FLAGS := -C link-args=-Wl,-undefined,dynamic_lookup
-endif
-
-ifeq ($(findstring stage1,$(RUST_BUILD_STAGE)),stage1)
-# ignore stage1
-all:
-
-else
-# Windows doesn't correctly handle include statements with escaping paths,
-# so this test will not get run on Windows.
-ifdef IS_WINDOWS
-all:
-else
-all: $(call NATIVE_STATICLIB,llvm-function-pass) $(call NATIVE_STATICLIB,llvm-module-pass)
- $(RUSTC) plugin.rs -C prefer-dynamic $(PLUGIN_FLAGS)
- $(RUSTC) main.rs
-
-$(TMPDIR)/libllvm-function-pass.o:
- $(CXX) $(CFLAGS) $(LLVM_CXXFLAGS) -c llvm-function-pass.so.cc -o $(TMPDIR)/libllvm-function-pass.o
-
-$(TMPDIR)/libllvm-module-pass.o:
- $(CXX) $(CFLAGS) $(LLVM_CXXFLAGS) -c llvm-module-pass.so.cc -o $(TMPDIR)/libllvm-module-pass.o
-endif
-
-endif
diff --git a/src/test/run-make-fulldeps/llvm-pass/llvm-function-pass.so.cc b/src/test/run-make-fulldeps/llvm-pass/llvm-function-pass.so.cc
deleted file mode 100644
index 267c68853ac43..0000000000000
--- a/src/test/run-make-fulldeps/llvm-pass/llvm-function-pass.so.cc
+++ /dev/null
@@ -1,46 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "llvm/Pass.h"
-#include "llvm/IR/Function.h"
-
-using namespace llvm;
-
-namespace {
-
- class TestLLVMPass : public FunctionPass {
-
- public:
-
- static char ID;
- TestLLVMPass() : FunctionPass(ID) { }
-
- bool runOnFunction(Function &F) override;
-
- StringRef getPassName() const override {
- return "Some LLVM pass";
- }
-
- };
-
-}
-
-bool TestLLVMPass::runOnFunction(Function &F) {
- // A couple examples of operations that previously caused segmentation faults
- // https://github.com/rust-lang/rust/issues/31067
-
- for (auto N = F.begin(); N != F.end(); ++N) {
- /* code */
- }
-
- LLVMContext &C = F.getContext();
- IntegerType *Int8Ty = IntegerType::getInt8Ty(C);
- PointerType::get(Int8Ty, 0);
- return true;
-}
-
-char TestLLVMPass::ID = 0;
-
-static RegisterPass<TestLLVMPass> RegisterAFLPass(
- "some-llvm-function-pass", "Some LLVM pass");
diff --git a/src/test/run-make-fulldeps/llvm-pass/llvm-module-pass.so.cc b/src/test/run-make-fulldeps/llvm-pass/llvm-module-pass.so.cc
deleted file mode 100644
index b3ee0c1a6c10b..0000000000000
--- a/src/test/run-make-fulldeps/llvm-pass/llvm-module-pass.so.cc
+++ /dev/null
@@ -1,45 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "llvm/IR/Module.h"
-
-using namespace llvm;
-
-namespace {
-
- class TestLLVMPass : public ModulePass {
-
- public:
-
- static char ID;
- TestLLVMPass() : ModulePass(ID) { }
-
- bool runOnModule(Module &M) override;
-
- StringRef getPassName() const override {
- return "Some LLVM pass";
- }
-
- };
-
-}
-
-bool TestLLVMPass::runOnModule(Module &M) {
- // A couple examples of operations that previously caused segmentation faults
- // https://github.com/rust-lang/rust/issues/31067
-
- for (auto F = M.begin(); F != M.end(); ++F) {
- /* code */
- }
-
- LLVMContext &C = M.getContext();
- IntegerType *Int8Ty = IntegerType::getInt8Ty(C);
- PointerType::get(Int8Ty, 0);
- return true;
-}
-
-char TestLLVMPass::ID = 0;
-
-static RegisterPass<TestLLVMPass> RegisterAFLPass(
- "some-llvm-module-pass", "Some LLVM pass");
diff --git a/src/test/run-make-fulldeps/llvm-pass/main.rs b/src/test/run-make-fulldeps/llvm-pass/main.rs
deleted file mode 100644
index 0c13b890c6e3a..0000000000000
--- a/src/test/run-make-fulldeps/llvm-pass/main.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-#![feature(plugin)]
-#![plugin(some_plugin)]
-
-fn main() {}
diff --git a/src/test/run-make-fulldeps/llvm-pass/plugin.rs b/src/test/run-make-fulldeps/llvm-pass/plugin.rs
deleted file mode 100644
index f0e4800046cf4..0000000000000
--- a/src/test/run-make-fulldeps/llvm-pass/plugin.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-#![feature(plugin_registrar, rustc_private)]
-#![crate_type = "dylib"]
-#![crate_name = "some_plugin"]
-
-extern crate rustc;
-extern crate rustc_plugin;
-
-#[link(name = "llvm-function-pass", kind = "static")]
-#[link(name = "llvm-module-pass", kind = "static")]
-extern {}
-
-use rustc_plugin::registry::Registry;
-
-#[plugin_registrar]
-pub fn plugin_registrar(reg: &mut Registry) {
- reg.register_llvm_pass("some-llvm-function-pass");
- reg.register_llvm_pass("some-llvm-module-pass");
-}
From fa4f014110fdd111742d75f7976ffc754f7720da Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 18 Dec 2018 20:33:10 -0800
Subject: [PATCH 4/5] Remove now stray comment
---
src/librustc_driver/lib.rs | 7 -------
1 file changed, 7 deletions(-)
diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs
index 62f4895354fd0..1ec14fbb06c79 100644
--- a/src/librustc_driver/lib.rs
+++ b/src/librustc_driver/lib.rs
@@ -200,13 +200,6 @@ pub fn run<F>(run_compiler: F) -> isize
}
fn load_backend_from_dylib(path: &Path) -> fn() -> Box<dyn CodegenBackend> {
- // Note that we're specifically using `open_global_now` here rather than
- // `open`, namely we want the behavior on Unix of RTLD_GLOBAL and RTLD_NOW,
- // where NOW means "bind everything right now" because we don't want
- // surprises later on and RTLD_GLOBAL allows the symbols to be made
- // available for future dynamic libraries opened. This is currently used by
- // loading LLVM and then making its symbols available for other dynamic
- // libraries.
let lib = DynamicLibrary::open(Some(path)).unwrap_or_else(|err| {
let err = format!("couldn't load codegen backend {:?}: {:?}", path, err);
early_error(ErrorOutputType::default(), &err);
From 71fed3a8ab479ec46d9fd9a7c5c8f4ffb97786dc Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Wed, 2 Jan 2019 13:07:26 -0800
Subject: [PATCH 5/5] Don't package up libLLVM.so with libstd
It's only meant for rustc, so we should only install it once!
---
src/bootstrap/dist.rs | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 97a359639cb83..38869bf441a82 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -671,10 +671,18 @@ impl Step for Std {
let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
builder.cp_filtered(&src, &dst, &|path| {
- let name = path.file_name().and_then(|s| s.to_str());
- name != Some(builder.config.rust_codegen_backends_dir.as_str()) &&
- name != Some("bin")
-
+ if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
+ if name == builder.config.rust_codegen_backends_dir.as_str() {
+ return false
+ }
+ if name == "bin" {
+ return false
+ }
+ if name.contains("LLVM") {
+ return false
+ }
+ }
+ true
});
let mut cmd = rust_installer(builder);