diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index 2c5ec9dad59f1..1d2e39119817a 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -378,6 +378,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile"));
}
+ // patchable-function is only implemented on x86 on LLVM
+ if cx.sess().opts.unstable_opts.hotpatch && cx.sess().target.is_x86() {
+ to_add.push(llvm::CreateAttrStringValue(
+ cx.llcx,
+ "patchable-function",
+ "prologue-short-redirect",
+ ));
+ }
+
// FIXME: none of these functions interact with source level attributes.
to_add.extend(frame_pointer_type_attr(cx));
to_add.extend(function_return_attr(cx));
diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
index 44c30d22a9e9c..c866c77495cd0 100644
--- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
+++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
@@ -35,6 +35,7 @@ impl OwnedTargetMachine {
emit_stack_size_section: bool,
relax_elf_relocations: bool,
use_init_array: bool,
+ is_hotpatchable: bool,
split_dwarf_file: &CStr,
output_obj_file: &CStr,
debug_info_compression: &CStr,
@@ -67,6 +68,7 @@ impl OwnedTargetMachine {
emit_stack_size_section,
relax_elf_relocations,
use_init_array,
+ is_hotpatchable,
split_dwarf_file.as_ptr(),
output_obj_file.as_ptr(),
debug_info_compression.as_ptr(),
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index afdd2b581b86e..b2212950b7377 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -225,6 +225,11 @@ pub(crate) fn target_machine_factory(
let use_init_array =
!sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
+ // this makes LLVM add a hotpatch flag in the codeview S_COMPILE3 record,
+ // which is required by linkers for the functionpadmin option
+ // aarch64 is always hotpatchable
+ let is_hotpatchable = sess.opts.unstable_opts.hotpatch || sess.target.arch.contains("aarch64");
+
let path_mapping = sess.source_map().path_mapping().clone();
let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
@@ -297,6 +302,7 @@ pub(crate) fn target_machine_factory(
emit_stack_size_section,
relax_elf_relocations,
use_init_array,
+ is_hotpatchable,
&split_dwarf_file,
&output_obj_file,
&debuginfo_compression,
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 661debbb9f126..2eb661b0fc2a1 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -2170,6 +2170,7 @@ unsafe extern "C" {
EmitStackSizeSection: bool,
RelaxELFRelocations: bool,
UseInitArray: bool,
+ IsHotpatchable: bool,
SplitDwarfFile: *const c_char,
OutputObjFile: *const c_char,
DebugInfoCompression: *const c_char,
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 536ce154cd071..b030271a64245 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -788,6 +788,7 @@ fn test_unstable_options_tracking_hash() {
tracked!(fuel, Some(("abc".to_string(), 99)));
tracked!(function_return, FunctionReturn::ThunkExtern);
tracked!(function_sections, Some(false));
+ tracked!(hotpatch, true);
tracked!(human_readable_cgu_names, true);
tracked!(incremental_ignore_spans, true);
tracked!(inline_in_all_cgus, Some(true));
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
index 8f0b1b8127657..69fab1a1f58bd 100644
--- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
@@ -396,7 +396,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
bool FunctionSections, bool DataSections, bool UniqueSectionNames,
bool TrapUnreachable, bool Singlethread, bool VerboseAsm,
bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray,
- const char *SplitDwarfFile, const char *OutputObjFile,
+ bool IsHotpatchable, const char *SplitDwarfFile, const char *OutputObjFile,
const char *DebugInfoCompression, bool UseEmulatedTls,
const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
@@ -426,6 +426,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
// Always preserve comments that were written by the user
Options.MCOptions.PreserveAsmComments = true;
Options.MCOptions.ABIName = ABIStr;
+ Options.Hotpatch = IsHotpatchable;
if (SplitDwarfFile) {
Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
}
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index d63276db4938b..e520154fb78cf 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1810,6 +1810,10 @@ options! {
"explicitly enable the `cfg(target_thread_local)` directive"),
hir_stats: bool = (false, parse_bool, [UNTRACKED],
"print some statistics about AST and HIR (default: no)"),
+ hotpatch: bool = (false, parse_bool, [TRACKED],
+ "ensures hotpatching is always possible by ensuring that the first instruction of \
+ each function is at least two bytes, and no jump within the function goes to the first instruction. \
+ Should be combined with link-arg passing -functionpadmin to the linker. Currently only supported for x86 (default: false)"),
human_readable_cgu_names: bool = (false, parse_bool, [TRACKED],
"generate human-readable, predictable names for codegen units (default: no)"),
identify_regions: bool = (false, parse_bool, [UNTRACKED],
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 82e11a3afce32..77b1be581c3fb 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -2070,6 +2070,9 @@ impl Target {
Ok(dl)
}
+ pub fn is_x86(&self) -> bool {
+ ["x86", "x86_64"].contains(&&self.arch[..])
+ }
}
pub trait HasTargetSpec {
diff --git a/src/tools/run-make-support/src/external_deps/llvm.rs b/src/tools/run-make-support/src/external_deps/llvm.rs
index 38a9ac923b4dc..d644488548251 100644
--- a/src/tools/run-make-support/src/external_deps/llvm.rs
+++ b/src/tools/run-make-support/src/external_deps/llvm.rs
@@ -254,6 +254,13 @@ impl LlvmFilecheck {
self
}
+ /// Specify the prefix (without :) for patterns to match. By default, these patterns are prefixed with "CHECK:".
+ pub fn check_prefix(&mut self, prefix: &str) -> &mut Self {
+ self.cmd.arg("--check-prefix");
+ self.cmd.arg(prefix);
+ self
+ }
+
/// `--input-file` option.
pub fn input_file<P: AsRef<Path>>(&mut self, input_file: P) -> &mut Self {
self.cmd.arg("--input-file");
diff --git a/tests/codegen/hotpatch.rs b/tests/codegen/hotpatch.rs
new file mode 100644
index 0000000000000..70ca8b27bff91
--- /dev/null
+++ b/tests/codegen/hotpatch.rs
@@ -0,0 +1,15 @@
+// check if functions get the attribute, so that LLVM ensures they are hotpatchable
+// the attribute is only implemented for x86, aarch64 does not require it
+
+//@ revisions: x32 x64
+//@[x32] only-x86
+//@[x64] only-x86_64
+//@ compile-flags: -Z hotpatch
+
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn foo() {}
+
+// CHECK-LABEL: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}} "patchable-function"="prologue-short-redirect" {{.*}}}
diff --git a/tests/run-make/hotpatch-unaffected/lib.rs b/tests/run-make/hotpatch-unaffected/lib.rs
new file mode 100644
index 0000000000000..d441d3e412544
--- /dev/null
+++ b/tests/run-make/hotpatch-unaffected/lib.rs
@@ -0,0 +1,25 @@
+// hotpatch has two requirements:
+// 1. the first instruction of a functin must be at least two bytes long
+// 2. there must not be a jump to the first instruction
+
+// the functions in this file already fulfill the conditions so hotpatch should not affect them
+
+// --------------------------------------------------------------------------------------------
+
+#[no_mangle]
+#[inline(never)]
+pub fn return_42() -> i32 {
+ 42
+}
+
+// --------------------------------------------------------------------------------------------
+// This tailcall does not jump to the first instruction so hotpatch should leave it unaffected
+
+#[no_mangle]
+pub fn tailcall(a: i32) -> i32 {
+ if a > 10000 {
+ return a;
+ }
+
+ if a % 2 == 0 { tailcall(a / 2) } else { tailcall(a * 3 + 1) }
+}
diff --git a/tests/run-make/hotpatch-unaffected/rmake.rs b/tests/run-make/hotpatch-unaffected/rmake.rs
new file mode 100644
index 0000000000000..b50047e529c41
--- /dev/null
+++ b/tests/run-make/hotpatch-unaffected/rmake.rs
@@ -0,0 +1,48 @@
+// Check if hotpatch leaves the functions that are already hotpatchable untouched
+
+use run_make_support::{assertion_helpers, llvm, rustc};
+
+fn main() {
+ // hotpatch is only implemented for X86 and aarch64
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
+ {
+ fn base_rustc() -> rustc::Rustc {
+ let mut rustc = rustc();
+ rustc.input("lib.rs").crate_type("lib").opt_level("3");
+ rustc
+ }
+
+ fn dump_lib(libname: &str) -> String {
+ llvm::llvm_objdump()
+ .arg("--disassemble-symbols=return_42,tailcall")
+ .input(libname)
+ .run()
+ .stdout_utf8()
+ }
+
+ base_rustc().crate_name("regular").run();
+ let regular_dump = dump_lib("libregular.rlib");
+
+ base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run();
+ let hotpatch_dump = dump_lib("libhotpatch.rlib");
+
+ {
+ let mut lines_regular = regular_dump.lines();
+ let mut lines_hotpatch = hotpatch_dump.lines();
+
+ loop {
+ match (lines_regular.next(), lines_hotpatch.next()) {
+ (None, None) => break,
+ (Some(r), Some(h)) => {
+ if r.contains("libregular.rlib") {
+ assertion_helpers::assert_contains(h, "libhotpatch.rlib")
+ } else {
+ assertion_helpers::assert_equals(&r, &h)
+ }
+ }
+ _ => panic!("expected files to have equal number of lines"),
+ }
+ }
+ }
+ }
+}
diff --git a/tests/run-make/hotpatch/lib.rs b/tests/run-make/hotpatch/lib.rs
new file mode 100644
index 0000000000000..627f035fbb66d
--- /dev/null
+++ b/tests/run-make/hotpatch/lib.rs
@@ -0,0 +1,27 @@
+// hotpatch has two requirements:
+// 1) the first instruction of a functin must be at least two bytes long
+// 2) there must not be a jump to the first instruction
+
+// The LLVM attribute we use '"patchable-function", "prologue-short-redirect"' only ensures 1)
+// However in practice 2) rarely matters. Its rare that it occurs and the problems it caused can be
+// avoided by the hotpatch tool.
+// In this test we check if 1) is ensured by inserted nops as needed
+
+// ----------------------------------------------------------------------------------------------
+
+// empty_fn just returns. Note that 'ret' is a single byte instruction, but hotpatch requires
+// a two or more byte instructions to be at the start of the functions.
+// Preferably we would also tests a different single byte instruction,
+// but I was not able to find an example with another one byte intstruction.
+
+// check that if the first instruction is just a single byte, so our test is valid
+// CHECK-LABEL: <empty_fn>:
+// CHECK-NOT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}}
+
+// check that the first instruction is at least 2 bytes long
+// HOTPATCH-LABEL: <empty_fn>:
+// HOTPATCH-NEXT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}}
+
+#[no_mangle]
+#[inline(never)]
+pub fn empty_fn() {}
diff --git a/tests/run-make/hotpatch/rmake.rs b/tests/run-make/hotpatch/rmake.rs
new file mode 100644
index 0000000000000..4f43aaa985143
--- /dev/null
+++ b/tests/run-make/hotpatch/rmake.rs
@@ -0,0 +1,42 @@
+// Check if hotpatch makes the functions hotpachable that were not
+// More details in lib.rs
+
+use run_make_support::{llvm, rustc};
+
+fn main() {
+ // hotpatch is only implemented for x86 and aarch64, but for aarch64 functions
+ // are always hotpatchable so we don't need to check it
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
+ {
+ fn base_rustc() -> rustc::Rustc {
+ let mut rustc = rustc();
+ rustc.input("lib.rs").crate_type("lib").opt_level("3");
+ rustc
+ }
+
+ fn dump_lib(libname: &str) -> String {
+ llvm::llvm_objdump()
+ .arg("--disassemble-symbols=empty_fn")
+ .input(libname)
+ .run()
+ .stdout_utf8()
+ }
+
+ {
+ base_rustc().crate_name("regular").run();
+ let regular_dump = dump_lib("libregular.rlib");
+ llvm::llvm_filecheck().patterns("lib.rs").stdin_buf(regular_dump).run();
+ }
+
+ {
+ base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run();
+ let hotpatch_dump = dump_lib("libhotpatch.rlib");
+
+ llvm::llvm_filecheck()
+ .patterns("lib.rs")
+ .check_prefix("HOTPATCH")
+ .stdin_buf(hotpatch_dump)
+ .run();
+ }
+ }
+}
diff --git a/tests/run-make/hotpatch_pdb/main.rs b/tests/run-make/hotpatch_pdb/main.rs
new file mode 100644
index 0000000000000..248ccf37d6115
--- /dev/null
+++ b/tests/run-make/hotpatch_pdb/main.rs
@@ -0,0 +1,6 @@
+// CHECK: S_OBJNAME{{.*}}hotpatch_pdb{{.*}}.o
+// CHECK: S_COMPILE3
+// CHECK-NOT: S_
+// CHECK: flags = {{.*}}hot patchable
+
+pub fn main() {}
diff --git a/tests/run-make/hotpatch_pdb/rmake.rs b/tests/run-make/hotpatch_pdb/rmake.rs
new file mode 100644
index 0000000000000..800245f87cdc8
--- /dev/null
+++ b/tests/run-make/hotpatch_pdb/rmake.rs
@@ -0,0 +1,30 @@
+// Check if hotpatch flag is present in the Codeview.
+// This is need so linkers actually pad functions when given the functionpadmin arg.
+
+use run_make_support::{llvm, rustc};
+
+fn main() {
+ // PDBs are windows only and hotpatch is only implemented for x86 and aarch64
+ #[cfg(all(
+ target_os = "windows",
+ any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")
+ ))]
+ {
+ let output = rustc()
+ .input("main.rs")
+ .arg("-g")
+ .arg("-Zhotpatch")
+ .crate_name("hotpatch_pdb")
+ .crate_type("bin")
+ .run();
+
+ let pdbutil_output = llvm::llvm_pdbutil()
+ .arg("dump")
+ .arg("-symbols")
+ .input("hotpatch_pdb.pdb")
+ .run()
+ .stdout_utf8();
+
+ llvm::llvm_filecheck().patterns("main.rs").stdin_buf(&pdbutil_output).run();
+ }
+}