diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index edd34b52eade7..7200ed8eeb440 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -162,11 +162,14 @@ pub fn compile_codegen_unit( } } - // Create the llvm.used variable - // This variable has type [N x i8*] and is stored in the llvm.metadata section + // Create the llvm.used and llvm.compiler.used variables + // These have type [N x i8*] and are stored in the llvm.metadata section if !cx.used_statics().borrow().is_empty() { cx.create_used_variable() } + if !cx.compiler_used_statics().borrow().is_empty() { + cx.create_compiler_used_variable() + } // Finalize debuginfo if cx.sess().opts.debuginfo != DebugInfo::None { diff --git a/src/librustc_codegen_llvm/consts.rs b/src/librustc_codegen_llvm/consts.rs index fd7054a5a0ada..f534f12500c13 100644 --- a/src/librustc_codegen_llvm/consts.rs +++ b/src/librustc_codegen_llvm/consts.rs @@ -356,6 +356,61 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { gv } + fn append_vtable_lookup( + &self, + vtable_record: &'ll Value, + align: Align, + ) { + // Add a pointer to the vtable to a special section. The linker can provide pointers to the + // start and end of this section, enabling user code to retrieve an array of all + // materializable vtable pointers. + // + // On Windows, the contents of sections is sorted by the string after the $. Static + // variables with link_sections .rdata.__rust_vtables$A and .rdata.__rust_vtables$C can be + // used as the start and end pointers. Unreferenced data elimination (rustc enables this + // with /OPT:REF) removes the section if it's not used. + // + // On Mac and iOS, the linker-defined symbols section$start$__DATA$__rust_vtables and + // section$end$__DATA$__rust_vtables are the start and end pointers. Mac aggressively strips + // ostensibly-dead symbols (llvm and rustc enable this with .subsections_via_symbols and + // -dead_strip). live_support inverts the live-marking process: symbols that reference live + // symbols are themselves marked live. + // + // On Linux and Android, the linker-defined symbols __start___rust_vtables and + // __stop___rust_vtables are the start and end pointers. Section garbage collection (rustc + // enables this with --gc-sections) removes the section if it's not used. + // + // A few other platforms work much the same way as Linux here, but for now just do nothing. + let sect_name = if self.tcx.sess.target.target.options.is_like_windows { + const_cstr!(".rdata.__rust_vtables$B") + } else if self.tcx.sess.target.target.options.is_like_osx { + const_cstr!("__DATA,__rust_vtables,regular,live_support") + } else if self.tcx.sess.opts.target_triple.triple().contains("-linux-") { + const_cstr!("__rust_vtables") + } else { + return; + }; + // members of llvm.compiler.used must be named + let name = self.generate_local_symbol_name("vtable_record"); + let gv = self.define_global(&name[..], self.val_ty(vtable_record)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", name); + }); + unsafe { + llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); + llvm::LLVMSetInitializer(gv, vtable_record); + set_global_alignment(&self, gv, align); + SetUnnamedAddr(gv, true); + llvm::LLVMSetGlobalConstant(gv, True); + llvm::LLVMSetSection(gv, sect_name.as_ptr()); + + // Add this static to the special llvm.compiler.used variable, which is an array of i8* + // that prevents LLVM from optimizing away referenced values. Use this rather than + // llvm.used so that the linker can optimize away the section if it is unused. + let cast = llvm::LLVMConstPointerCast(gv, self.type_i8p()); + self.compiler_used_statics.borrow_mut().push(cast); + } + } + fn codegen_static( &self, def_id: DefId, diff --git a/src/librustc_codegen_llvm/context.rs b/src/librustc_codegen_llvm/context.rs index 2da9387717214..0c2343edd6acc 100644 --- a/src/librustc_codegen_llvm/context.rs +++ b/src/librustc_codegen_llvm/context.rs @@ -75,6 +75,9 @@ pub struct CodegenCx<'ll, 'tcx> { /// Statics that will be placed in the llvm.used variable /// See for details pub used_statics: RefCell>, + /// Statics that will be placed in the llvm.compiler.used variable + /// see for details + pub compiler_used_statics: RefCell>, pub lltypes: RefCell, Option), &'ll Type>>, pub scalar_lltypes: RefCell, &'ll Type>>, @@ -301,6 +304,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { const_globals: Default::default(), statics_to_rauw: RefCell::new(Vec::new()), used_statics: RefCell::new(Vec::new()), + compiler_used_statics: RefCell::new(Vec::new()), lltypes: Default::default(), scalar_lltypes: Default::default(), pointee_infos: Default::default(), @@ -438,6 +442,10 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { &self.used_statics } + fn compiler_used_statics(&self) -> &RefCell> { + &self.compiler_used_statics + } + fn set_frame_pointer_elimination(&self, llfn: &'ll Value) { attributes::set_frame_pointer_elimination(self, llfn) } @@ -463,6 +471,24 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMSetSection(g, section.as_ptr()); } } + + fn create_compiler_used_variable(&self) { + let name = const_cstr!("llvm.compiler.used"); + let section = const_cstr!("llvm.metadata"); + let array = self.const_array( + &self.type_ptr_to(self.type_i8()), + &*self.compiler_used_statics.borrow() + ); + + unsafe { + let g = llvm::LLVMAddGlobal(self.llmod, + self.val_ty(array), + name.as_ptr()); + llvm::LLVMSetInitializer(g, array); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); + llvm::LLVMSetSection(g, section.as_ptr()); + } + } } impl CodegenCx<'b, 'tcx> { diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index ee4ec7fb41eac..92249f14f11e5 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -140,7 +140,7 @@ pub fn unsized_info<'tcx, Cx: CodegenMethods<'tcx>>( (_, &ty::Dynamic(ref data, ..)) => { let vtable_ptr = cx.layout_of(cx.tcx().mk_mut_ptr(target)) .field(cx, FAT_PTR_EXTRA); - cx.const_ptrcast(meth::get_vtable(cx, source, data.principal()), + cx.const_ptrcast(meth::get_vtable(cx, source, target, data.principal()), cx.backend_type(vtable_ptr)) } _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", diff --git a/src/librustc_codegen_ssa/meth.rs b/src/librustc_codegen_ssa/meth.rs index 266d2e5b18d22..1d545f40e977a 100644 --- a/src/librustc_codegen_ssa/meth.rs +++ b/src/librustc_codegen_ssa/meth.rs @@ -67,6 +67,7 @@ impl<'a, 'tcx> VirtualIndex { pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( cx: &Cx, ty: Ty<'tcx>, + trait_ty: Ty<'tcx>, trait_ref: Option>, ) -> Cx::Value { let tcx = cx.tcx(); @@ -117,6 +118,11 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( let align = cx.data_layout().pointer_align.abi; let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); + let type_id = tcx.type_id_hash(trait_ty); + let type_id = cx.const_u64(type_id); + let vtable_record = cx.const_struct(&[type_id, vtable], true); + cx.append_vtable_lookup(vtable_record, align); + cx.create_vtable_metadata(ty, vtable); cx.vtables().borrow_mut().insert((ty, trait_ref), vtable); diff --git a/src/librustc_codegen_ssa/traits/misc.rs b/src/librustc_codegen_ssa/traits/misc.rs index 658ddd0028076..8ae4c88e9a994 100644 --- a/src/librustc_codegen_ssa/traits/misc.rs +++ b/src/librustc_codegen_ssa/traits/misc.rs @@ -18,7 +18,9 @@ pub trait MiscMethods<'tcx>: BackendTypes { fn sess(&self) -> &Session; fn codegen_unit(&self) -> &Arc>; fn used_statics(&self) -> &RefCell>; + fn compiler_used_statics(&self) -> &RefCell>; fn set_frame_pointer_elimination(&self, llfn: Self::Function); fn apply_target_cpu_attr(&self, llfn: Self::Function); fn create_used_variable(&self); + fn create_compiler_used_variable(&self); } diff --git a/src/librustc_codegen_ssa/traits/statics.rs b/src/librustc_codegen_ssa/traits/statics.rs index 5c108f9fa6cc5..00a39df8c5260 100644 --- a/src/librustc_codegen_ssa/traits/statics.rs +++ b/src/librustc_codegen_ssa/traits/statics.rs @@ -4,6 +4,7 @@ use rustc::ty::layout::Align; pub trait StaticMethods: BackendTypes { fn static_addr_of(&self, cv: Self::Value, align: Align, kind: Option<&str>) -> Self::Value; + fn append_vtable_lookup(&self, cv: Self::Value, align: Align); fn codegen_static(&self, def_id: DefId, is_mutable: bool); } diff --git a/src/librustc_mir/interpret/cast.rs b/src/librustc_mir/interpret/cast.rs index 9ab347957f97a..0e589fdcb654a 100644 --- a/src/librustc_mir/interpret/cast.rs +++ b/src/librustc_mir/interpret/cast.rs @@ -278,7 +278,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } (_, &ty::Dynamic(ref data, _)) => { // Initial cast from sized to dyn trait - let vtable = self.get_vtable(src_pointee_ty, data.principal())?; + let vtable = self.get_vtable(src_pointee_ty, dest_pointee_ty, data.principal())?; let ptr = self.read_immediate(src)?.to_scalar_ptr()?; let val = Immediate::new_dyn_trait(ptr, vtable); self.write_immediate(val, dest) diff --git a/src/librustc_mir/interpret/traits.rs b/src/librustc_mir/interpret/traits.rs index 10b767ebba191..018f15c92ee1b 100644 --- a/src/librustc_mir/interpret/traits.rs +++ b/src/librustc_mir/interpret/traits.rs @@ -14,6 +14,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub fn get_vtable( &mut self, ty: Ty<'tcx>, + _poly_trait_ty: Ty<'tcx>, poly_trait_ref: Option>, ) -> InterpResult<'tcx, Pointer> { trace!("get_vtable(trait_ref={:?})", poly_trait_ref); diff --git a/src/test/ui/vtables/lookup.rs b/src/test/ui/vtables/lookup.rs new file mode 100644 index 0000000000000..70e697d58cae3 --- /dev/null +++ b/src/test/ui/vtables/lookup.rs @@ -0,0 +1,96 @@ +// Only Windows, Linux, Android, macOS and iOS have this implemented for now. + +// run-pass + +// ignore-cloudabi +// ignore-dragonfly +// ignore-emscripten +// ignore-freebsd +// ignore-haiku +// ignore-netbsd +// ignore-openbsd +// ignore-solaris +// ignore-sgx + +#![feature(core_intrinsics)] +#![feature(ptr_offset_from)] +#![feature(raw)] +#![feature(test)] + +use std::intrinsics::type_id; +use std::mem::transmute; +use std::raw::TraitObject; +use std::slice; + +#[derive(Copy, Clone)] +#[repr(C, packed)] +struct Record { + type_id: u64, + vtable: &'static (), +} + +fn vtables() -> &'static [Record] { + // This must be kept in sync with append_vtable_lookup in src/librustc_codegen_llvm/consts.rs + // + // \u{1} is used for the apple link_names to tell llvm not to prefix with an underscore + #[cfg(any(target_os = "linux", target_os = "android", target_os = "macos", target_os = "ios"))] + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr( + any(target_os = "linux", target_os = "android"), + link_name = "__start___rust_vtables" + )] + #[cfg_attr( + any(target_os = "macos", target_os = "ios"), + link_name = "\u{1}section$start$__DATA$__rust_vtables" + )] + static START: [Record; 0]; + #[cfg_attr( + any(target_os = "linux", target_os = "android"), + link_name = "__stop___rust_vtables" + )] + #[cfg_attr( + any(target_os = "macos", target_os = "ios"), + link_name = "\u{1}section$end$__DATA$__rust_vtables" + )] + static END: [Record; 0]; + } + + #[cfg(target_os = "windows")] + { + #[link_section = ".rdata.__rust_vtables$A"] + static START: [Record; 0] = []; + #[link_section = ".rdata.__rust_vtables$C"] + static END: [Record; 0] = []; + } + + unsafe { + let (start_ptr, end_ptr) = (&START as *const Record, &END as *const Record); + slice::from_raw_parts(start_ptr, end_ptr.offset_from(start_ptr) as usize) + } +} + +trait Trait {} +struct Struct; +impl Trait for Struct {} + +#[inline(never)] +fn multiple_upcasts() { + let a: &dyn Trait = &Struct; + let b: &(dyn Trait + Send) = &Struct; + std::hint::black_box((a, b)); +} + +fn main() { + let vtable: &'static () = unsafe { &*transmute::<&dyn Trait, TraitObject>(&Struct).vtable }; + let type_id = unsafe { type_id::() }; + let count = vtables().iter().filter(|&record| record.type_id == type_id).count(); + assert_ne!(count, 0, "The vtable record for dyn Trait is missing"); + assert_eq!(count, 1, "Duplicate vtable records found for dyn Trait"); + let record = vtables().iter().find(|&record| record.vtable as *const () == vtable).unwrap(); + assert_eq!( + record.vtable as *const (), vtable, + "The vtable for Struct as dyn Trait is incorrect" + ); + multiple_upcasts(); +}