diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 91b1547cb6ea6..40db18a4656f4 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -8,6 +8,7 @@ use rustc_index::IndexVec;
use rustc_middle::ty::adjustment::PointerCoercion;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::GenericArg;
use crate::constant::ConstantCx;
use crate::debuginfo::FunctionDebugContext;
@@ -354,6 +355,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
rustc_hir::LangItem::PanicBoundsCheck,
&[index, len, location],
source_info.span,
+ None,
);
}
AssertKind::MisalignedPointerDereference { ref required, ref found } => {
@@ -366,8 +368,25 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
rustc_hir::LangItem::PanicMisalignedPointerDereference,
&[required, found, location],
source_info.span,
+ None,
);
}
+ AssertKind::OccupiedNiche { ref found, ref start, ref end } => {
+ let found = codegen_operand(fx, found);
+ let generic_arg = ty::GenericArg::from(found.layout().ty);
+ let found = found.load_scalar(fx);
+ let start = codegen_operand(fx, start).load_scalar(fx);
+ let end = codegen_operand(fx, end).load_scalar(fx);
+ let location = fx.get_caller_location(source_info).load_scalar(fx);
+
+ codegen_panic_inner(
+ fx,
+ rustc_hir::LangItem::PanicOccupiedNiche,
+ &[found, start, end, location],
+ source_info.span,
+ Some(generic_arg),
+ )
+ }
_ => {
let msg_str = msg.description();
codegen_panic(fx, msg_str, source_info);
@@ -945,7 +964,7 @@ pub(crate) fn codegen_panic<'tcx>(
let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
let args = [msg_ptr, msg_len, location];
- codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span);
+ codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span, None);
}
pub(crate) fn codegen_panic_nounwind<'tcx>(
@@ -957,7 +976,7 @@ pub(crate) fn codegen_panic_nounwind<'tcx>(
let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
let args = [msg_ptr, msg_len];
- codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span);
+ codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span, None);
}
pub(crate) fn codegen_unwind_terminate<'tcx>(
@@ -967,7 +986,7 @@ pub(crate) fn codegen_unwind_terminate<'tcx>(
) {
let args = [];
- codegen_panic_inner(fx, reason.lang_item(), &args, source_info.span);
+ codegen_panic_inner(fx, reason.lang_item(), &args, source_info.span, None);
}
fn codegen_panic_inner<'tcx>(
@@ -975,10 +994,16 @@ fn codegen_panic_inner<'tcx>(
lang_item: rustc_hir::LangItem,
args: &[Value],
span: Span,
+ generic: Option<GenericArg<'tcx>>,
) {
let def_id = fx.tcx.require_lang_item(lang_item, Some(span));
- let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
+ let instance = if let Some(arg) = generic {
+ Instance::new(def_id, fx.tcx.mk_args(&[arg]))
+ } else {
+ Instance::mono(fx.tcx, def_id)
+ }
+ .polymorphize(fx.tcx);
let symbol_name = fx.tcx.symbol_name(instance).name;
fx.lib_call(
diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs
index 641ac3eb80872..9e2372f614ce8 100644
--- a/compiler/rustc_codegen_ssa/src/common.rs
+++ b/compiler/rustc_codegen_ssa/src/common.rs
@@ -2,7 +2,7 @@
use rustc_hir::LangItem;
use rustc_middle::mir;
-use rustc_middle::ty::{self, layout::TyAndLayout, Ty, TyCtxt};
+use rustc_middle::ty::{self, layout::TyAndLayout, GenericArg, Ty, TyCtxt};
use rustc_span::Span;
use crate::base;
@@ -120,10 +120,15 @@ pub fn build_langcall<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
bx: &Bx,
span: Option<Span>,
li: LangItem,
+ generic: Option<GenericArg<'tcx>>,
) -> (Bx::FnAbiOfResult, Bx::Value) {
let tcx = bx.tcx();
let def_id = tcx.require_lang_item(li, span);
- let instance = ty::Instance::mono(tcx, def_id);
+ let instance = if let Some(arg) = generic {
+ ty::Instance::new(def_id, tcx.mk_args(&[arg]))
+ } else {
+ ty::Instance::mono(tcx, def_id)
+ };
(bx.fn_abi_of_instance(instance, ty::List::empty()), bx.get_fn_addr(instance))
}
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 3d2d8f8b50990..ec784d10fa5f1 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -572,7 +572,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
mergeable_succ: bool,
) -> MergingSucc {
let span = terminator.source_info.span;
- let cond = self.codegen_operand(bx, cond).immediate();
+ let mut cond = self.codegen_operand(bx, cond).immediate();
let mut const_cond = bx.const_to_opt_u128(cond, false).map(|c| c == 1);
// This case can currently arise only from functions marked
@@ -588,8 +588,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
return helper.funclet_br(self, bx, target, mergeable_succ);
}
- // Pass the condition through llvm.expect for branch hinting.
- let cond = bx.expect(cond, expected);
+ if bx.tcx().sess.opts.optimize != OptLevel::No {
+ // Pass the condition through llvm.expect for branch hinting.
+ cond = bx.expect(cond, expected);
+ }
// Create the failure block and the conditional branch to it.
let lltarget = helper.llbb_with_cleanup(self, target);
@@ -608,30 +610,40 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let location = self.get_caller_location(bx, terminator.source_info).immediate();
// Put together the arguments to the panic entry point.
- let (lang_item, args) = match msg {
+ let (lang_item, args, generic) = match msg {
AssertKind::BoundsCheck { ref len, ref index } => {
let len = self.codegen_operand(bx, len).immediate();
let index = self.codegen_operand(bx, index).immediate();
// It's `fn panic_bounds_check(index: usize, len: usize)`,
// and `#[track_caller]` adds an implicit third argument.
- (LangItem::PanicBoundsCheck, vec![index, len, location])
+ (LangItem::PanicBoundsCheck, vec![index, len, location], None)
}
AssertKind::MisalignedPointerDereference { ref required, ref found } => {
let required = self.codegen_operand(bx, required).immediate();
let found = self.codegen_operand(bx, found).immediate();
// It's `fn panic_misaligned_pointer_dereference(required: usize, found: usize)`,
// and `#[track_caller]` adds an implicit third argument.
- (LangItem::PanicMisalignedPointerDereference, vec![required, found, location])
+ (LangItem::PanicMisalignedPointerDereference, vec![required, found, location], None)
+ }
+ AssertKind::OccupiedNiche { ref found, ref start, ref end } => {
+ let found = self.codegen_operand(bx, found);
+ let generic_arg = ty::GenericArg::from(found.layout.ty);
+ let found = found.immediate();
+ let start = self.codegen_operand(bx, start).immediate();
+ let end = self.codegen_operand(bx, end).immediate();
+ // It's `fn panic_occupied_niche<T>(found: T, start: T, end: T)`,
+ // and `#[track_caller]` adds an implicit fourth argument.
+ (LangItem::PanicOccupiedNiche, vec![found, start, end, location], Some(generic_arg))
}
_ => {
let msg = bx.const_str(msg.description());
// It's `pub fn panic(expr: &str)`, with the wide reference being passed
// as two arguments, and `#[track_caller]` adds an implicit third argument.
- (LangItem::Panic, vec![msg.0, msg.1, location])
+ (LangItem::Panic, vec![msg.0, msg.1, location], None)
}
};
- let (fn_abi, llfn) = common::build_langcall(bx, Some(span), lang_item);
+ let (fn_abi, llfn) = common::build_langcall(bx, Some(span), lang_item, generic);
// Codegen the actual panic invoke/call.
let merging_succ = helper.do_call(self, bx, fn_abi, llfn, &args, None, unwind, &[], false);
@@ -650,7 +662,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
self.set_debug_loc(bx, terminator.source_info);
// Obtain the panic entry point.
- let (fn_abi, llfn) = common::build_langcall(bx, Some(span), reason.lang_item());
+ let (fn_abi, llfn) = common::build_langcall(bx, Some(span), reason.lang_item(), None);
// Codegen the actual panic invoke/call.
let merging_succ = helper.do_call(
@@ -711,8 +723,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let msg = bx.const_str(&msg_str);
// Obtain the panic entry point.
- let (fn_abi, llfn) =
- common::build_langcall(bx, Some(source_info.span), LangItem::PanicNounwind);
+ let (fn_abi, llfn) = common::build_langcall(
+ bx,
+ Some(source_info.span),
+ LangItem::PanicNounwind,
+ None,
+ );
// Codegen the actual panic invoke/call.
helper.do_call(
@@ -1586,7 +1602,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span));
- let (fn_abi, fn_ptr) = common::build_langcall(&bx, None, reason.lang_item());
+ let (fn_abi, fn_ptr) = common::build_langcall(&bx, None, reason.lang_item(), None);
let fn_ty = bx.fn_decl_backend_type(&fn_abi);
let llret = bx.call(fn_ty, None, Some(&fn_abi), fn_ptr, &[], funclet.as_ref());
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index 4b447229c5f61..8e949ca1e5b62 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -561,6 +561,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
found: eval_to_int(found)?,
}
}
+ OccupiedNiche { ref found, ref start, ref end } => OccupiedNiche {
+ found: eval_to_int(found)?,
+ start: eval_to_int(start)?,
+ end: eval_to_int(end)?,
+ },
};
Err(ConstEvalErrKind::AssertFailure(err).into())
}
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 1d1a1ee886272..65377efdb45a6 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -234,6 +234,7 @@ language_item_table! {
ConstPanicFmt, sym::const_panic_fmt, const_panic_fmt, Target::Fn, GenericRequirement::None;
PanicBoundsCheck, sym::panic_bounds_check, panic_bounds_check_fn, Target::Fn, GenericRequirement::Exact(0);
PanicMisalignedPointerDereference, sym::panic_misaligned_pointer_dereference, panic_misaligned_pointer_dereference_fn, Target::Fn, GenericRequirement::Exact(0);
+ PanicOccupiedNiche, sym::panic_occupied_niche, panic_occupied_niche_fn, Target::Fn, GenericRequirement::Exact(1);
PanicInfo, sym::panic_info, panic_info, Target::Struct, GenericRequirement::None;
PanicLocation, sym::panic_location, panic_location, Target::Struct, GenericRequirement::None;
PanicImpl, sym::panic_impl, panic_impl, Target::Fn, GenericRequirement::None;
diff --git a/compiler/rustc_middle/messages.ftl b/compiler/rustc_middle/messages.ftl
index 27d555d7e26c7..0c613cb3718f6 100644
--- a/compiler/rustc_middle/messages.ftl
+++ b/compiler/rustc_middle/messages.ftl
@@ -17,6 +17,9 @@ middle_assert_gen_resume_after_panic = `gen` fn or block cannot be further itera
middle_assert_misaligned_ptr_deref =
misaligned pointer dereference: address must be a multiple of {$required} but is {$found}
+middle_assert_occupied_niche =
+ occupied niche: {$found} must be in {$start}..={$end}
+
middle_assert_op_overflow =
attempt to compute `{$left} {$op} {$right}`, which would overflow
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 7b0f27f9b348c..448de1682756d 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -886,6 +886,7 @@ pub enum AssertKind<O> {
ResumedAfterReturn(CoroutineKind),
ResumedAfterPanic(CoroutineKind),
MisalignedPointerDereference { required: O, found: O },
+ OccupiedNiche { found: O, start: O, end: O },
}
#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 9dfbe1733cc51..27900ab10cf24 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -157,7 +157,7 @@ impl<O> AssertKind<O> {
"`gen fn` should just keep returning `None` after panicking"
}
- BoundsCheck { .. } | MisalignedPointerDereference { .. } => {
+ BoundsCheck { .. } | MisalignedPointerDereference { .. } | OccupiedNiche { .. } => {
bug!("Unexpected AssertKind")
}
}
@@ -220,6 +220,13 @@ impl<O> AssertKind<O> {
"\"misaligned pointer dereference: address must be a multiple of {{}} but is {{}}\", {required:?}, {found:?}"
)
}
+ OccupiedNiche { found, start, end } => {
+ write!(
+ f,
+ "\"occupied niche: {{}} must be in {{}}..={{}}\", {:?}, {:?}, {:?}",
+ found, start, end
+ )
+ }
_ => write!(f, "\"{}\"", self.description()),
}
}
@@ -254,8 +261,8 @@ impl<O> AssertKind<O> {
ResumedAfterPanic(CoroutineKind::Coroutine) => {
middle_assert_coroutine_resume_after_panic
}
-
MisalignedPointerDereference { .. } => middle_assert_misaligned_ptr_deref,
+ OccupiedNiche { .. } => middle_assert_occupied_niche,
}
}
@@ -292,6 +299,11 @@ impl<O> AssertKind<O> {
add!("required", format!("{required:#?}"));
add!("found", format!("{found:#?}"));
}
+ OccupiedNiche { found, start, end } => {
+ add!("found", format!("{found:?}"));
+ add!("start", format!("{start:?}"));
+ add!("end", format!("{end:?}"));
+ }
}
}
}
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index d47cfd5712f8b..6c5e5a87b00f4 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -625,6 +625,11 @@ macro_rules! make_mir_visitor {
self.visit_operand(required, location);
self.visit_operand(found, location);
}
+ OccupiedNiche { found, start, end } => {
+ self.visit_operand(found, location);
+ self.visit_operand(start, location);
+ self.visit_operand(end, location);
+ }
}
}
diff --git a/compiler/rustc_mir_transform/src/check_niches.rs b/compiler/rustc_mir_transform/src/check_niches.rs
new file mode 100644
index 0000000000000..0b67acda1db4f
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/check_niches.rs
@@ -0,0 +1,484 @@
+use crate::MirPass;
+use rustc_hir::def::DefKind;
+use rustc_hir::lang_items::LangItem;
+use rustc_hir::Unsafety;
+use rustc_index::IndexVec;
+use rustc_middle::mir::interpret::{Pointer, Scalar};
+use rustc_middle::mir::visit::NonMutatingUseContext;
+use rustc_middle::mir::visit::PlaceContext;
+use rustc_middle::mir::visit::Visitor;
+use rustc_middle::mir::*;
+use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::{ParamEnv, Ty, TyCtxt, TypeAndMut};
+use rustc_session::Session;
+use rustc_target::abi::{Integer, Niche, Primitive, Size};
+
+pub struct CheckNiches;
+
+impl<'tcx> MirPass<'tcx> for CheckNiches {
+ fn is_enabled(&self, sess: &Session) -> bool {
+ sess.opts.debug_assertions
+ }
+
+ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+ // This pass emits new panics. If for whatever reason we do not have a panic
+ // implementation, running this pass may cause otherwise-valid code to not compile.
+ if tcx.lang_items().get(LangItem::PanicImpl).is_none() {
+ return;
+ }
+
+ let def_id = body.source.def_id();
+ if tcx.type_of(def_id).instantiate_identity().is_coroutine() {
+ return;
+ }
+
+ // This pass will in general insert an enormous amount of checks, so we need some way to
+ // trim them down a bit.
+ // Our tactic here is to only emit checks if we are compiling an `unsafe fn` or a function
+ // that contains an unsafe block. Yes this means that we can fail to check some operations.
+ // If you have a better strategy that doesn't impose 2x compile time overhead, please
+ // share.
+ let is_safe_code =
+ tcx.unsafety_check_result(def_id.expect_local()).used_unsafe_blocks.is_empty();
+ let is_unsafe_fn = match tcx.def_kind(def_id) {
+ DefKind::Closure => false,
+ _ => tcx.fn_sig(def_id).skip_binder().unsafety() == Unsafety::Unsafe,
+ };
+ if is_safe_code && !is_unsafe_fn {
+ return;
+ }
+
+ with_no_trimmed_paths!(debug!("Inserting niche checks for {:?}", body.source));
+
+ let basic_blocks = &body.basic_blocks;
+ let param_env = tcx.param_env_reveal_all_normalized(def_id);
+
+ // This pass inserts new blocks. Each insertion changes the Location for all
+ // statements/blocks after. Iterating or visiting the MIR in order would require updating
+ // our current location after every insertion. By iterating backwards, we dodge this issue:
+ // The only Locations that an insertion changes have already been handled.
+ for block in (0..basic_blocks.len()).rev() {
+ let block = block.into();
+ let basic_blocks = &body.basic_blocks;
+ for statement_index in (0..basic_blocks[block].statements.len()).rev() {
+ let location = Location { block, statement_index };
+ let statement = &body.basic_blocks[block].statements[statement_index];
+ let source_info = statement.source_info;
+
+ let mut finder = NicheFinder { tcx, param_env, body, places: Vec::new() };
+ finder.visit_statement(statement, location);
+ let places = finder.places;
+
+ let mut checker = NicheChecker { tcx, local_decls: &mut body.local_decls };
+ for (place, niche) in places {
+ let basic_blocks = body.basic_blocks.as_mut();
+ let (block_data, new_block) = split_block(basic_blocks, location);
+ checker.insert_niche_check(block_data, new_block, place, niche, source_info);
+ }
+ }
+ }
+ }
+}
+
+struct NicheFinder<'a, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ param_env: ParamEnv<'tcx>,
+ body: &'a Body<'tcx>,
+ places: Vec<(Place<'tcx>, NicheKind)>,
+}
+
+impl<'a, 'tcx> Visitor<'tcx> for NicheFinder<'a, 'tcx> {
+ fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
+ if let Rvalue::Cast(CastKind::Transmute, op, ty) = rvalue {
+ if let Some(niche) = self.get_niche(*ty) && let Some(place) = op.place() {
+ with_no_trimmed_paths!(debug!(
+ "Found place {place:?}: {ty:?} with niche {niche:?} due to Transmute: {:?}",
+ self.body.stmt_at(location)
+ ));
+ self.places.push((place, niche));
+ }
+ } else {
+ self.super_rvalue(rvalue, location);
+ }
+ }
+
+ fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) {
+ match context {
+ PlaceContext::NonMutatingUse(
+ NonMutatingUseContext::Copy | NonMutatingUseContext::Move,
+ ) => {}
+ _ => {
+ return;
+ }
+ }
+
+ let ty = place.ty(self.body, self.tcx).ty;
+ // bool is actually an i1 to LLVM which means a bool local can never be invalid.
+ if ty == self.tcx.types.bool && place.projection.is_empty() {
+ return;
+ }
+ let Some(niche) = self.get_niche(ty) else {
+ return;
+ };
+
+ with_no_trimmed_paths!(debug!(
+ "Found place {place:?}: {ty:?} with niche {niche:?} due to {:?}",
+ self.body.stmt_at(location)
+ ));
+ self.places.push((*place, niche));
+ }
+}
+
+impl<'a, 'tcx> NicheFinder<'a, 'tcx> {
+ fn get_niche(&self, ty: Ty<'tcx>) -> Option<NicheKind> {
+ // If we can't get the layout of this, just skip the check.
+ let Ok(layout) = self.tcx.layout_of(self.param_env.and(ty)) else {
+ return None;
+ };
+
+ let niche = layout.largest_niche?;
+
+ if niche.size(self.tcx) == layout.size {
+ Some(NicheKind::Full(niche))
+ } else {
+ Some(NicheKind::Partial(niche, layout.size))
+ }
+ }
+}
+
+struct NicheChecker<'a, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ local_decls: &'a mut IndexVec<Local, LocalDecl<'tcx>>,
+}
+
+impl<'a, 'tcx> NicheChecker<'a, 'tcx> {
+ fn insert_niche_check(
+ &mut self,
+ block_data: &mut BasicBlockData<'tcx>,
+ new_block: BasicBlock,
+ place: Place<'tcx>,
+ niche: NicheKind,
+ source_info: SourceInfo,
+ ) {
+ let mut value_in_niche = self
+ .local_decls
+ .push(LocalDecl::with_source_info(niche.niche().ty(self.tcx), source_info))
+ .into();
+
+ match niche {
+ NicheKind::Full(niche) => {
+ // The niche occupies the entire source Operand, so we can just transmute
+ // directly to the niche primitive.
+ let rvalue =
+ Rvalue::Cast(CastKind::Transmute, Operand::Copy(place), niche.ty(self.tcx));
+ block_data.statements.push(Statement {
+ source_info,
+ kind: StatementKind::Assign(Box::new((value_in_niche, rvalue))),
+ });
+ }
+ NicheKind::Partial(niche, size) => {
+ if niche.offset == Size::ZERO {
+ // FIXME: Delete value_in_niche, we don't need it on this path
+ self.local_decls.pop();
+
+ // Take the address of the place
+ let full_ptr = self
+ .local_decls
+ .push(LocalDecl::with_source_info(
+ Ty::new_ptr(
+ self.tcx,
+ TypeAndMut {
+ ty: place.ty(self.local_decls, self.tcx).ty,
+ mutbl: Mutability::Not,
+ },
+ ),
+ source_info,
+ ))
+ .into();
+ let rvalue = Rvalue::AddressOf(Mutability::Not, place);
+ block_data.statements.push(Statement {
+ source_info,
+ kind: StatementKind::Assign(Box::new((full_ptr, rvalue))),
+ });
+
+ // Cast to a pointer to the niche type
+ let niche_ptr_ty = Ty::new_ptr(
+ self.tcx,
+ TypeAndMut { ty: niche.ty(self.tcx), mutbl: Mutability::Not },
+ );
+ let niche_ptr = self
+ .local_decls
+ .push(LocalDecl::with_source_info(niche_ptr_ty, source_info))
+ .into();
+ let rvalue =
+ Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(full_ptr), niche_ptr_ty);
+ block_data.statements.push(Statement {
+ source_info,
+ kind: StatementKind::Assign(Box::new((niche_ptr, rvalue))),
+ });
+
+ // Set value_in_niche to a projection of our pointer
+ value_in_niche = niche_ptr.project_deeper(&[ProjectionElem::Deref], self.tcx);
+ } else {
+ let mu = Ty::new_maybe_uninit(self.tcx, niche.ty(self.tcx));
+
+ // Transmute the niche-containing type to a [MaybeUninit; N]
+ let array_len = size.bytes() / niche.size(self.tcx).bytes();
+ let mu_array_ty = Ty::new_array(self.tcx, mu, array_len);
+ let mu_array = self
+ .local_decls
+ .push(LocalDecl::with_source_info(mu_array_ty, source_info))
+ .into();
+ let rvalue =
+ Rvalue::Cast(CastKind::Transmute, Operand::Copy(place), mu_array_ty);
+ block_data.statements.push(Statement {
+ source_info,
+ kind: StatementKind::Assign(Box::new((mu_array, rvalue))),
+ });
+
+ // Convert the niche byte offset into an array index
+ assert_eq!(niche.offset.bytes() % niche.size(self.tcx).bytes(), 0);
+ let offset = niche.offset.bytes() / niche.size(self.tcx).bytes();
+
+ let niche_as_mu = mu_array.project_deeper(
+ &[ProjectionElem::ConstantIndex {
+ offset,
+ min_length: array_len,
+ from_end: false,
+ }],
+ self.tcx,
+ );
+
+ // Transmute the MaybeUninit<T> to the niche primitive
+ let rvalue = Rvalue::Cast(
+ CastKind::Transmute,
+ Operand::Copy(niche_as_mu),
+ niche.ty(self.tcx),
+ );
+ block_data.statements.push(Statement {
+ source_info,
+ kind: StatementKind::Assign(Box::new((value_in_niche, rvalue))),
+ });
+ }
+ }
+ }
+
+ let is_in_range = self
+ .local_decls
+ .push(LocalDecl::with_source_info(self.tcx.types.bool, source_info))
+ .into();
+
+ NicheCheckBuilder {
+ tcx: self.tcx,
+ local_decls: self.local_decls,
+ block_data,
+ new_block,
+ niche: niche.niche(),
+ value_in_niche,
+ is_in_range,
+ source_info,
+ }
+ .insert_niche_check();
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+enum NicheKind {
+ Full(Niche),
+ // We need the full Size of the type in order to do the transmute-to-MU approach
+ Partial(Niche, Size),
+}
+
+impl NicheKind {
+ fn niche(self) -> Niche {
+ use NicheKind::*;
+ match self {
+ Full(niche) => niche,
+ Partial(niche, _size) => niche,
+ }
+ }
+}
+
+fn split_block<'a, 'tcx>(
+ basic_blocks: &'a mut IndexVec<BasicBlock, BasicBlockData<'tcx>>,
+ location: Location,
+) -> (&'a mut BasicBlockData<'tcx>, BasicBlock) {
+ let block_data = &mut basic_blocks[location.block];
+
+ // Drain every statement after this one and move the current terminator to a new basic block
+ let new_block = BasicBlockData {
+ statements: block_data.statements.drain(location.statement_index..).collect(),
+ terminator: block_data.terminator.take(),
+ is_cleanup: block_data.is_cleanup,
+ };
+
+ let new_block = basic_blocks.push(new_block);
+ let block_data = &mut basic_blocks[location.block];
+
+ (block_data, new_block)
+}
+
+struct NicheCheckBuilder<'a, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ block_data: &'a mut BasicBlockData<'tcx>,
+ local_decls: &'a mut IndexVec<Local, LocalDecl<'tcx>>,
+ new_block: BasicBlock,
+ niche: Niche,
+ value_in_niche: Place<'tcx>,
+ is_in_range: Place<'tcx>,
+ source_info: SourceInfo,
+}
+
+impl<'a, 'tcx> NicheCheckBuilder<'a, 'tcx> {
+ fn insert_niche_check(&mut self) {
+ let niche = self.niche;
+
+ let size = niche.size(self.tcx);
+
+ if niche.valid_range.start == 0 {
+ // The niche starts at 0, so we can just check if it is Le the end
+ self.check_end_only();
+ } else if niche.valid_range.end == (u128::MAX >> (128 - size.bits())) {
+ // The niche ends at the max, so we can just check if it is Ge the start
+ self.check_start_only();
+ } else {
+ self.general_case();
+ }
+
+ let start = self.niche_const(niche.valid_range.start);
+ let end = self.niche_const(niche.valid_range.end);
+
+ self.block_data.terminator = Some(Terminator {
+ source_info: self.source_info,
+ kind: TerminatorKind::Assert {
+ cond: Operand::Copy(self.is_in_range),
+ expected: true,
+ target: self.new_block,
+ msg: Box::new(AssertKind::OccupiedNiche {
+ found: Operand::Copy(self.value_in_niche),
+ start,
+ end,
+ }),
+ // This calls panic_occupied_niche, which is #[rustc_nounwind].
+ // We never want to insert an unwind into unsafe code, because unwinding could
+ // make a failing UB check turn into much worse UB when we start unwinding.
+ unwind: UnwindAction::Unreachable,
+ },
+ });
+ }
+
+ fn niche_const(&self, val: u128) -> Operand<'tcx> {
+ let niche_ty = self.niche.ty(self.tcx);
+ if niche_ty.is_any_ptr() {
+ Operand::Constant(Box::new(ConstOperand {
+ span: self.source_info.span,
+ user_ty: None,
+ const_: Const::Val(
+ ConstValue::Scalar(Scalar::from_maybe_pointer(
+ Pointer::from_addr_invalid(val as u64),
+ &self.tcx,
+ )),
+ niche_ty,
+ ),
+ }))
+ } else {
+ Operand::Constant(Box::new(ConstOperand {
+ span: self.source_info.span,
+ user_ty: None,
+ const_: Const::Val(
+ ConstValue::Scalar(Scalar::from_uint(val, self.niche.size(self.tcx))),
+ niche_ty,
+ ),
+ }))
+ }
+ }
+
+ fn add_assignment(&mut self, place: Place<'tcx>, rvalue: Rvalue<'tcx>) {
+ self.block_data.statements.push(Statement {
+ source_info: self.source_info,
+ kind: StatementKind::Assign(Box::new((place, rvalue))),
+ });
+ }
+
+ fn check_start_only(&mut self) {
+ let rvalue = if self.niche.valid_range.start == 1 {
+ let bound = self.niche_const(0);
+ Rvalue::BinaryOp(BinOp::Ne, Box::new((Operand::Copy(self.value_in_niche), bound)))
+ } else {
+ let bound = self.niche_const(self.niche.valid_range.start);
+ Rvalue::BinaryOp(BinOp::Ge, Box::new((Operand::Copy(self.value_in_niche), bound)))
+ };
+ self.add_assignment(self.is_in_range, rvalue);
+ }
+
+ fn check_end_only(&mut self) {
+ let end = self.niche_const(self.niche.valid_range.end);
+
+ let rvalue =
+ Rvalue::BinaryOp(BinOp::Le, Box::new((Operand::Copy(self.value_in_niche), end)));
+ self.add_assignment(self.is_in_range, rvalue);
+ }
+
+ fn general_case(&mut self) {
+ let mut max = self.niche.valid_range.end.wrapping_sub(self.niche.valid_range.start);
+ let size = self.niche.size(self.tcx);
+ if size.bits() < 128 {
+ let mask = (1 << size.bits()) - 1;
+ max &= mask;
+ }
+
+ let start = self.niche_const(self.niche.valid_range.start);
+ let max_adjusted_allowed_value = self.niche_const(max);
+
+ let biased = self
+ .local_decls
+ .push(LocalDecl::with_source_info(self.niche.ty(self.tcx), self.source_info))
+ .into();
+ let rvalue =
+ Rvalue::BinaryOp(BinOp::Sub, Box::new((Operand::Copy(self.value_in_niche), start)));
+ self.add_assignment(biased, rvalue);
+
+ let rvalue = Rvalue::BinaryOp(
+ BinOp::Le,
+ Box::new((Operand::Copy(biased), max_adjusted_allowed_value)),
+ );
+ self.add_assignment(self.is_in_range, rvalue);
+ }
+}
+
+trait NicheExt {
+ fn ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx>;
+ fn size<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Size;
+}
+
+impl NicheExt for Niche {
+ fn ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
+ let types = &tcx.types;
+ match self.value {
+ Primitive::Int(Integer::I8, _) => types.u8,
+ Primitive::Int(Integer::I16, _) => types.u16,
+ Primitive::Int(Integer::I32, _) => types.u32,
+ Primitive::Int(Integer::I64, _) => types.u64,
+ Primitive::Int(Integer::I128, _) => types.u128,
+ Primitive::Pointer(_) => {
+ Ty::new_ptr(tcx, TypeAndMut { ty: types.unit, mutbl: Mutability::Not })
+ }
+ Primitive::F32 => types.u32,
+ Primitive::F64 => types.u64,
+ }
+ }
+
+ fn size<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Size {
+ let bits = match self.value {
+ Primitive::Int(Integer::I8, _) => 8,
+ Primitive::Int(Integer::I16, _) => 16,
+ Primitive::Int(Integer::I32, _) => 32,
+ Primitive::Int(Integer::I64, _) => 64,
+ Primitive::Int(Integer::I128, _) => 128,
+ Primitive::Pointer(_) => tcx.sess.target.pointer_width as usize,
+ Primitive::F32 => 32,
+ Primitive::F64 => 64,
+ };
+ Size::from_bits(bits)
+ }
+}
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index bf5f0ca7cbd23..8a3f0fb6fbbf9 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -52,6 +52,7 @@ mod add_call_guards;
mod add_moves_for_packed_drops;
mod add_retag;
mod check_const_item_mutation;
+mod check_niches;
mod check_packed_ref;
pub mod check_unsafety;
mod remove_place_mention;
@@ -321,6 +322,7 @@ fn mir_promoted(
tcx: TyCtxt<'_>,
def: LocalDefId,
) -> (&Steal<Body<'_>>, &Steal<IndexVec<Promoted, Body<'_>>>) {
+ tcx.ensure_with_value().unsafety_check_result(def);
// Ensure that we compute the `mir_const_qualif` for constants at
// this point, before we steal the mir-const result.
// Also this means promotion can rely on all const checks having been done.
@@ -567,6 +569,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
body,
&[
&check_alignment::CheckAlignment,
+ &check_niches::CheckNiches,
&lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first
&inline::Inline,
// Substitutions during inlining may introduce switch on enums with uninhabited branches.
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 65bdcf1076225..f22910a812269 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -844,6 +844,20 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
mir::TerminatorKind::Assert { ref msg, .. } => {
let lang_item = match &**msg {
mir::AssertKind::BoundsCheck { .. } => LangItem::PanicBoundsCheck,
+ mir::AssertKind::MisalignedPointerDereference { .. } => {
+ LangItem::PanicMisalignedPointerDereference
+ }
+ mir::AssertKind::OccupiedNiche { found, .. } => {
+ let lang_item = LangItem::PanicOccupiedNiche;
+ let instance = Instance::new(
+ tcx.require_lang_item(lang_item, Some(source)),
+ tcx.mk_args(&[ty::GenericArg::from(found.ty(self.body, tcx))]),
+ );
+ if should_codegen_locally(tcx, &instance) {
+ self.output.push(create_fn_mono_item(tcx, instance, source));
+ }
+ LangItem::Panic
+ }
_ => LangItem::Panic,
};
push_mono_lang_item(self, lang_item);
diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
index 27596c08f1c1e..71ad5cefd19b8 100644
--- a/compiler/rustc_smir/src/rustc_smir/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -812,6 +812,13 @@ impl<'tcx> Stable<'tcx> for mir::AssertMessage<'tcx> {
found: found.stable(tables),
}
}
+ AssertKind::OccupiedNiche { found, start, end } => {
+ stable_mir::mir::AssertMessage::OccupiedNiche {
+ found: found.stable(tables),
+ start: start.stable(tables),
+ end: end.stable(tables),
+ }
+ }
}
}
}
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index f287862cc23c5..47d07617ba20f 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1157,6 +1157,7 @@ symbols! {
panic_location,
panic_misaligned_pointer_dereference,
panic_nounwind,
+ panic_occupied_niche,
panic_runtime,
panic_str,
panic_unwind,
diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index 0693378368534..d9f1d04b4d1d6 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -149,6 +149,7 @@ pub enum AssertMessage {
ResumedAfterReturn(CoroutineKind),
ResumedAfterPanic(CoroutineKind),
MisalignedPointerDereference { required: Operand, found: Operand },
+ OccupiedNiche { found: Operand, start: Operand, end: Operand },
}
#[derive(Clone, Debug, Eq, PartialEq)]
diff --git a/compiler/stable_mir/src/mir/visit.rs b/compiler/stable_mir/src/mir/visit.rs
index 806dced71ff3e..636108c739cd5 100644
--- a/compiler/stable_mir/src/mir/visit.rs
+++ b/compiler/stable_mir/src/mir/visit.rs
@@ -372,6 +372,11 @@ pub trait MirVisitor {
self.visit_operand(required, location);
self.visit_operand(found, location);
}
+ AssertMessage::OccupiedNiche { found, start, end } => {
+ self.visit_operand(found, location);
+ self.visit_operand(start, location);
+ self.visit_operand(end, location);
+ }
}
}
}
diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs
index 39a5e8d9fe2ec..687ee8ce0c18f 100644
--- a/library/core/src/panicking.rs
+++ b/library/core/src/panicking.rs
@@ -208,6 +208,22 @@ fn panic_misaligned_pointer_dereference(required: usize, found: usize) -> ! {
)
}
+#[cold]
+#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
+#[track_caller]
+#[cfg_attr(not(bootstrap), lang = "panic_occupied_niche")] // needed by codegen for panic on occupied niches
+#[rustc_nounwind] // `CheckNiches` MIR pass requires this function to never unwind
+fn panic_occupied_niche<T: crate::fmt::Debug>(found: T, min: T, max: T) -> ! {
+ if cfg!(feature = "panic_immediate_abort") {
+ super::intrinsics::abort()
+ }
+
+ panic_nounwind_fmt(
+ format_args!("occupied niche: found {found:?} but must be in {min:?}..={max:?}"),
+ /* force_no_backtrace */ false,
+ )
+}
+
/// Panic because we cannot unwind out of a function.
///
/// This is a separate function to avoid the codesize impact of each crate containing the string to
diff --git a/src/tools/miri/src/lib.rs b/src/tools/miri/src/lib.rs
index b12aae6d4148c..44f5c99ba9128 100644
--- a/src/tools/miri/src/lib.rs
+++ b/src/tools/miri/src/lib.rs
@@ -139,5 +139,5 @@ pub const MIRI_DEFAULT_ARGS: &[&str] = &[
"-Zmir-emit-retag",
"-Zmir-keep-place-mention",
"-Zmir-opt-level=0",
- "-Zmir-enable-passes=-CheckAlignment",
+ "-Zmir-enable-passes=-CheckAlignment,-CheckNiches",
];
diff --git a/tests/codegen/array-equality.rs b/tests/codegen/array-equality.rs
index 1941452ea6159..0edd45ecf8d90 100644
--- a/tests/codegen/array-equality.rs
+++ b/tests/codegen/array-equality.rs
@@ -1,5 +1,6 @@
// compile-flags: -O -Z merge-functions=disabled
// only-x86_64
+// ignore-debug: array comparison sometimes transmutes references, so we have niche checks in std
#![crate_type = "lib"]
diff --git a/tests/codegen/debug-vtable.rs b/tests/codegen/debug-vtable.rs
index e52392b260bd4..8bcb28735ade0 100644
--- a/tests/codegen/debug-vtable.rs
+++ b/tests/codegen/debug-vtable.rs
@@ -6,7 +6,7 @@
// legacy mangling scheme rustc version and generic parameters are both hashed into a single part
// of the name, thus randomizing item order with respect to rustc version.
-// compile-flags: -Cdebuginfo=2 -Copt-level=0 -Csymbol-mangling-version=v0
+// compile-flags: -Cdebuginfo=2 -Copt-level=0 -Csymbol-mangling-version=v0 -Zmir-enable-passes=-CheckNiches
// ignore-tidy-linelength
// Make sure that vtables don't have the unnamed_addr attribute when debuginfo is enabled.
diff --git a/tests/codegen/intrinsics/transmute-niched.rs b/tests/codegen/intrinsics/transmute-niched.rs
index e9c8d803cb9e9..86acb47bc8266 100644
--- a/tests/codegen/intrinsics/transmute-niched.rs
+++ b/tests/codegen/intrinsics/transmute-niched.rs
@@ -1,6 +1,6 @@
// revisions: OPT DBG
// [OPT] compile-flags: -C opt-level=3 -C no-prepopulate-passes
-// [DBG] compile-flags: -C opt-level=0 -C no-prepopulate-passes
+// [DBG] compile-flags: -C opt-level=0 -C no-prepopulate-passes -Zmir-enable-passes=-CheckNiches
// only-64bit (so I don't need to worry about usize)
#![crate_type = "lib"]
diff --git a/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs b/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs
index a3cd16e3dd5a5..e53ea03574319 100644
--- a/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs
+++ b/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs
@@ -1,7 +1,7 @@
// Verifies that pointer type membership tests for indirect calls are omitted.
//
// needs-sanitizer-cfi
-// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -Zmir-enable-passes=-CheckNiches
#![crate_type="lib"]
#![feature(no_sanitize)]
diff --git a/tests/codegen/sanitizer/cfi-emit-type-checks.rs b/tests/codegen/sanitizer/cfi-emit-type-checks.rs
index f0fe5de9f66c1..567597393f774 100644
--- a/tests/codegen/sanitizer/cfi-emit-type-checks.rs
+++ b/tests/codegen/sanitizer/cfi-emit-type-checks.rs
@@ -1,7 +1,7 @@
// Verifies that pointer type membership tests for indirect calls are emitted.
//
// needs-sanitizer-cfi
-// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -Zmir-enable-passes=-CheckNiches
#![crate_type="lib"]
diff --git a/tests/codegen/thread-local.rs b/tests/codegen/thread-local.rs
index caf0366d2c144..383014d0c3132 100644
--- a/tests/codegen/thread-local.rs
+++ b/tests/codegen/thread-local.rs
@@ -1,4 +1,5 @@
// compile-flags: -O
+// ignore-debug: niche checks in std interfere with the codegen we are looking for
// aux-build:thread_local_aux.rs
// ignore-windows FIXME(#84933)
// ignore-wasm globals are used instead of thread locals
diff --git a/tests/codegen/transmute-scalar.rs b/tests/codegen/transmute-scalar.rs
index 39126b024a6e9..8b29a06f66969 100644
--- a/tests/codegen/transmute-scalar.rs
+++ b/tests/codegen/transmute-scalar.rs
@@ -1,4 +1,4 @@
-// compile-flags: -C opt-level=0 -C no-prepopulate-passes
+// compile-flags: -C opt-level=0 -C no-prepopulate-passes -Zmir-enable-passes=-CheckNiches
#![crate_type = "lib"]
diff --git a/tests/mir-opt/remove_storage_markers.rs b/tests/mir-opt/remove_storage_markers.rs
index 6666ff3b7263b..a1a4c15c2bd53 100644
--- a/tests/mir-opt/remove_storage_markers.rs
+++ b/tests/mir-opt/remove_storage_markers.rs
@@ -4,7 +4,7 @@
// Checks that storage markers are removed at opt-level=0.
//
-// compile-flags: -C opt-level=0 -Coverflow-checks=off
+// compile-flags: -C opt-level=0 -Cdebug-assertions=off
// EMIT_MIR remove_storage_markers.main.RemoveStorageMarkers.diff
fn main() {
diff --git a/tests/ui-fulldeps/stable-mir/crate-info.rs b/tests/ui-fulldeps/stable-mir/crate-info.rs
index ed6b786f5e1de..692716e337fbc 100644
--- a/tests/ui-fulldeps/stable-mir/crate-info.rs
+++ b/tests/ui-fulldeps/stable-mir/crate-info.rs
@@ -186,6 +186,7 @@ fn main() {
let args = vec![
"rustc".to_string(),
"--crate-type=lib".to_string(),
+ "-Zmir-enable-passes=-CheckNiches".to_string(),
"--crate-name".to_string(),
CRATE_NAME.to_string(),
path.to_string(),
diff --git a/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs b/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs
index 33113642023a2..99282829dbf29 100644
--- a/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs
+++ b/tests/ui/lint/large_assignments/box_rc_arc_allowed.rs
@@ -5,7 +5,7 @@
// only-x86_64
// edition:2018
-// compile-flags: -Zmir-opt-level=0
+// compile-flags: -Zmir-opt-level=0 -Zmir-enable-passes=-CheckNiches
use std::{sync::Arc, rc::Rc};
diff --git a/tests/ui/mir/check_niches/invalid_enums.rs b/tests/ui/mir/check_niches/invalid_enums.rs
new file mode 100644
index 0000000000000..a5117f8ba27a2
--- /dev/null
+++ b/tests/ui/mir/check_niches/invalid_enums.rs
@@ -0,0 +1,23 @@
+// run-fail
+// ignore-wasm32-bare: No panic messages
+// compile-flags: -C debug-assertions -Zmir-opt-level=0
+
+#[repr(C)]
+struct Thing {
+ x: usize,
+ y: Contents,
+ z: usize,
+}
+
+#[repr(usize)]
+enum Contents {
+ A = 8usize,
+ B = 9usize,
+ C = 10usize,
+}
+
+fn main() {
+ unsafe {
+ let _thing = std::mem::transmute::<(usize, usize, usize), Thing>((0, 3, 0));
+ }
+}
diff --git a/tests/ui/mir/check_niches/invalid_nonnull_transmute.rs b/tests/ui/mir/check_niches/invalid_nonnull_transmute.rs
new file mode 100644
index 0000000000000..99ed52b81f8d4
--- /dev/null
+++ b/tests/ui/mir/check_niches/invalid_nonnull_transmute.rs
@@ -0,0 +1,10 @@
+// run-fail
+// ignore-wasm32-bare: No panic messages
+// compile-flags: -C debug-assertions -Zmir-opt-level=0
+// error-pattern: occupied niche: found 0x0 but must be in 0x1..=0xffffffff
+
+fn main() {
+ unsafe {
+ std::mem::transmute::<*const u8, std::ptr::NonNull<u8>>(std::ptr::null());
+ }
+}
diff --git a/tests/ui/mir/check_niches/invalid_nonzero_argument.rs b/tests/ui/mir/check_niches/invalid_nonzero_argument.rs
new file mode 100644
index 0000000000000..a5d03013e5ff6
--- /dev/null
+++ b/tests/ui/mir/check_niches/invalid_nonzero_argument.rs
@@ -0,0 +1,14 @@
+// run-fail
+// ignore-wasm32-bare: No panic messages
+// compile-flags: -C debug-assertions -Zmir-opt-level=0
+// error-pattern: occupied niche: found 0 but must be in 1..=255
+
+fn main() {
+ let mut bad = std::num::NonZeroU8::new(1u8).unwrap();
+ unsafe {
+ std::ptr::write_bytes(&mut bad, 0u8, 1usize);
+ }
+ func(bad);
+}
+
+fn func<T>(_t: T) {}
diff --git a/tests/ui/mir/check_niches/valid_nonzero_argument.rs b/tests/ui/mir/check_niches/valid_nonzero_argument.rs
new file mode 100644
index 0000000000000..727d3e0d67c53
--- /dev/null
+++ b/tests/ui/mir/check_niches/valid_nonzero_argument.rs
@@ -0,0 +1,13 @@
+// run-pass
+// compile-flags: -C debug-assertions -Zmir-opt-level=0
+
+fn main() {
+ for val in i8::MIN..=i8::MAX {
+ if val != 0 {
+ let x = std::num::NonZeroI8::new(val).unwrap();
+ if val != i8::MIN {
+ let _y = -x;
+ }
+ }
+ }
+}
diff --git a/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs b/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs
index cad05917391be..7557e5ce34bb0 100644
--- a/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs
+++ b/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs
@@ -1,6 +1,6 @@
// run-pass
// only-wasm32
-// compile-flags: -Zmir-opt-level=0 -C target-feature=+nontrapping-fptoint
+// compile-flags: -Zmir-opt-level=0 -C target-feature=+nontrapping-fptoint -Zmir-enable-passes=-CheckNiches
#![feature(test, stmt_expr_attributes)]
#![deny(overflowing_literals)]
diff --git a/tests/ui/numbers-arithmetic/saturating-float-casts.rs b/tests/ui/numbers-arithmetic/saturating-float-casts.rs
index cc248a9bea087..d26273bc8345f 100644
--- a/tests/ui/numbers-arithmetic/saturating-float-casts.rs
+++ b/tests/ui/numbers-arithmetic/saturating-float-casts.rs
@@ -1,5 +1,5 @@
// run-pass
-// compile-flags:-Zmir-opt-level=0
+// compile-flags:-Zmir-opt-level=0 -Zmir-enable-passes=-CheckNiches
#![feature(test, stmt_expr_attributes)]
#![deny(overflowing_literals)]
diff --git a/tests/ui/print_type_sizes/niche-filling.rs b/tests/ui/print_type_sizes/niche-filling.rs
index 5e620f248b65d..82afbd2495706 100644
--- a/tests/ui/print_type_sizes/niche-filling.rs
+++ b/tests/ui/print_type_sizes/niche-filling.rs
@@ -1,4 +1,4 @@
-// compile-flags: -Z print-type-sizes --crate-type=lib
+// compile-flags: -Z print-type-sizes --crate-type=lib -Zmir-enable-passes=-CheckNiches
// build-pass
// ignore-pass
// ^-- needed because `--pass check` does not emit the output needed.
diff --git a/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs b/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs
index 8dadd77fc16d5..9091b3ba6b8aa 100644
--- a/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs
+++ b/tests/ui/type-alias-enum-variants/self-in-enum-definition.rs
@@ -1,3 +1,5 @@
+// compile-flags: -Cdebug-assertions=no
+
#[repr(u8)]
enum Alpha {
V1 = 41,
diff --git a/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr b/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr
index aa79b1a57c4c8..5c9d75fffdc50 100644
--- a/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr
+++ b/tests/ui/type-alias-enum-variants/self-in-enum-definition.stderr
@@ -1,63 +1,53 @@
error[E0391]: cycle detected when simplifying constant for the type system `Alpha::V3::{constant#0}`
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
|
note: ...which requires simplifying constant for the type system `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
note: ...which requires const-evaluating + checking `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
note: ...which requires caching mir of `Alpha::V3::{constant#0}` for CTFE...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
note: ...which requires elaborating drops for `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
note: ...which requires borrow-checking `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
note: ...which requires promoting constants in MIR for `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
- |
-LL | V3 = Self::V1 {} as u8 + 2,
- | ^^^^^^^^^^^^^^^^^^^^^
-note: ...which requires const checking `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
- |
-LL | V3 = Self::V1 {} as u8 + 2,
- | ^^^^^^^^^^^^^^^^^^^^^
-note: ...which requires preparing `Alpha::V3::{constant#0}` for borrow checking...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
note: ...which requires unsafety-checking `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
note: ...which requires building MIR for `Alpha::V3::{constant#0}`...
- --> $DIR/self-in-enum-definition.rs:5:10
+ --> $DIR/self-in-enum-definition.rs:7:10
|
LL | V3 = Self::V1 {} as u8 + 2,
| ^^^^^^^^^^^^^^^^^^^^^
= note: ...which requires computing layout of `Alpha`...
= note: ...which again requires simplifying constant for the type system `Alpha::V3::{constant#0}`, completing the cycle
note: cycle used when collecting item types in top-level module
- --> $DIR/self-in-enum-definition.rs:1:1
+ --> $DIR/self-in-enum-definition.rs:3:1
|
LL | / #[repr(u8)]
LL | | enum Alpha {