Skip to content

Commit 361c4ea

Browse files
borsbors
committed
Auto merge of #79049 - tmiasko:lower-intrinsics, r=jonas-schievink
Lower intrinsics calls: forget, size_of, unreachable, wrapping_* This allows constant propagation to evaluate `size_of` and `wrapping_*`, and unreachable propagation to propagate a call to `unreachable`. The lowering is performed as a MIR optimization, rather than during MIR building to preserve the special status of intrinsics with respect to unsafety checks and promotion. Currently enabled by default to determine the performance impact (no significant impact expected). In practice only useful when combined with inlining since intrinsics are rarely used directly (with exception of `unreachable` and `discriminant_value` used by built-in derive macros). Closes #32716.
2 parents 98d6634 + 6903273 commit 361c4ea

10 files changed

+373
-5
lines changed

‎compiler/rustc_mir/src/transform/lower_intrinsics.rs

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
//! Lowers intrinsic calls
2+
3+
use crate::transform::MirPass;
4+
use rustc_middle::mir::*;
5+
use rustc_middle::ty::subst::SubstsRef;
6+
use rustc_middle::ty::{self, Ty, TyCtxt};
7+
use rustc_span::symbol::{sym, Symbol};
8+
use rustc_target::spec::abi::Abi;
9+
10+
pub struct LowerIntrinsics;
11+
12+
impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
13+
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14+
for block in body.basic_blocks_mut() {
15+
let terminator = block.terminator.as_mut().unwrap();
16+
if let TerminatorKind::Call {
17+
func: Operand::Constant(box Constant { literal: ty::Const { ty: func_ty, .. }, .. }),
18+
args,
19+
destination,
20+
..
21+
} = &mut terminator.kind
22+
{
23+
let (intrinsic_name, substs) = match resolve_rust_intrinsic(tcx, func_ty) {
24+
None => continue,
25+
Some(it) => it,
26+
};
27+
match intrinsic_name {
28+
sym::unreachable => {
29+
terminator.kind = TerminatorKind::Unreachable;
30+
}
31+
sym::forget => {
32+
if let Some((destination, target)) = *destination {
33+
block.statements.push(Statement {
34+
source_info: terminator.source_info,
35+
kind: StatementKind::Assign(box (
36+
destination,
37+
Rvalue::Use(Operand::Constant(box Constant {
38+
span: terminator.source_info.span,
39+
user_ty: None,
40+
literal: ty::Const::zero_sized(tcx, tcx.types.unit),
41+
})),
42+
)),
43+
});
44+
terminator.kind = TerminatorKind::Goto { target };
45+
}
46+
}
47+
sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
48+
if let Some((destination, target)) = *destination {
49+
let lhs;
50+
let rhs;
51+
{
52+
let mut args = args.drain(..);
53+
lhs = args.next().unwrap();
54+
rhs = args.next().unwrap();
55+
}
56+
let bin_op = match intrinsic_name {
57+
sym::wrapping_add => BinOp::Add,
58+
sym::wrapping_sub => BinOp::Sub,
59+
sym::wrapping_mul => BinOp::Mul,
60+
_ => bug!("unexpected intrinsic"),
61+
};
62+
block.statements.push(Statement {
63+
source_info: terminator.source_info,
64+
kind: StatementKind::Assign(box (
65+
destination,
66+
Rvalue::BinaryOp(bin_op, lhs, rhs),
67+
)),
68+
});
69+
terminator.kind = TerminatorKind::Goto { target };
70+
}
71+
}
72+
sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
73+
// The checked binary operations are not suitable target for lowering here,
74+
// since their semantics depend on the value of overflow-checks flag used
75+
// during codegen. Issue #35310.
76+
}
77+
sym::size_of => {
78+
if let Some((destination, target)) = *destination {
79+
let tp_ty = substs.type_at(0);
80+
block.statements.push(Statement {
81+
source_info: terminator.source_info,
82+
kind: StatementKind::Assign(box (
83+
destination,
84+
Rvalue::NullaryOp(NullOp::SizeOf, tp_ty),
85+
)),
86+
});
87+
terminator.kind = TerminatorKind::Goto { target };
88+
}
89+
}
90+
_ => {}
91+
}
92+
}
93+
}
94+
}
95+
}
96+
97+
fn resolve_rust_intrinsic(
98+
tcx: TyCtxt<'tcx>,
99+
func_ty: Ty<'tcx>,
100+
) -> Option<(Symbol, SubstsRef<'tcx>)> {
101+
if let ty::FnDef(def_id, substs) = *func_ty.kind() {
102+
let fn_sig = func_ty.fn_sig(tcx);
103+
if fn_sig.abi() == Abi::RustIntrinsic {
104+
return Some((tcx.item_name(def_id), substs));
105+
}
106+
}
107+
None
108+
}

‎compiler/rustc_mir/src/transform/mod.rs

+2
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ pub mod function_item_references;
3232
pub mod generator;
3333
pub mod inline;
3434
pub mod instcombine;
35+
pub mod lower_intrinsics;
3536
pub mod match_branches;
3637
pub mod multiple_return_terminators;
3738
pub mod no_landing_pads;
@@ -390,6 +391,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
390391

391392
// The main optimizations that we do on MIR.
392393
let optimizations: &[&dyn MirPass<'tcx>] = &[
394+
&lower_intrinsics::LowerIntrinsics,
393395
&remove_unneeded_drops::RemoveUnneededDrops,
394396
&match_branches::MatchBranchSimplification,
395397
// inst combine is after MatchBranchSimplification to clean up Ne(_1, false)

‎src/test/mir-opt/lower_intrinsics.f_u64.PreCodegen.before.mir

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// MIR for `f_u64` before PreCodegen
2+
3+
fn f_u64() -> () {
4+
let mut _0: (); // return place in scope 0 at $DIR/lower_intrinsics.rs:34:16: 34:16
5+
scope 1 (inlined f_dispatch::<u64>) { // at $DIR/lower_intrinsics.rs:35:5: 35:21
6+
debug t => _2; // in scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
7+
let _1: (); // in scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
8+
let mut _2: u64; // in scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
9+
scope 2 (inlined std::mem::size_of::<u64>) { // at $DIR/lower_intrinsics.rs:35:5: 35:21
10+
}
11+
}
12+
13+
bb0: {
14+
_2 = const 0_u64; // scope 0 at $DIR/lower_intrinsics.rs:35:5: 35:21
15+
StorageLive(_1); // scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
16+
_1 = f_non_zst::<u64>(move _2) -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
17+
// mir::Constant
18+
// + span: $DIR/lower_intrinsics.rs:35:5: 35:21
19+
// + literal: Const { ty: fn(u64) {f_non_zst::<u64>}, val: Value(Scalar(<ZST>)) }
20+
}
21+
22+
bb1: {
23+
StorageDead(_1); // scope 1 at $DIR/lower_intrinsics.rs:35:5: 35:21
24+
_0 = const (); // scope 0 at $DIR/lower_intrinsics.rs:34:16: 36:2
25+
return; // scope 0 at $DIR/lower_intrinsics.rs:36:2: 36:2
26+
}
27+
}

‎src/test/mir-opt/lower_intrinsics.f_unit.PreCodegen.before.mir

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// MIR for `f_unit` before PreCodegen
2+
3+
fn f_unit() -> () {
4+
let mut _0: (); // return place in scope 0 at $DIR/lower_intrinsics.rs:28:17: 28:17
5+
let mut _1: (); // in scope 0 at $DIR/lower_intrinsics.rs:29:16: 29:18
6+
scope 1 (inlined f_dispatch::<()>) { // at $DIR/lower_intrinsics.rs:29:5: 29:19
7+
debug t => _1; // in scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
8+
let _2: (); // in scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
9+
scope 2 (inlined std::mem::size_of::<()>) { // at $DIR/lower_intrinsics.rs:29:5: 29:19
10+
}
11+
}
12+
13+
bb0: {
14+
StorageLive(_1); // scope 0 at $DIR/lower_intrinsics.rs:29:16: 29:18
15+
StorageLive(_2); // scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
16+
_2 = f_zst::<()>(const ()) -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
17+
// mir::Constant
18+
// + span: $DIR/lower_intrinsics.rs:29:5: 29:19
19+
// + literal: Const { ty: fn(()) {f_zst::<()>}, val: Value(Scalar(<ZST>)) }
20+
}
21+
22+
bb1: {
23+
StorageDead(_2); // scope 1 at $DIR/lower_intrinsics.rs:29:5: 29:19
24+
StorageDead(_1); // scope 0 at $DIR/lower_intrinsics.rs:29:18: 29:19
25+
_0 = const (); // scope 0 at $DIR/lower_intrinsics.rs:28:17: 30:2
26+
return; // scope 0 at $DIR/lower_intrinsics.rs:30:2: 30:2
27+
}
28+
}

‎src/test/mir-opt/lower_intrinsics.forget.LowerIntrinsics.diff

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
- // MIR for `forget` before LowerIntrinsics
2+
+ // MIR for `forget` after LowerIntrinsics
3+
4+
fn forget(_1: T) -> () {
5+
debug t => _1; // in scope 0 at $DIR/lower_intrinsics.rs:18:18: 18:19
6+
let mut _0: (); // return place in scope 0 at $DIR/lower_intrinsics.rs:18:24: 18:24
7+
let _2: (); // in scope 0 at $DIR/lower_intrinsics.rs:19:14: 19:41
8+
let mut _3: T; // in scope 0 at $DIR/lower_intrinsics.rs:19:39: 19:40
9+
scope 1 {
10+
}
11+
12+
bb0: {
13+
StorageLive(_2); // scope 0 at $DIR/lower_intrinsics.rs:19:5: 19:43
14+
StorageLive(_3); // scope 1 at $DIR/lower_intrinsics.rs:19:39: 19:40
15+
_3 = move _1; // scope 1 at $DIR/lower_intrinsics.rs:19:39: 19:40
16+
- _2 = std::intrinsics::forget::<T>(move _3) -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:19:14: 19:41
17+
- // mir::Constant
18+
- // + span: $DIR/lower_intrinsics.rs:19:14: 19:38
19+
- // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(T) {std::intrinsics::forget::<T>}, val: Value(Scalar(<ZST>)) }
20+
+ _2 = const (); // scope 1 at $DIR/lower_intrinsics.rs:19:14: 19:41
21+
+ goto -> bb1; // scope 1 at $DIR/lower_intrinsics.rs:19:14: 19:41
22+
}
23+
24+
bb1: {
25+
StorageDead(_3); // scope 1 at $DIR/lower_intrinsics.rs:19:40: 19:41
26+
StorageDead(_2); // scope 0 at $DIR/lower_intrinsics.rs:19:43: 19:44
27+
_0 = const (); // scope 0 at $DIR/lower_intrinsics.rs:18:24: 20:2
28+
return; // scope 0 at $DIR/lower_intrinsics.rs:20:2: 20:2
29+
}
30+
}
31+

‎src/test/mir-opt/lower_intrinsics.rs

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// compile-flags: -Cpanic=abort
2+
#![feature(core_intrinsics)]
3+
#![crate_type = "lib"]
4+
5+
// EMIT_MIR lower_intrinsics.wrapping.LowerIntrinsics.diff
6+
pub fn wrapping<T: Copy>(a: T, b: T) {
7+
let _x = core::intrinsics::wrapping_add(a, b);
8+
let _y = core::intrinsics::wrapping_sub(a, b);
9+
let _z = core::intrinsics::wrapping_mul(a, b);
10+
}
11+
12+
// EMIT_MIR lower_intrinsics.size_of.LowerIntrinsics.diff
13+
pub fn size_of<T>() -> usize {
14+
core::intrinsics::size_of::<T>()
15+
}
16+
17+
// EMIT_MIR lower_intrinsics.forget.LowerIntrinsics.diff
18+
pub fn forget<T>(t: T) {
19+
unsafe { core::intrinsics::forget(t) };
20+
}
21+
22+
// EMIT_MIR lower_intrinsics.unreachable.LowerIntrinsics.diff
23+
pub fn unreachable() -> ! {
24+
unsafe { core::intrinsics::unreachable() };
25+
}
26+
27+
// EMIT_MIR lower_intrinsics.f_unit.PreCodegen.before.mir
28+
pub fn f_unit() {
29+
f_dispatch(());
30+
}
31+
32+
33+
// EMIT_MIR lower_intrinsics.f_u64.PreCodegen.before.mir
34+
pub fn f_u64() {
35+
f_dispatch(0u64);
36+
}
37+
38+
#[inline(always)]
39+
pub fn f_dispatch<T>(t: T) {
40+
if std::mem::size_of::<T>() == 0 {
41+
f_zst(t);
42+
} else {
43+
f_non_zst(t);
44+
}
45+
}
46+
47+
#[inline(never)]
48+
pub fn f_zst<T>(t: T) {
49+
}
50+
51+
#[inline(never)]
52+
pub fn f_non_zst<T>(t: T) {}

‎src/test/mir-opt/lower_intrinsics.size_of.LowerIntrinsics.diff

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
- // MIR for `size_of` before LowerIntrinsics
2+
+ // MIR for `size_of` after LowerIntrinsics
3+
4+
fn size_of() -> usize {
5+
let mut _0: usize; // return place in scope 0 at $DIR/lower_intrinsics.rs:13:24: 13:29
6+
7+
bb0: {
8+
- _0 = std::intrinsics::size_of::<T>() -> bb1; // scope 0 at $DIR/lower_intrinsics.rs:14:5: 14:37
9+
- // mir::Constant
10+
- // + span: $DIR/lower_intrinsics.rs:14:5: 14:35
11+
- // + literal: Const { ty: extern "rust-intrinsic" fn() -> usize {std::intrinsics::size_of::<T>}, val: Value(Scalar(<ZST>)) }
12+
+ _0 = SizeOf(T); // scope 0 at $DIR/lower_intrinsics.rs:14:5: 14:37
13+
+ goto -> bb1; // scope 0 at $DIR/lower_intrinsics.rs:14:5: 14:37
14+
}
15+
16+
bb1: {
17+
return; // scope 0 at $DIR/lower_intrinsics.rs:15:2: 15:2
18+
}
19+
}
20+

‎src/test/mir-opt/lower_intrinsics.unreachable.LowerIntrinsics.diff

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
- // MIR for `unreachable` before LowerIntrinsics
2+
+ // MIR for `unreachable` after LowerIntrinsics
3+
4+
fn unreachable() -> ! {
5+
let mut _0: !; // return place in scope 0 at $DIR/lower_intrinsics.rs:23:25: 23:26
6+
let mut _1: !; // in scope 0 at $DIR/lower_intrinsics.rs:23:27: 25:2
7+
let _2: (); // in scope 0 at $DIR/lower_intrinsics.rs:24:14: 24:45
8+
let mut _3: !; // in scope 0 at $DIR/lower_intrinsics.rs:24:14: 24:45
9+
scope 1 {
10+
}
11+
12+
bb0: {
13+
StorageLive(_2); // scope 0 at $DIR/lower_intrinsics.rs:24:5: 24:47
14+
StorageLive(_3); // scope 1 at $DIR/lower_intrinsics.rs:24:14: 24:45
15+
- std::intrinsics::unreachable(); // scope 1 at $DIR/lower_intrinsics.rs:24:14: 24:45
16+
- // mir::Constant
17+
- // + span: $DIR/lower_intrinsics.rs:24:14: 24:43
18+
- // + literal: Const { ty: unsafe extern "rust-intrinsic" fn() -> ! {std::intrinsics::unreachable}, val: Value(Scalar(<ZST>)) }
19+
+ unreachable; // scope 1 at $DIR/lower_intrinsics.rs:24:14: 24:45
20+
}
21+
}
22+

0 commit comments

Comments
 (0)