Skip to content

Commit 3f104b1

Browse files
scottmcmscottmcm
committed
Inline more when we do inline
To offset the less inlining in simple forwarding functions.
1 parent 251a69c commit 3f104b1

19 files changed

+2465
-349
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,7 @@ impl<'tcx> Inliner<'tcx> {
486486
let mut threshold = if self.caller_is_inline_forwarder {
487487
self.tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
488488
} else if cross_crate_inlinable {
489-
self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
489+
self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(150)
490490
} else {
491491
self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
492492
};

‎compiler/rustc_session/src/options.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1769,7 +1769,7 @@ options! {
17691769
inline_mir_forwarder_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
17701770
"inlining threshold when the caller is a simple forwarding function (default: 30)"),
17711771
inline_mir_hint_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
1772-
"inlining threshold for functions with inline hint (default: 100)"),
1772+
"inlining threshold for functions with inline hint (default: 150)"),
17731773
inline_mir_preserve_debug: Option<bool> = (None, parse_opt_bool, [TRACKED],
17741774
"when MIR inlining, whether to preserve debug info for callee variables \
17751775
(default: preserve for debuginfo != None, otherwise remove)"),

‎tests/codegen/issues/issue-37945.rs

+15-12
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,35 @@
33

44
// Check that LLVM understands that `Iter` pointer is not null. Issue #37945.
55

6+
// Note that this is not about assumes or metadata, which are covered better by
7+
// `slice-iter-nonnull.rs`, rather about being sure that it's not emitting
8+
// a null check as part of computing the value, which we do by verifying that
9+
// the only return is only from comparing the two pointers.
10+
611
#![crate_type = "lib"]
712

813
use std::slice::Iter;
914

1015
#[no_mangle]
1116
pub fn is_empty_1(xs: Iter<f32>) -> bool {
12-
// CHECK-LABEL: @is_empty_1(
13-
// CHECK-NEXT: start:
14-
// CHECK-NEXT: [[A:%.*]] = icmp ne ptr {{%xs.0|%xs.1}}, null
15-
// CHECK-NEXT: tail call void @llvm.assume(i1 [[A]])
17+
// CHECK-LABEL: i1 @is_empty_1(
18+
// CHECK-NOT: ret i1
1619
// The order between %xs.0 and %xs.1 on the next line doesn't matter
1720
// and different LLVM versions produce different order.
18-
// CHECK-NEXT: [[B:%.*]] = icmp eq ptr {{%xs.0, %xs.1|%xs.1, %xs.0}}
19-
// CHECK-NEXT: ret i1 [[B:%.*]]
21+
// CHECK: [[B:%.*]] = icmp eq ptr {{%xs.0, %xs.1|%xs.1, %xs.0}}
22+
// CHECK-NEXT: ret i1 [[B:%.*]]
23+
// CHECK-NOT: ret i1
2024
{ xs }.next().is_none()
2125
}
2226

2327
#[no_mangle]
2428
pub fn is_empty_2(xs: Iter<f32>) -> bool {
25-
// CHECK-LABEL: @is_empty_2
26-
// CHECK-NEXT: start:
27-
// CHECK-NEXT: [[C:%.*]] = icmp ne ptr {{%xs.0|%xs.1}}, null
28-
// CHECK-NEXT: tail call void @llvm.assume(i1 [[C]])
29+
// CHECK-LABEL: i1 @is_empty_2
30+
// CHECK-NOT: ret i1
2931
// The order between %xs.0 and %xs.1 on the next line doesn't matter
3032
// and different LLVM versions produce different order.
31-
// CHECK-NEXT: [[D:%.*]] = icmp eq ptr {{%xs.0, %xs.1|%xs.1, %xs.0}}
32-
// CHECK-NEXT: ret i1 [[D:%.*]]
33+
// CHECK: [[B:%.*]] = icmp eq ptr {{%xs.0, %xs.1|%xs.1, %xs.0}}
34+
// CHECK-NEXT: ret i1 [[B:%.*]]
35+
// CHECK-NOT: ret i1
3336
xs.map(|&x| x).next().is_none()
3437
}

‎tests/mir-opt/issues/issue_59352.num_to_digit.PreCodegen.after.panic-abort.mir

+34-31
Original file line numberDiff line numberDiff line change
@@ -3,67 +3,70 @@
33
fn num_to_digit(_1: char) -> u32 {
44
debug num => _1;
55
let mut _0: u32;
6-
let mut _4: std::option::Option<u32>;
7-
scope 1 (inlined char::methods::<impl char>::is_digit) {
8-
let _2: std::option::Option<u32>;
9-
scope 2 (inlined Option::<u32>::is_some) {
10-
let mut _3: isize;
6+
let mut _2: bool;
7+
let mut _7: std::option::Option<u32>;
8+
scope 1 (inlined char::methods::<impl char>::to_digit) {
9+
let mut _3: u32;
10+
let mut _4: u32;
11+
let mut _5: bool;
12+
scope 2 {
13+
scope 4 (inlined Arguments::<'_>::new_const::<1>) {
14+
}
15+
scope 5 (inlined core::num::<impl u32>::wrapping_sub) {
16+
}
17+
scope 6 (inlined core::num::<impl u32>::saturating_add) {
18+
}
19+
}
20+
scope 3 (inlined core::num::<impl u32>::wrapping_sub) {
1121
}
1222
}
13-
scope 3 (inlined #[track_caller] Option::<u32>::unwrap) {
14-
let mut _5: isize;
23+
scope 7 (inlined #[track_caller] Option::<u32>::unwrap) {
1524
let mut _6: !;
16-
scope 4 {
25+
scope 8 {
1726
}
1827
}
1928

2029
bb0: {
2130
StorageLive(_2);
22-
_2 = char::methods::<impl char>::to_digit(_1, const 8_u32) -> [return: bb1, unwind unreachable];
31+
_2 = char::methods::<impl char>::is_digit(_1, const 8_u32) -> [return: bb1, unwind unreachable];
2332
}
2433

2534
bb1: {
26-
StorageLive(_3);
27-
_3 = discriminant(_2);
28-
switchInt(move _3) -> [1: bb2, 0: bb6, otherwise: bb8];
35+
switchInt(move _2) -> [0: bb2, otherwise: bb3];
2936
}
3037

3138
bb2: {
32-
StorageDead(_3);
33-
StorageDead(_2);
34-
StorageLive(_4);
35-
_4 = char::methods::<impl char>::to_digit(move _1, const 8_u32) -> [return: bb3, unwind unreachable];
39+
_0 = const 0_u32;
40+
goto -> bb6;
3641
}
3742

3843
bb3: {
44+
StorageLive(_7);
45+
StorageLive(_3);
46+
_3 = _1 as u32 (IntToInt);
47+
_4 = Sub(_3, const 48_u32);
3948
StorageLive(_5);
40-
_5 = discriminant(_4);
41-
switchInt(move _5) -> [0: bb4, 1: bb5, otherwise: bb8];
49+
_5 = Lt(_4, const 8_u32);
50+
switchInt(move _5) -> [0: bb4, otherwise: bb5];
4251
}
4352

4453
bb4: {
54+
StorageDead(_5);
55+
StorageDead(_3);
4556
_6 = option::unwrap_failed() -> unwind unreachable;
4657
}
4758

4859
bb5: {
49-
_0 = move ((_4 as Some).0: u32);
60+
_7 = Option::<u32>::Some(_4);
5061
StorageDead(_5);
51-
StorageDead(_4);
52-
goto -> bb7;
62+
StorageDead(_3);
63+
_0 = move ((_7 as Some).0: u32);
64+
StorageDead(_7);
65+
goto -> bb6;
5366
}
5467

5568
bb6: {
56-
StorageDead(_3);
5769
StorageDead(_2);
58-
_0 = const 0_u32;
59-
goto -> bb7;
60-
}
61-
62-
bb7: {
6370
return;
6471
}
65-
66-
bb8: {
67-
unreachable;
68-
}
6972
}

‎tests/mir-opt/issues/issue_59352.num_to_digit.PreCodegen.after.panic-unwind.mir

+34-31
Original file line numberDiff line numberDiff line change
@@ -3,67 +3,70 @@
33
fn num_to_digit(_1: char) -> u32 {
44
debug num => _1;
55
let mut _0: u32;
6-
let mut _4: std::option::Option<u32>;
7-
scope 1 (inlined char::methods::<impl char>::is_digit) {
8-
let _2: std::option::Option<u32>;
9-
scope 2 (inlined Option::<u32>::is_some) {
10-
let mut _3: isize;
6+
let mut _2: bool;
7+
let mut _7: std::option::Option<u32>;
8+
scope 1 (inlined char::methods::<impl char>::to_digit) {
9+
let mut _3: u32;
10+
let mut _4: u32;
11+
let mut _5: bool;
12+
scope 2 {
13+
scope 4 (inlined Arguments::<'_>::new_const::<1>) {
14+
}
15+
scope 5 (inlined core::num::<impl u32>::wrapping_sub) {
16+
}
17+
scope 6 (inlined core::num::<impl u32>::saturating_add) {
18+
}
19+
}
20+
scope 3 (inlined core::num::<impl u32>::wrapping_sub) {
1121
}
1222
}
13-
scope 3 (inlined #[track_caller] Option::<u32>::unwrap) {
14-
let mut _5: isize;
23+
scope 7 (inlined #[track_caller] Option::<u32>::unwrap) {
1524
let mut _6: !;
16-
scope 4 {
25+
scope 8 {
1726
}
1827
}
1928

2029
bb0: {
2130
StorageLive(_2);
22-
_2 = char::methods::<impl char>::to_digit(_1, const 8_u32) -> [return: bb1, unwind continue];
31+
_2 = char::methods::<impl char>::is_digit(_1, const 8_u32) -> [return: bb1, unwind continue];
2332
}
2433

2534
bb1: {
26-
StorageLive(_3);
27-
_3 = discriminant(_2);
28-
switchInt(move _3) -> [1: bb2, 0: bb6, otherwise: bb8];
35+
switchInt(move _2) -> [0: bb2, otherwise: bb3];
2936
}
3037

3138
bb2: {
32-
StorageDead(_3);
33-
StorageDead(_2);
34-
StorageLive(_4);
35-
_4 = char::methods::<impl char>::to_digit(move _1, const 8_u32) -> [return: bb3, unwind continue];
39+
_0 = const 0_u32;
40+
goto -> bb6;
3641
}
3742

3843
bb3: {
44+
StorageLive(_7);
45+
StorageLive(_3);
46+
_3 = _1 as u32 (IntToInt);
47+
_4 = Sub(_3, const 48_u32);
3948
StorageLive(_5);
40-
_5 = discriminant(_4);
41-
switchInt(move _5) -> [0: bb4, 1: bb5, otherwise: bb8];
49+
_5 = Lt(_4, const 8_u32);
50+
switchInt(move _5) -> [0: bb4, otherwise: bb5];
4251
}
4352

4453
bb4: {
54+
StorageDead(_5);
55+
StorageDead(_3);
4556
_6 = option::unwrap_failed() -> unwind continue;
4657
}
4758

4859
bb5: {
49-
_0 = move ((_4 as Some).0: u32);
60+
_7 = Option::<u32>::Some(_4);
5061
StorageDead(_5);
51-
StorageDead(_4);
52-
goto -> bb7;
62+
StorageDead(_3);
63+
_0 = move ((_7 as Some).0: u32);
64+
StorageDead(_7);
65+
goto -> bb6;
5366
}
5467

5568
bb6: {
56-
StorageDead(_3);
5769
StorageDead(_2);
58-
_0 = const 0_u32;
59-
goto -> bb7;
60-
}
61-
62-
bb7: {
6370
return;
6471
}
65-
66-
bb8: {
67-
unreachable;
68-
}
6972
}

‎tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff

+57-29
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,19 @@
1919
}
2020
scope 5 (inlined <std::alloc::Global as Allocator>::allocate) {
2121
}
22-
scope 6 (inlined NonNull::<[u8]>::as_ptr) {
23-
let mut _12: *const [u8];
22+
scope 6 (inlined #[track_caller] Result::<NonNull<[u8]>, std::alloc::AllocError>::unwrap) {
23+
let mut _12: isize;
24+
let _13: std::alloc::AllocError;
25+
let mut _14: !;
26+
let mut _15: &dyn std::fmt::Debug;
27+
let mut _16: &std::alloc::AllocError;
28+
scope 7 {
29+
}
30+
scope 8 {
31+
}
32+
}
33+
scope 9 (inlined NonNull::<[u8]>::as_ptr) {
34+
let mut _17: *const [u8];
2435
}
2536
}
2637
scope 3 (inlined #[track_caller] Option::<Layout>::unwrap) {
@@ -38,39 +49,20 @@
3849
+ _2 = const Option::<Layout>::None;
3950
StorageLive(_10);
4051
- _10 = discriminant(_2);
41-
- switchInt(move _10) -> [0: bb3, 1: bb4, otherwise: bb2];
52+
- switchInt(move _10) -> [0: bb2, 1: bb3, otherwise: bb1];
4253
+ _10 = const 0_isize;
43-
+ switchInt(const 0_isize) -> [0: bb3, 1: bb4, otherwise: bb2];
54+
+ switchInt(const 0_isize) -> [0: bb2, 1: bb3, otherwise: bb1];
4455
}
4556

4657
bb1: {
47-
StorageDead(_6);
48-
- StorageLive(_12);
49-
+ nop;
50-
_12 = (_5.0: *const [u8]);
51-
- _4 = move _12 as *mut [u8] (PtrToPtr);
52-
- StorageDead(_12);
53-
+ _4 = _12 as *mut [u8] (PtrToPtr);
54-
+ nop;
55-
StorageDead(_5);
56-
- _3 = move _4 as *mut u8 (PtrToPtr);
57-
+ _3 = _12 as *mut u8 (PtrToPtr);
58-
StorageDead(_4);
59-
StorageDead(_3);
60-
- StorageDead(_1);
61-
+ nop;
62-
return;
63-
}
64-
65-
bb2: {
6658
unreachable;
6759
}
6860

69-
bb3: {
61+
bb2: {
7062
_11 = option::unwrap_failed() -> unwind continue;
7163
}
7264

73-
bb4: {
65+
bb3: {
7466
- _1 = move ((_2 as Some).0: std::alloc::Layout);
7567
+ _1 = const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(4 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x00000000): std::ptr::alignment::AlignmentEnum) }};
7668
StorageDead(_10);
@@ -85,15 +77,51 @@
8577
+ _7 = const {ALLOC1<imm>: &std::alloc::Global};
8678
StorageLive(_8);
8779
- _8 = _1;
88-
- _6 = std::alloc::Global::alloc_impl(move _7, move _8, const false) -> [return: bb5, unwind continue];
80+
- _6 = std::alloc::Global::alloc_impl(move _7, move _8, const false) -> [return: bb4, unwind continue];
8981
+ _8 = const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(4 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x00000000): std::ptr::alignment::AlignmentEnum) }};
90-
+ _6 = std::alloc::Global::alloc_impl(const {ALLOC1<imm>: &std::alloc::Global}, const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(4 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x00000000): std::ptr::alignment::AlignmentEnum) }}, const false) -> [return: bb5, unwind continue];
82+
+ _6 = std::alloc::Global::alloc_impl(const {ALLOC1<imm>: &std::alloc::Global}, const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(4 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x00000000): std::ptr::alignment::AlignmentEnum) }}, const false) -> [return: bb4, unwind continue];
9183
}
9284

93-
bb5: {
85+
bb4: {
9486
StorageDead(_8);
9587
StorageDead(_7);
96-
_5 = Result::<NonNull<[u8]>, std::alloc::AllocError>::unwrap(move _6) -> [return: bb1, unwind continue];
88+
StorageLive(_12);
89+
_12 = discriminant(_6);
90+
switchInt(move _12) -> [0: bb6, 1: bb5, otherwise: bb1];
91+
}
92+
93+
bb5: {
94+
StorageLive(_15);
95+
StorageLive(_16);
96+
_16 = &_13;
97+
_15 = move _16 as &dyn std::fmt::Debug (PointerCoercion(Unsize));
98+
StorageDead(_16);
99+
_14 = result::unwrap_failed(const "called `Result::unwrap()` on an `Err` value", move _15) -> bb7;
100+
}
101+
102+
bb6: {
103+
_5 = move ((_6 as Ok).0: std::ptr::NonNull<[u8]>);
104+
StorageDead(_12);
105+
StorageDead(_6);
106+
- StorageLive(_17);
107+
+ nop;
108+
_17 = (_5.0: *const [u8]);
109+
- _4 = move _17 as *mut [u8] (PtrToPtr);
110+
- StorageDead(_17);
111+
+ _4 = _17 as *mut [u8] (PtrToPtr);
112+
+ nop;
113+
StorageDead(_5);
114+
- _3 = move _4 as *mut u8 (PtrToPtr);
115+
+ _3 = _17 as *mut u8 (PtrToPtr);
116+
StorageDead(_4);
117+
StorageDead(_3);
118+
- StorageDead(_1);
119+
+ nop;
120+
return;
121+
}
122+
123+
bb7 (cleanup): {
124+
resume;
97125
}
98126
}
99127
+

0 commit comments

Comments
 (0)