Skip to content

Commit 844c3a8

Browse files
vg0204vg0204
authored
[WaveTransform] Fix iterator invalidation for DenseSet in fixRegMaskClobberedPhysRegLiveness (#3061)
The fixRegMaskClobberedPhysRegLiveness() function crashes with an `isHandleInSync() assertion` when processing functions with indirect calls inside waterfall loops for given test. It happens because the sub-register deduplication loop attempts to erase elements from a SmallDenseSet while iterating over it that results in the invalidation bug. So, this patch aims to convert erase-during-iteration pattern with a two-phase collect-then-erase pattern for the redundant sub-registers.
1 parent 0809a95 commit 844c3a8

2 files changed

Lines changed: 103 additions & 90 deletions

File tree

‎llvm/lib/Target/AMDGPU/AMDGPUWaveTransform.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2466,8 +2466,6 @@ static bool fixRegMaskClobberedPhysRegLiveness(
24662466
Candidates.insert(Pred);
24672467
}
24682468

2469-
SmallDenseSet<MCPhysReg, 8> RegsWithLivenessGap;
2470-
SmallVector<MCPhysReg, 8> SortedRegs;
24712469
for (MachineBasicBlock *MBB : Candidates) {
24722470
// Locate the first MI with a regmask operand in MBB via forward walk.
24732471
// So, later during backward traversl during liveness computation, it can
@@ -2488,7 +2486,8 @@ static bool fixRegMaskClobberedPhysRegLiveness(
24882486
if (LiveRegs.empty())
24892487
continue;
24902488

2491-
RegsWithLivenessGap.clear();
2489+
SmallDenseSet<MCPhysReg, 8> RegsWithLivenessGap;
2490+
SmallVector<MCPhysReg, 8> SortedRegs;
24922491
for (MachineInstr &MI : reverse(*MBB)) {
24932492
for (const MachineOperand &MO : MI.operands()) {
24942493
if (!MO.isRegMask())
@@ -2512,11 +2511,14 @@ static bool fixRegMaskClobberedPhysRegLiveness(
25122511
continue;
25132512

25142513
// Remove any sub-regs already covered by a super-register in the gap.
2514+
SmallVector<MCPhysReg, 4> SubRegsToRemove;
25152515
for (MCPhysReg Reg : RegsWithLivenessGap) {
25162516
if (any_of(TRI.superregs(Reg),
25172517
[&](MCPhysReg Super) { return RegsWithLivenessGap.contains(Super); }))
2518-
RegsWithLivenessGap.erase(Reg);
2518+
SubRegsToRemove.push_back(Reg);
25192519
}
2520+
for (MCPhysReg Reg : SubRegsToRemove)
2521+
RegsWithLivenessGap.erase(Reg);
25202522

25212523
// Insert in a deterministic (register-number) order for stable output.
25222524
SortedRegs.assign(RegsWithLivenessGap.begin(), RegsWithLivenessGap.end());

‎llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll‎

Lines changed: 97 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,27 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2-
; RUN: llc -amdgpu-late-wave-transform=0 -mtriple=amdgcn -mcpu=gfx1010 -amdgpu-opt-vgpr-liverange=true < %s | FileCheck -check-prefix=SI %s
2+
; RUN: llc -amdgpu-late-wave-transform=1 -mtriple=amdgcn -mcpu=gfx1010 -amdgpu-opt-vgpr-liverange=true < %s | FileCheck -check-prefix=SI %s
33

44
; a normal if-else
55
define amdgpu_ps float @else1(i32 %z, float %v) #0 {
66
; SI-LABEL: else1:
77
; SI: ; %bb.0: ; %main_body
88
; SI-NEXT: v_cmp_gt_i32_e32 vcc_lo, 6, v0
9-
; SI-NEXT: ; implicit-def: $vgpr0
10-
; SI-NEXT: s_and_saveexec_b32 s0, vcc_lo
11-
; SI-NEXT: s_xor_b32 s0, exec_lo, s0
12-
; SI-NEXT: s_cbranch_execnz .LBB0_3
13-
; SI-NEXT: ; %bb.1: ; %Flow
14-
; SI-NEXT: s_andn2_saveexec_b32 s0, s0
15-
; SI-NEXT: s_cbranch_execnz .LBB0_4
16-
; SI-NEXT: .LBB0_2: ; %end
17-
; SI-NEXT: s_or_b32 exec_lo, exec_lo, s0
18-
; SI-NEXT: s_branch .LBB0_5
19-
; SI-NEXT: .LBB0_3: ; %else
20-
; SI-NEXT: v_mul_f32_e32 v0, 0x40400000, v1
21-
; SI-NEXT: ; implicit-def: $vgpr1
22-
; SI-NEXT: s_andn2_saveexec_b32 s0, s0
9+
; SI-NEXT: s_xor_b32 exec_lo, vcc_lo, exec_lo
10+
; SI-NEXT: ; divergent control-flow edge
2311
; SI-NEXT: s_cbranch_execz .LBB0_2
24-
; SI-NEXT: .LBB0_4: ; %if
12+
; SI-NEXT: .LBB0_1: ; %if
2513
; SI-NEXT: v_add_f32_e32 v0, v1, v1
14+
; SI-NEXT: .LBB0_2:
15+
; SI-NEXT: s_or_b32 exec_lo, exec_lo, vcc_lo
16+
; SI-NEXT: s_xor_b32 s0, exec_lo, vcc_lo
17+
; SI-NEXT: s_mov_b32 exec_lo, vcc_lo
18+
; SI-NEXT: ; divergent control-flow edge
19+
; SI-NEXT: s_cbranch_execz .LBB0_4
20+
; SI-NEXT: .LBB0_3: ; %else
21+
; SI-NEXT: v_mul_f32_e32 v0, 0x40400000, v1
22+
; SI-NEXT: .LBB0_4: ; %end
2623
; SI-NEXT: s_or_b32 exec_lo, exec_lo, s0
27-
; SI-NEXT: s_branch .LBB0_5
28-
; SI-NEXT: .LBB0_5:
24+
; SI-NEXT: ; return to shader part epilog
2925
main_body:
3026
%cc = icmp sgt i32 %z, 5
3127
br i1 %cc, label %if, label %else
@@ -49,17 +45,21 @@ define amdgpu_ps float @else2(i32 %z, float %v) #0 {
4945
; SI-LABEL: else2:
5046
; SI: ; %bb.0: ; %main_body
5147
; SI-NEXT: v_cmp_gt_i32_e32 vcc_lo, 6, v0
52-
; SI-NEXT: ; implicit-def: $vgpr0
53-
; SI-NEXT: s_and_saveexec_b32 s0, vcc_lo
54-
; SI-NEXT: s_xor_b32 s0, exec_lo, s0
55-
; SI-NEXT: ; %bb.1: ; %else
56-
; SI-NEXT: v_mul_f32_e32 v0, 0x40400000, v1
57-
; SI-NEXT: ; %bb.2: ; %Flow
58-
; SI-NEXT: s_andn2_saveexec_b32 s0, s0
59-
; SI-NEXT: ; %bb.3: ; %if
48+
; SI-NEXT: s_xor_b32 exec_lo, vcc_lo, exec_lo
49+
; SI-NEXT: ; divergent control-flow edge
50+
; SI-NEXT: s_cbranch_execz .LBB1_2
51+
; SI-NEXT: .LBB1_1: ; %if
6052
; SI-NEXT: v_add_f32_e32 v1, v1, v1
6153
; SI-NEXT: v_mov_b32_e32 v0, v1
62-
; SI-NEXT: ; %bb.4: ; %end
54+
; SI-NEXT: .LBB1_2:
55+
; SI-NEXT: s_or_b32 exec_lo, exec_lo, vcc_lo
56+
; SI-NEXT: s_xor_b32 s0, exec_lo, vcc_lo
57+
; SI-NEXT: s_mov_b32 exec_lo, vcc_lo
58+
; SI-NEXT: ; divergent control-flow edge
59+
; SI-NEXT: s_cbranch_execz .LBB1_4
60+
; SI-NEXT: .LBB1_3: ; %else
61+
; SI-NEXT: v_mul_f32_e32 v0, 0x40400000, v1
62+
; SI-NEXT: .LBB1_4: ; %end
6363
; SI-NEXT: s_or_b32 exec_lo, exec_lo, s0
6464
; SI-NEXT: v_add_f32_e32 v0, v1, v0
6565
; SI-NEXT: ; return to shader part epilog
@@ -86,38 +86,46 @@ end:
8686
define amdgpu_ps float @else3(i32 %z, float %v, i32 inreg %bound, i32 %x0) #0 {
8787
; SI-LABEL: else3:
8888
; SI: ; %bb.0: ; %entry
89+
; SI-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v0
8990
; SI-NEXT: s_mov_b32 s1, 0
90-
; SI-NEXT: v_cmp_gt_i32_e32 vcc_lo, 6, v0
91+
; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
9192
; SI-NEXT: s_branch .LBB2_2
9293
; SI-NEXT: .LBB2_1: ; %if.end
9394
; SI-NEXT: ; in Loop: Header=BB2_2 Depth=1
9495
; SI-NEXT: s_or_b32 exec_lo, exec_lo, s2
95-
; SI-NEXT: v_add_nc_u32_e32 v2, 1, v3
96+
; SI-NEXT: v_add_nc_u32_e32 v2, 1, v4
9697
; SI-NEXT: s_add_i32 s1, s1, 1
9798
; SI-NEXT: s_cmp_lt_i32 s1, s0
9899
; SI-NEXT: s_cbranch_scc0 .LBB2_6
99100
; SI-NEXT: .LBB2_2: ; %for.body
100101
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
101-
; SI-NEXT: ; implicit-def: $vgpr3
102-
; SI-NEXT: ; implicit-def: $vgpr0
103-
; SI-NEXT: s_and_saveexec_b32 s2, vcc_lo
104-
; SI-NEXT: s_xor_b32 s2, exec_lo, s2
105-
; SI-NEXT: ; %bb.3: ; %else
102+
; SI-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
103+
; SI-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
104+
; SI-NEXT: v_cmp_ne_u32_e32 vcc_lo, 1, v3
105+
; SI-NEXT: s_xor_b32 s3, vcc_lo, exec_lo
106+
; SI-NEXT: s_xor_b32 s2, exec_lo, s3
107+
; SI-NEXT: s_and_b32 s2, s2, exec_lo
108+
; SI-NEXT: s_mov_b32 exec_lo, s3
109+
; SI-NEXT: ; divergent control-flow edge
110+
; SI-NEXT: s_cbranch_execz .LBB2_4
111+
; SI-NEXT: .LBB2_3: ; %if
106112
; SI-NEXT: ; in Loop: Header=BB2_2 Depth=1
107-
; SI-NEXT: v_mul_f32_e32 v0, v1, v2
108-
; SI-NEXT: v_lshl_add_u32 v3, v2, 1, v2
109-
; SI-NEXT: ; implicit-def: $vgpr2
110-
; SI-NEXT: ; %bb.4: ; %Flow
111-
; SI-NEXT: ; in Loop: Header=BB2_2 Depth=1
112-
; SI-NEXT: s_andn2_saveexec_b32 s2, s2
113+
; SI-NEXT: v_mul_f32_e32 v3, s1, v1
114+
; SI-NEXT: v_add_nc_u32_e32 v4, 1, v2
115+
; SI-NEXT: .LBB2_4: ; in Loop: Header=BB2_2 Depth=1
116+
; SI-NEXT: s_or_b32 exec_lo, exec_lo, s2
117+
; SI-NEXT: s_xor_b32 s2, exec_lo, vcc_lo
118+
; SI-NEXT: s_and_b32 s2, s2, exec_lo
119+
; SI-NEXT: s_mov_b32 exec_lo, vcc_lo
120+
; SI-NEXT: ; divergent control-flow edge
113121
; SI-NEXT: s_cbranch_execz .LBB2_1
114-
; SI-NEXT: ; %bb.5: ; %if
122+
; SI-NEXT: .LBB2_5: ; %else
115123
; SI-NEXT: ; in Loop: Header=BB2_2 Depth=1
116-
; SI-NEXT: v_mul_f32_e32 v0, s1, v1
117-
; SI-NEXT: v_add_nc_u32_e32 v3, 1, v2
124+
; SI-NEXT: v_mul_f32_e32 v3, v1, v2
125+
; SI-NEXT: v_lshl_add_u32 v4, v2, 1, v2
118126
; SI-NEXT: s_branch .LBB2_1
119127
; SI-NEXT: .LBB2_6: ; %for.end
120-
; SI-NEXT: v_add_f32_e32 v0, v3, v0
128+
; SI-NEXT: v_add_f32_e32 v0, v4, v3
121129
; SI-NEXT: ; return to shader part epilog
122130
entry:
123131
; %break = icmp sgt i32 %bound, 0
@@ -166,52 +174,53 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
166174
; SI-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
167175
; SI-NEXT: s_mov_b32 s14, -1
168176
; SI-NEXT: v_mov_b32_e32 v0, v1
169-
; SI-NEXT: v_cmp_gt_i32_e32 vcc_lo, 6, v6
177+
; SI-NEXT: v_cmp_gt_i32_e64 s4, 6, v6
170178
; SI-NEXT: s_mov_b32 s15, 0x31c16000
171179
; SI-NEXT: s_add_u32 s12, s12, s1
172180
; SI-NEXT: s_addc_u32 s13, s13, 0
173181
; SI-NEXT: s_mov_b32 s32, 0
174-
; SI-NEXT: ; implicit-def: $vgpr1
175-
; SI-NEXT: s_and_saveexec_b32 s0, vcc_lo
176-
; SI-NEXT: s_xor_b32 s6, exec_lo, s0
182+
; SI-NEXT: s_xor_b32 exec_lo, s4, exec_lo
183+
; SI-NEXT: ; divergent control-flow edge
177184
; SI-NEXT: s_cbranch_execz .LBB3_4
178-
; SI-NEXT: ; %bb.1: ; %else
179-
; SI-NEXT: s_mov_b32 s7, exec_lo
180-
; SI-NEXT: s_mov_b32 s8, s7
185+
; SI-NEXT: .LBB3_1: ; %if
186+
; SI-NEXT: s_mov_b32 s5, exec_lo
187+
; SI-NEXT: s_mov_b32 s8, s5
181188
; SI-NEXT: .LBB3_2: ; =>This Inner Loop Header: Depth=1
182-
; SI-NEXT: v_readfirstlane_b32 s4, v4
183-
; SI-NEXT: v_readfirstlane_b32 s5, v5
189+
; SI-NEXT: v_readfirstlane_b32 s6, v2
190+
; SI-NEXT: v_readfirstlane_b32 s7, v3
184191
; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
185-
; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
192+
; SI-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
186193
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
187194
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
188-
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
195+
; SI-NEXT: s_swappc_b64 s[30:31], s[6:7]
189196
; SI-NEXT: v_mov_b32_e32 v1, v0
190197
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
191-
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
198+
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
192199
; SI-NEXT: ; implicit-def: $vgpr0
200+
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
193201
; SI-NEXT: s_cbranch_execnz .LBB3_2
194202
; SI-NEXT: ; %bb.3:
195-
; SI-NEXT: s_mov_b32 exec_lo, s7
196-
; SI-NEXT: ; implicit-def: $vgpr0
197-
; SI-NEXT: ; implicit-def: $vgpr2
198-
; SI-NEXT: .LBB3_4: ; %Flow
199-
; SI-NEXT: s_andn2_saveexec_b32 s6, s6
203+
; SI-NEXT: s_mov_b32 exec_lo, s5
204+
; SI-NEXT: .LBB3_4:
205+
; SI-NEXT: s_or_b32 exec_lo, exec_lo, s4
206+
; SI-NEXT: s_xor_b32 s6, exec_lo, s4
207+
; SI-NEXT: s_mov_b32 exec_lo, s4
208+
; SI-NEXT: ; divergent control-flow edge
200209
; SI-NEXT: s_cbranch_execz .LBB3_8
201-
; SI-NEXT: ; %bb.5: ; %if
210+
; SI-NEXT: .LBB3_5: ; %else
202211
; SI-NEXT: s_mov_b32 s7, exec_lo
203212
; SI-NEXT: s_mov_b32 s8, s7
204213
; SI-NEXT: .LBB3_6: ; =>This Inner Loop Header: Depth=1
205-
; SI-NEXT: v_readfirstlane_b32 s4, v2
206-
; SI-NEXT: v_readfirstlane_b32 s5, v3
214+
; SI-NEXT: v_readfirstlane_b32 s4, v4
215+
; SI-NEXT: v_readfirstlane_b32 s5, v5
207216
; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
208-
; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
217+
; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
209218
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
210219
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
211220
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
212221
; SI-NEXT: v_mov_b32_e32 v1, v0
213222
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
214-
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
223+
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
215224
; SI-NEXT: ; implicit-def: $vgpr0
216225
; SI-NEXT: s_cbranch_execnz .LBB3_6
217226
; SI-NEXT: ; %bb.7:
@@ -245,50 +254,52 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
245254
; SI-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
246255
; SI-NEXT: s_mov_b32 s14, -1
247256
; SI-NEXT: v_mov_b32_e32 v40, v1
248-
; SI-NEXT: v_cmp_gt_i32_e32 vcc_lo, 6, v0
257+
; SI-NEXT: v_cmp_gt_i32_e64 s4, 6, v0
249258
; SI-NEXT: s_mov_b32 s15, 0x31c16000
250259
; SI-NEXT: s_add_u32 s12, s12, s1
251260
; SI-NEXT: s_addc_u32 s13, s13, 0
252261
; SI-NEXT: s_mov_b32 s32, 0
253-
; SI-NEXT: ; implicit-def: $vgpr0
254-
; SI-NEXT: s_and_saveexec_b32 s0, vcc_lo
255-
; SI-NEXT: s_xor_b32 s6, exec_lo, s0
262+
; SI-NEXT: s_xor_b32 exec_lo, s4, exec_lo
263+
; SI-NEXT: ; divergent control-flow edge
256264
; SI-NEXT: s_cbranch_execz .LBB4_4
257-
; SI-NEXT: ; %bb.1: ; %else
258-
; SI-NEXT: s_mov_b32 s7, exec_lo
259-
; SI-NEXT: s_mov_b32 s8, s7
265+
; SI-NEXT: .LBB4_1: ; %if
266+
; SI-NEXT: s_mov_b32 s5, exec_lo
267+
; SI-NEXT: s_mov_b32 s8, s5
260268
; SI-NEXT: .LBB4_2: ; =>This Inner Loop Header: Depth=1
261-
; SI-NEXT: v_readfirstlane_b32 s4, v4
262-
; SI-NEXT: v_readfirstlane_b32 s5, v5
269+
; SI-NEXT: v_readfirstlane_b32 s6, v2
270+
; SI-NEXT: v_readfirstlane_b32 s7, v3
263271
; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
264-
; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
272+
; SI-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
265273
; SI-NEXT: v_mov_b32_e32 v0, v40
266274
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
267275
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
268-
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
276+
; SI-NEXT: s_swappc_b64 s[30:31], s[6:7]
269277
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
278+
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
270279
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
271280
; SI-NEXT: s_cbranch_execnz .LBB4_2
272281
; SI-NEXT: ; %bb.3:
273-
; SI-NEXT: s_mov_b32 exec_lo, s7
274-
; SI-NEXT: ; implicit-def: $vgpr2
275-
; SI-NEXT: .LBB4_4: ; %Flow
276-
; SI-NEXT: s_andn2_saveexec_b32 s6, s6
282+
; SI-NEXT: s_mov_b32 exec_lo, s5
283+
; SI-NEXT: .LBB4_4:
284+
; SI-NEXT: s_or_b32 exec_lo, exec_lo, s4
285+
; SI-NEXT: s_xor_b32 s6, exec_lo, s4
286+
; SI-NEXT: s_mov_b32 exec_lo, s4
287+
; SI-NEXT: ; divergent control-flow edge
277288
; SI-NEXT: s_cbranch_execz .LBB4_8
278-
; SI-NEXT: ; %bb.5: ; %if
289+
; SI-NEXT: .LBB4_5: ; %else
279290
; SI-NEXT: s_mov_b32 s7, exec_lo
280291
; SI-NEXT: s_mov_b32 s8, s7
281292
; SI-NEXT: .LBB4_6: ; =>This Inner Loop Header: Depth=1
282-
; SI-NEXT: v_readfirstlane_b32 s4, v2
283-
; SI-NEXT: v_readfirstlane_b32 s5, v3
293+
; SI-NEXT: v_readfirstlane_b32 s4, v4
294+
; SI-NEXT: v_readfirstlane_b32 s5, v5
284295
; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
285-
; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
296+
; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
286297
; SI-NEXT: v_mov_b32_e32 v0, v40
287298
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
288299
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
289300
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
290301
; SI-NEXT: s_andn2_wrexec_b32 s8, s8
291-
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
302+
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
292303
; SI-NEXT: s_cbranch_execnz .LBB4_6
293304
; SI-NEXT: ; %bb.7:
294305
; SI-NEXT: s_mov_b32 exec_lo, s7

0 commit comments

Comments
 (0)