Skip to content

Commit 2501ae5

Browse files
jayfoadjayfoad
authored
[CodeGen] Really renumber slot indexes before register allocation (#67038)
PR #66334 tried to renumber slot indexes before register allocation, but the numbering was still affected by list entries for instructions which had been erased. Fix this to make the register allocator's live range length heuristics even less dependent on the history of how instructions have been added to and removed from SlotIndexes's maps.
1 parent 648046d commit 2501ae5

736 files changed

Lines changed: 250920 additions & 259056 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎llvm/lib/CodeGen/SlotIndexes.cpp‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,8 +238,38 @@ void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB,
238238
}
239239

240240
void SlotIndexes::packIndexes() {
241-
for (auto [Index, Entry] : enumerate(indexList))
242-
Entry.setIndex(Index * SlotIndex::InstrDist);
241+
unsigned Index = 0;
242+
// Check that the dummy entry for the start of the first block does not need
243+
// updating. It should always be 0.
244+
assert(idx2MBBMap[0].second->getNumber() == 0 &&
245+
"First MBB should be number 0!");
246+
assert(MBBRanges[0].first.getIndex() == Index && "First index should be 0!");
247+
Index += SlotIndex::InstrDist;
248+
// Iterate over basic blocks in slot index order.
249+
for (MachineBasicBlock *MBB : make_second_range(idx2MBBMap)) {
250+
auto [MBBStartIdx, MBBEndIdx] = MBBRanges[MBB->getNumber()];
251+
auto Start = MBBStartIdx.listEntry()->getIterator();
252+
auto End = MBBEndIdx.listEntry()->getIterator();
253+
// Update entries for each instruction in the block.
254+
for (auto &I : make_early_inc_range(make_range(std::next(Start), End))) {
255+
if (I.getInstr()) {
256+
I.setIndex(Index);
257+
Index += SlotIndex::InstrDist;
258+
} else {
259+
// Remove entries for deleted instructions.
260+
// FIXME: Eventually we want to remove them in
261+
// removeMachineInstrFromMaps but that is not currently possible because
262+
// some SlotIndexes API functions are called in a transiently broken
263+
// state where some live ranges still refer to indexes of deleted
264+
// instructions.
265+
// TODO: Add removed entries to a free list so they can be reused?
266+
indexList.remove(I);
267+
}
268+
}
269+
// Update the dummy entry for the end of the block.
270+
End->setIndex(Index);
271+
Index += SlotIndex::InstrDist;
272+
}
243273
}
244274

245275
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

‎llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-outline_atomics.ll‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,8 @@ define dso_local i128 @load_atomic_i128_aligned_unordered(ptr %ptr) {
236236
; -O0: stxp w8, x0, x1, [x9]
237237
;
238238
; -O1-LABEL: load_atomic_i128_aligned_unordered:
239-
; -O1: ldxp x0, x1, [x8]
240-
; -O1: stxp w9, x0, x1, [x8]
239+
; -O1: ldxp x8, x1, [x0]
240+
; -O1: stxp w9, x8, x1, [x0]
241241
%r = load atomic i128, ptr %ptr unordered, align 16
242242
ret i128 %r
243243
}
@@ -251,8 +251,8 @@ define dso_local i128 @load_atomic_i128_aligned_unordered_const(ptr readonly %pt
251251
; -O0: stxp w8, x0, x1, [x9]
252252
;
253253
; -O1-LABEL: load_atomic_i128_aligned_unordered_const:
254-
; -O1: ldxp x0, x1, [x8]
255-
; -O1: stxp w9, x0, x1, [x8]
254+
; -O1: ldxp x8, x1, [x0]
255+
; -O1: stxp w9, x8, x1, [x0]
256256
%r = load atomic i128, ptr %ptr unordered, align 16
257257
ret i128 %r
258258
}
@@ -266,8 +266,8 @@ define dso_local i128 @load_atomic_i128_aligned_monotonic(ptr %ptr) {
266266
; -O0: stxp w8, x0, x1, [x9]
267267
;
268268
; -O1-LABEL: load_atomic_i128_aligned_monotonic:
269-
; -O1: ldxp x0, x1, [x8]
270-
; -O1: stxp w9, x0, x1, [x8]
269+
; -O1: ldxp x8, x1, [x0]
270+
; -O1: stxp w9, x8, x1, [x0]
271271
%r = load atomic i128, ptr %ptr monotonic, align 16
272272
ret i128 %r
273273
}
@@ -281,8 +281,8 @@ define dso_local i128 @load_atomic_i128_aligned_monotonic_const(ptr readonly %pt
281281
; -O0: stxp w8, x0, x1, [x9]
282282
;
283283
; -O1-LABEL: load_atomic_i128_aligned_monotonic_const:
284-
; -O1: ldxp x0, x1, [x8]
285-
; -O1: stxp w9, x0, x1, [x8]
284+
; -O1: ldxp x8, x1, [x0]
285+
; -O1: stxp w9, x8, x1, [x0]
286286
%r = load atomic i128, ptr %ptr monotonic, align 16
287287
ret i128 %r
288288
}
@@ -296,8 +296,8 @@ define dso_local i128 @load_atomic_i128_aligned_acquire(ptr %ptr) {
296296
; -O0: stxp w8, x0, x1, [x9]
297297
;
298298
; -O1-LABEL: load_atomic_i128_aligned_acquire:
299-
; -O1: ldaxp x0, x1, [x8]
300-
; -O1: stxp w9, x0, x1, [x8]
299+
; -O1: ldaxp x8, x1, [x0]
300+
; -O1: stxp w9, x8, x1, [x0]
301301
%r = load atomic i128, ptr %ptr acquire, align 16
302302
ret i128 %r
303303
}
@@ -311,8 +311,8 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
311311
; -O0: stxp w8, x0, x1, [x9]
312312
;
313313
; -O1-LABEL: load_atomic_i128_aligned_acquire_const:
314-
; -O1: ldaxp x0, x1, [x8]
315-
; -O1: stxp w9, x0, x1, [x8]
314+
; -O1: ldaxp x8, x1, [x0]
315+
; -O1: stxp w9, x8, x1, [x0]
316316
%r = load atomic i128, ptr %ptr acquire, align 16
317317
ret i128 %r
318318
}
@@ -326,8 +326,8 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
326326
; -O0: stlxp w8, x0, x1, [x9]
327327
;
328328
; -O1-LABEL: load_atomic_i128_aligned_seq_cst:
329-
; -O1: ldaxp x0, x1, [x8]
330-
; -O1: stlxp w9, x0, x1, [x8]
329+
; -O1: ldaxp x8, x1, [x0]
330+
; -O1: stlxp w9, x8, x1, [x0]
331331
%r = load atomic i128, ptr %ptr seq_cst, align 16
332332
ret i128 %r
333333
}
@@ -341,8 +341,8 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr)
341341
; -O0: stlxp w8, x0, x1, [x9]
342342
;
343343
; -O1-LABEL: load_atomic_i128_aligned_seq_cst_const:
344-
; -O1: ldaxp x0, x1, [x8]
345-
; -O1: stlxp w9, x0, x1, [x8]
344+
; -O1: ldaxp x8, x1, [x0]
345+
; -O1: stlxp w9, x8, x1, [x0]
346346
%r = load atomic i128, ptr %ptr seq_cst, align 16
347347
ret i128 %r
348348
}

‎llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc.ll‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,8 @@ define dso_local i128 @load_atomic_i128_aligned_unordered(ptr %ptr) {
236236
; -O0: stxp w8, x0, x1, [x9]
237237
;
238238
; -O1-LABEL: load_atomic_i128_aligned_unordered:
239-
; -O1: ldxp x0, x1, [x8]
240-
; -O1: stxp w9, x0, x1, [x8]
239+
; -O1: ldxp x8, x1, [x0]
240+
; -O1: stxp w9, x8, x1, [x0]
241241
%r = load atomic i128, ptr %ptr unordered, align 16
242242
ret i128 %r
243243
}
@@ -251,8 +251,8 @@ define dso_local i128 @load_atomic_i128_aligned_unordered_const(ptr readonly %pt
251251
; -O0: stxp w8, x0, x1, [x9]
252252
;
253253
; -O1-LABEL: load_atomic_i128_aligned_unordered_const:
254-
; -O1: ldxp x0, x1, [x8]
255-
; -O1: stxp w9, x0, x1, [x8]
254+
; -O1: ldxp x8, x1, [x0]
255+
; -O1: stxp w9, x8, x1, [x0]
256256
%r = load atomic i128, ptr %ptr unordered, align 16
257257
ret i128 %r
258258
}
@@ -266,8 +266,8 @@ define dso_local i128 @load_atomic_i128_aligned_monotonic(ptr %ptr) {
266266
; -O0: stxp w8, x0, x1, [x9]
267267
;
268268
; -O1-LABEL: load_atomic_i128_aligned_monotonic:
269-
; -O1: ldxp x0, x1, [x8]
270-
; -O1: stxp w9, x0, x1, [x8]
269+
; -O1: ldxp x8, x1, [x0]
270+
; -O1: stxp w9, x8, x1, [x0]
271271
%r = load atomic i128, ptr %ptr monotonic, align 16
272272
ret i128 %r
273273
}
@@ -281,8 +281,8 @@ define dso_local i128 @load_atomic_i128_aligned_monotonic_const(ptr readonly %pt
281281
; -O0: stxp w8, x0, x1, [x9]
282282
;
283283
; -O1-LABEL: load_atomic_i128_aligned_monotonic_const:
284-
; -O1: ldxp x0, x1, [x8]
285-
; -O1: stxp w9, x0, x1, [x8]
284+
; -O1: ldxp x8, x1, [x0]
285+
; -O1: stxp w9, x8, x1, [x0]
286286
%r = load atomic i128, ptr %ptr monotonic, align 16
287287
ret i128 %r
288288
}
@@ -296,8 +296,8 @@ define dso_local i128 @load_atomic_i128_aligned_acquire(ptr %ptr) {
296296
; -O0: stxp w8, x0, x1, [x9]
297297
;
298298
; -O1-LABEL: load_atomic_i128_aligned_acquire:
299-
; -O1: ldaxp x0, x1, [x8]
300-
; -O1: stxp w9, x0, x1, [x8]
299+
; -O1: ldaxp x8, x1, [x0]
300+
; -O1: stxp w9, x8, x1, [x0]
301301
%r = load atomic i128, ptr %ptr acquire, align 16
302302
ret i128 %r
303303
}
@@ -311,8 +311,8 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
311311
; -O0: stxp w8, x0, x1, [x9]
312312
;
313313
; -O1-LABEL: load_atomic_i128_aligned_acquire_const:
314-
; -O1: ldaxp x0, x1, [x8]
315-
; -O1: stxp w9, x0, x1, [x8]
314+
; -O1: ldaxp x8, x1, [x0]
315+
; -O1: stxp w9, x8, x1, [x0]
316316
%r = load atomic i128, ptr %ptr acquire, align 16
317317
ret i128 %r
318318
}
@@ -326,8 +326,8 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
326326
; -O0: stlxp w8, x0, x1, [x9]
327327
;
328328
; -O1-LABEL: load_atomic_i128_aligned_seq_cst:
329-
; -O1: ldaxp x0, x1, [x8]
330-
; -O1: stlxp w9, x0, x1, [x8]
329+
; -O1: ldaxp x8, x1, [x0]
330+
; -O1: stlxp w9, x8, x1, [x0]
331331
%r = load atomic i128, ptr %ptr seq_cst, align 16
332332
ret i128 %r
333333
}
@@ -341,8 +341,8 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr)
341341
; -O0: stlxp w8, x0, x1, [x9]
342342
;
343343
; -O1-LABEL: load_atomic_i128_aligned_seq_cst_const:
344-
; -O1: ldaxp x0, x1, [x8]
345-
; -O1: stlxp w9, x0, x1, [x8]
344+
; -O1: ldaxp x8, x1, [x0]
345+
; -O1: stlxp w9, x8, x1, [x0]
346346
%r = load atomic i128, ptr %ptr seq_cst, align 16
347347
ret i128 %r
348348
}

‎llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-v8a.ll‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,8 @@ define dso_local i128 @load_atomic_i128_aligned_unordered(ptr %ptr) {
236236
; -O0: stxp w8, x0, x1, [x9]
237237
;
238238
; -O1-LABEL: load_atomic_i128_aligned_unordered:
239-
; -O1: ldxp x0, x1, [x8]
240-
; -O1: stxp w9, x0, x1, [x8]
239+
; -O1: ldxp x8, x1, [x0]
240+
; -O1: stxp w9, x8, x1, [x0]
241241
%r = load atomic i128, ptr %ptr unordered, align 16
242242
ret i128 %r
243243
}
@@ -251,8 +251,8 @@ define dso_local i128 @load_atomic_i128_aligned_unordered_const(ptr readonly %pt
251251
; -O0: stxp w8, x0, x1, [x9]
252252
;
253253
; -O1-LABEL: load_atomic_i128_aligned_unordered_const:
254-
; -O1: ldxp x0, x1, [x8]
255-
; -O1: stxp w9, x0, x1, [x8]
254+
; -O1: ldxp x8, x1, [x0]
255+
; -O1: stxp w9, x8, x1, [x0]
256256
%r = load atomic i128, ptr %ptr unordered, align 16
257257
ret i128 %r
258258
}
@@ -266,8 +266,8 @@ define dso_local i128 @load_atomic_i128_aligned_monotonic(ptr %ptr) {
266266
; -O0: stxp w8, x0, x1, [x9]
267267
;
268268
; -O1-LABEL: load_atomic_i128_aligned_monotonic:
269-
; -O1: ldxp x0, x1, [x8]
270-
; -O1: stxp w9, x0, x1, [x8]
269+
; -O1: ldxp x8, x1, [x0]
270+
; -O1: stxp w9, x8, x1, [x0]
271271
%r = load atomic i128, ptr %ptr monotonic, align 16
272272
ret i128 %r
273273
}
@@ -281,8 +281,8 @@ define dso_local i128 @load_atomic_i128_aligned_monotonic_const(ptr readonly %pt
281281
; -O0: stxp w8, x0, x1, [x9]
282282
;
283283
; -O1-LABEL: load_atomic_i128_aligned_monotonic_const:
284-
; -O1: ldxp x0, x1, [x8]
285-
; -O1: stxp w9, x0, x1, [x8]
284+
; -O1: ldxp x8, x1, [x0]
285+
; -O1: stxp w9, x8, x1, [x0]
286286
%r = load atomic i128, ptr %ptr monotonic, align 16
287287
ret i128 %r
288288
}
@@ -296,8 +296,8 @@ define dso_local i128 @load_atomic_i128_aligned_acquire(ptr %ptr) {
296296
; -O0: stxp w8, x0, x1, [x9]
297297
;
298298
; -O1-LABEL: load_atomic_i128_aligned_acquire:
299-
; -O1: ldaxp x0, x1, [x8]
300-
; -O1: stxp w9, x0, x1, [x8]
299+
; -O1: ldaxp x8, x1, [x0]
300+
; -O1: stxp w9, x8, x1, [x0]
301301
%r = load atomic i128, ptr %ptr acquire, align 16
302302
ret i128 %r
303303
}
@@ -311,8 +311,8 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
311311
; -O0: stxp w8, x0, x1, [x9]
312312
;
313313
; -O1-LABEL: load_atomic_i128_aligned_acquire_const:
314-
; -O1: ldaxp x0, x1, [x8]
315-
; -O1: stxp w9, x0, x1, [x8]
314+
; -O1: ldaxp x8, x1, [x0]
315+
; -O1: stxp w9, x8, x1, [x0]
316316
%r = load atomic i128, ptr %ptr acquire, align 16
317317
ret i128 %r
318318
}
@@ -326,8 +326,8 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
326326
; -O0: stlxp w8, x0, x1, [x9]
327327
;
328328
; -O1-LABEL: load_atomic_i128_aligned_seq_cst:
329-
; -O1: ldaxp x0, x1, [x8]
330-
; -O1: stlxp w9, x0, x1, [x8]
329+
; -O1: ldaxp x8, x1, [x0]
330+
; -O1: stlxp w9, x8, x1, [x0]
331331
%r = load atomic i128, ptr %ptr seq_cst, align 16
332332
ret i128 %r
333333
}
@@ -341,8 +341,8 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr)
341341
; -O0: stlxp w8, x0, x1, [x9]
342342
;
343343
; -O1-LABEL: load_atomic_i128_aligned_seq_cst_const:
344-
; -O1: ldaxp x0, x1, [x8]
345-
; -O1: stlxp w9, x0, x1, [x8]
344+
; -O1: ldaxp x8, x1, [x0]
345+
; -O1: stlxp w9, x8, x1, [x0]
346346
%r = load atomic i128, ptr %ptr seq_cst, align 16
347347
ret i128 %r
348348
}

0 commit comments

Comments
 (0)