Skip to content

Commit 533e6bb

Browse files
committed
[VPlan] Simplify live-ins if they are SCEVConstant.
The legacy cost model in some parts checks if any of the operands are constants via SCEV. Update VPlan construction to replace live-ins that are constants via SCEV with such constants. This means VPlans (and codegen) reflects what we computing the cost of and removes another case where the legacy and VPlan cost model diverged. Fixes #105722.
1 parent 84497c6 commit 533e6bb

File tree

2 files changed

+71
-0
lines changed

2 files changed

+71
-0
lines changed

llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "VPlan.h"
1414
#include "llvm/ADT/DenseMap.h"
1515
#include "llvm/ADT/PointerUnion.h"
16+
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
1617
#include "llvm/IR/IRBuilder.h"
1718

1819
namespace llvm {
@@ -173,6 +174,11 @@ class VPRecipeBuilder {
173174
if (auto *R = Ingredient2Recipe.lookup(I))
174175
return R->getVPSingleValue();
175176
}
177+
ScalarEvolution &SE = *PSE.getSE();
178+
if (!isa<Constant>(V) && SE.isSCEVable(V->getType()))
179+
if (auto *C = dyn_cast<SCEVConstant>(PSE.getSE()->getSCEV(V)))
180+
return Plan.getOrAddLiveIn(C->getValue());
181+
176182
return Plan.getOrAddLiveIn(V);
177183
}
178184
};

llvm/test/Transforms/LoopVectorize/X86/cost-model.ll

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -931,6 +931,71 @@ exit:
931931
ret void
932932
}
933933

934+
; Test case for https://github.com/llvm/llvm-project/issues/105722.
935+
define i64 @live_in_known_1_via_scev() {
936+
; CHECK-LABEL: @live_in_known_1_via_scev(
937+
; CHECK-NEXT: entry:
938+
; CHECK-NEXT: [[SEL:%.*]] = select i1 false, i32 3, i32 0
939+
; CHECK-NEXT: br label [[PH:%.*]]
940+
; CHECK: ph:
941+
; CHECK-NEXT: [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
942+
; CHECK-NEXT: [[N:%.*]] = add nuw nsw i32 [[SEL]], 6
943+
; CHECK-NEXT: [[P_EXT:%.*]] = zext nneg i32 [[P]] to i64
944+
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
945+
; CHECK: vector.ph:
946+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
947+
; CHECK: vector.body:
948+
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
949+
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ <i64 3, i64 1, i64 1, i64 1>, [[VECTOR_PH]] ], [ [[VEC_PHI]], [[VECTOR_BODY]] ]
950+
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
951+
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
952+
; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
953+
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], <i32 5, i32 5, i32 5, i32 5>
954+
; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[VEC_PHI]], <4 x i64> [[VEC_PHI]]
955+
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
956+
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
957+
; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
958+
; CHECK: middle.block:
959+
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> [[TMP1]])
960+
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
961+
; CHECK: scalar.ph:
962+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, [[MIDDLE_BLOCK]] ], [ 0, [[PH]] ]
963+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 3, [[PH]] ]
964+
; CHECK-NEXT: br label [[LOOP:%.*]]
965+
; CHECK: loop:
966+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
967+
; CHECK-NEXT: [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_MUL:%.*]], [[LOOP]] ]
968+
; CHECK-NEXT: [[RED_MUL]] = mul nsw i64 [[RED]], [[P_EXT]]
969+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
970+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
971+
; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
972+
; CHECK: exit:
973+
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[RED_MUL]], [[LOOP]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
974+
; CHECK-NEXT: ret i64 [[RES]]
975+
;
976+
entry:
977+
%sel = select i1 false, i32 3, i32 0
978+
br label %ph
979+
980+
ph:
981+
%p = phi i32 [ 1, %entry ]
982+
%N = add nuw nsw i32 %sel, 6
983+
%p.ext = zext nneg i32 %p to i64
984+
br label %loop
985+
986+
loop:
987+
%iv = phi i32 [ 0, %ph ], [ %iv.next, %loop ]
988+
%red = phi i64 [ 3, %ph ], [ %red.mul, %loop ]
989+
%red.mul = mul nsw i64 %red, %p.ext
990+
%iv.next = add nuw nsw i32 %iv, 1
991+
%ec = icmp eq i32 %iv.next, %N
992+
br i1 %ec, label %exit, label %loop
993+
994+
exit:
995+
%res = phi i64 [ %red.mul, %loop ]
996+
ret i64 %res
997+
}
998+
934999
declare void @llvm.assume(i1 noundef) #0
9351000

9361001
attributes #0 = { "target-cpu"="penryn" }

0 commit comments

Comments
 (0)