diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 90db89f745e89..4a0f8720266e1 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -800,8 +800,13 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy, Value *Ptr, PredicatedScalarEvolution &PSE) { // The access function must stride over the innermost loop. if (Lp != AR->getLoop()) { - LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not striding over innermost loop " - << *Ptr << " SCEV: " << *AR << "\n"); + LLVM_DEBUG({ + dbgs() << "LAA: Bad stride - Not striding over innermost loop "; + if (Ptr) + dbgs() << *Ptr << " "; + + dbgs() << "SCEV: " << *AR << "\n"; + }); return std::nullopt; } @@ -811,8 +816,12 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy, // Calculate the pointer stride and check if it is constant. const SCEVConstant *C = dyn_cast(Step); if (!C) { - LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not a constant strided " << *Ptr - << " SCEV: " << *AR << "\n"); + LLVM_DEBUG({ + dbgs() << "LAA: Bad stride - Not a constant strided "; + if (Ptr) + dbgs() << *Ptr << " "; + dbgs() << "SCEV: " << *AR << "\n"; + }); return std::nullopt; } @@ -839,8 +848,8 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy, static bool isNoWrapGEP(Value *Ptr, PredicatedScalarEvolution &PSE, const Loop *L); -/// Check whether \p AR is a non-wrapping AddRec, or if \p Ptr is a non-wrapping -/// GEP. +/// Check whether \p AR is a non-wrapping AddRec. If \p Ptr is not nullptr, use +/// informating from the IR pointer value to determine no-wrap. static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, Value *Ptr, Type *AccessTy, const Loop *L, bool Assume, std::optional Stride = std::nullopt) { @@ -848,12 +857,12 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, if (AR->getNoWrapFlags(SCEV::NoWrapMask)) return true; - if (PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW)) + if (Ptr && PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW)) return true; // The address calculation must not wrap. Otherwise, a dependence could be // inverted. - if (isNoWrapGEP(Ptr, PSE, L)) + if (Ptr && isNoWrapGEP(Ptr, PSE, L)) return true; // An nusw getelementptr that is an AddRec cannot wrap. If it would wrap, @@ -861,7 +870,7 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, // location will be larger than half the pointer index type space. In that // case, the GEP would be poison and any memory access dependent on it would // be immediate UB when executed. - if (auto *GEP = dyn_cast(Ptr); + if (auto *GEP = dyn_cast_if_present(Ptr); GEP && GEP->hasNoUnsignedSignedWrap()) return true; @@ -877,7 +886,7 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, return true; } - if (Assume) { + if (Ptr && Assume) { PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW); LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n" << "LAA: Pointer: " << *Ptr << "\n" @@ -1119,6 +1128,7 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck, SmallVector> TranslatedPtrs = findForkedPointer(PSE, StridesMap, Ptr, TheLoop); + assert(!TranslatedPtrs.empty() && "must have some translated pointers"); /// Check whether all pointers can participate in a runtime bounds check. They /// must either be invariant or AddRecs. If ShouldCheckWrap is true, they also @@ -1144,13 +1154,10 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck, // When we run after a failing dependency check we have to make sure // we don't have wrapping pointers. - if (ShouldCheckWrap) { - // Skip wrap checking when translating pointers. - if (TranslatedPtrs.size() > 1) - return false; - - if (!isNoWrap(PSE, AR, Ptr, AccessTy, TheLoop, Assume)) - return false; + if (ShouldCheckWrap && + !isNoWrap(PSE, AR, TranslatedPtrs.size() == 1 ? Ptr : nullptr, AccessTy, + TheLoop, Assume)) { + return false; } } diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll index 5e9dc7f2b91cc..38b7389ae9083 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis-forked-pointers.ll @@ -83,10 +83,52 @@ exit: define void @dependency_check_and_runtime_checks_needed_select_of_ptr_add_recs(ptr %a, ptr %b, ptr %c, i64 %offset, i64 %n) { ; CHECK-LABEL: 'dependency_check_and_runtime_checks_needed_select_of_ptr_add_recs' ; CHECK-NEXT: loop: -; CHECK-NEXT: Report: cannot check memory dependencies at runtime +; CHECK-NEXT: Memory dependences are safe with run-time checks ; CHECK-NEXT: Dependences: ; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Check 0: +; CHECK-NEXT: Comparing group ([[GRP5:0x[0-9a-f]+]]): +; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv +; CHECK-NEXT: Against group ([[GRP6:0x[0-9a-f]+]]): +; CHECK-NEXT: %select = select i1 %cmp, ptr %gep.b, ptr %gep.c +; CHECK-NEXT: Check 1: +; CHECK-NEXT: Comparing group ([[GRP5]]): +; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv +; CHECK-NEXT: Against group ([[GRP7:0x[0-9a-f]+]]): +; CHECK-NEXT: %select = select i1 %cmp, ptr %gep.b, ptr %gep.c +; CHECK-NEXT: Check 2: +; CHECK-NEXT: Comparing group ([[GRP5]]): +; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv +; CHECK-NEXT: Against group ([[GRP8:0x[0-9a-f]+]]): +; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset +; CHECK-NEXT: Check 3: +; CHECK-NEXT: Comparing group ([[GRP6]]): +; CHECK-NEXT: %select = select i1 %cmp, ptr %gep.b, ptr %gep.c +; CHECK-NEXT: Against group ([[GRP7]]): +; CHECK-NEXT: %select = select i1 %cmp, ptr %gep.b, ptr %gep.c +; CHECK-NEXT: Check 4: +; CHECK-NEXT: Comparing group ([[GRP6]]): +; CHECK-NEXT: %select = select i1 %cmp, ptr %gep.b, ptr %gep.c +; CHECK-NEXT: Against group ([[GRP8]]): +; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset +; CHECK-NEXT: Check 5: +; CHECK-NEXT: Comparing group ([[GRP7]]): +; CHECK-NEXT: %select = select i1 %cmp, ptr %gep.b, ptr %gep.c +; CHECK-NEXT: Against group ([[GRP8]]): +; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset ; CHECK-NEXT: Grouped accesses: +; CHECK-NEXT: Group [[GRP5]]: +; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a)) +; CHECK-NEXT: Member: {%a,+,4}<%loop> +; CHECK-NEXT: Group [[GRP6]]: +; CHECK-NEXT: (Low: %b High: ((4 * %n) + %b)) +; CHECK-NEXT: Member: {%b,+,4}<%loop> +; CHECK-NEXT: Group [[GRP7]]: +; CHECK-NEXT: (Low: %c High: ((4 * %n) + %c)) +; CHECK-NEXT: Member: {%c,+,4}<%loop> +; CHECK-NEXT: Group [[GRP8]]: +; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a)) +; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop> ; CHECK-EMPTY: ; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. ; CHECK-NEXT: SCEV assumptions: