Skip to content

[IVDesc] Unify calls to min-max patterns (NFC) #142769

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[IVDesc] Unify calls to min-max patterns (NFC) #142769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d6eb6d
[IVDesc] Unify calls to min-max patterns
artagnon Jun 4, 2025
16689f4
[IVDesc] Fix serious bug; patch is now NFC
artagnon Jun 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions llvm/include/llvm/Analysis/IVDescriptors.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ class RecurrenceDescriptor {
/// corresponding to a min(X, Y) or max(X, Y), matching the recurrence kind \p
/// Kind. \p Prev specifies the description of an already processed select
/// instruction, so its corresponding cmp can be matched to it.
LLVM_ABI static InstDesc isMinMaxPattern(Instruction *I, RecurKind Kind,
const InstDesc &Prev);
LLVM_ABI static InstDesc isMinMaxPattern(Instruction *I, const InstDesc &Prev,
FastMathFlags FuncFMF);

/// Returns a struct describing whether the instruction is either a
/// Select(ICmp(A, B), X, Y), or
Expand Down
176 changes: 83 additions & 93 deletions llvm/lib/Analysis/IVDescriptors.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,7 @@ bool RecurrenceDescriptor::AddReductionVar(
(!isConditionalRdxPattern(Kind, UI).isRecurrence() &&
!isAnyOfPattern(TheLoop, Phi, UI, IgnoredVal)
.isRecurrence() &&
!isMinMaxPattern(UI, Kind, IgnoredVal).isRecurrence())))
!isMinMaxPattern(UI, IgnoredVal, FuncFMF).isRecurrence())))
return false;

// Remember that we completed the cycle.
Expand Down Expand Up @@ -744,13 +744,10 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
}

RecurrenceDescriptor::InstDesc
RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
const InstDesc &Prev) {
RecurrenceDescriptor::isMinMaxPattern(Instruction *I, const InstDesc &Prev,
FastMathFlags FuncFMF) {
assert((isa<CmpInst>(I) || isa<SelectInst>(I) || isa<CallInst>(I)) &&
"Expected a cmp or select or call instruction");
if (!isMinMaxRecurrenceKind(Kind))
return InstDesc(false, I);

// We must handle the select(cmp()) as a single instruction. Advance to the
// select.
if (match(I, m_OneUse(m_Cmp()))) {
Expand All @@ -763,32 +760,80 @@ RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
!match(I, m_Select(m_OneUse(m_Cmp()), m_Value(), m_Value())))
return InstDesc(false, I);

bool HasRequiredFMF =
(FuncFMF.noNaNs() && FuncFMF.noSignedZeros()) ||
(isa<FPMathOperator>(I) && I->hasNoNaNs() && I->hasNoSignedZeros());

// Look for a min/max pattern.
if (match(I, m_UMin(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::UMin, I);
if (match(I, m_UMax(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::UMax, I);
if (match(I, m_SMax(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::SMax, I);
if (match(I, m_SMin(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::SMin, I);
if (match(I, m_OrdOrUnordFMin(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMin, I);
if (match(I, m_OrdOrUnordFMax(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMax, I);
if (match(I, m_FMinNum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMin, I);
if (match(I, m_FMaxNum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMax, I);
if (match(I, m_FMinimumNum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMinimumNum, I);
if (match(I, m_FMaximumNum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMaximumNum, I);
if (match(I, m_FMinimum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMinimum, I);
if (match(I, m_FMaximum(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMaximum, I);
auto CheckKind = [I, HasRequiredFMF](RecurKind Kind) {
switch (Kind) {
case RecurKind::UMin:
if (match(I, m_UMin(m_Value(), m_Value())))
return InstDesc(I, RecurKind::UMin);
break;
case RecurKind::UMax:
if (match(I, m_UMax(m_Value(), m_Value())))
return InstDesc(I, RecurKind::UMax);
break;
case RecurKind::SMax:
if (match(I, m_SMax(m_Value(), m_Value())))
return InstDesc(I, RecurKind::SMax);
break;
case RecurKind::SMin:
if (match(I, m_SMin(m_Value(), m_Value())))
return InstDesc(I, RecurKind::SMin);
break;

// minimum/minnum and maximum/maxnum intrinsics do not require nsz and nnan
// flags since NaN and signed zeroes are propagated in the intrinsic
// implementation.
case RecurKind::FMinimumNum:
if (match(I, m_FMinimumNum(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMinimumNum);
break;
case RecurKind::FMaximumNum:
if (match(I, m_FMaximumNum(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMaximumNum);
break;
case RecurKind::FMinimum:
if (match(I, m_FMinimum(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMinimum);
break;
case RecurKind::FMaximum:
if (match(I, m_FMaximum(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMaximum);
break;

case RecurKind::FMin:
if (HasRequiredFMF && match(I, m_OrdOrUnordFMin(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMin);
if (HasRequiredFMF && match(I, m_FMinNum(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMin);
break;
case RecurKind::FMax:
if (HasRequiredFMF && match(I, m_OrdOrUnordFMax(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMax);
if (HasRequiredFMF && match(I, m_FMaxNum(m_Value(), m_Value())))
return InstDesc(I, RecurKind::FMax);
break;
default:
break;
}
return InstDesc(false, I);
};

// The previous recurrence, if there is one, must match the current
// recurrence.
if (Prev.getRecKind() != RecurKind::None)
return CheckKind(Prev.getRecKind());
for (RecurKind K :
{RecurKind::SMin, RecurKind::SMax, RecurKind::UMin, RecurKind::UMax,
RecurKind::FMin, RecurKind::FMax, RecurKind::FMinimum,
RecurKind::FMaximum, RecurKind::FMinimumNum, RecurKind::FMaximumNum}) {
InstDesc D = CheckKind(K);
if (D.getRecKind() != RecurKind::None)
return D;
}
return InstDesc(false, I);
}

Expand Down Expand Up @@ -883,24 +928,9 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
case Instruction::Call:
if (isAnyOfRecurrenceKind(Kind))
return isAnyOfPattern(L, OrigPhi, I, Prev);
auto HasRequiredFMF = [&]() {
if (FuncFMF.noNaNs() && FuncFMF.noSignedZeros())
return true;
if (isa<FPMathOperator>(I) && I->hasNoNaNs() && I->hasNoSignedZeros())
return true;
// minimum/minnum and maximum/maxnum intrinsics do not require nsz and nnan
// flags since NaN and signed zeroes are propagated in the intrinsic
// implementation.
return match(I, m_Intrinsic<Intrinsic::minimum>(m_Value(), m_Value())) ||
match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value())) ||
match(I,
m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
};
if (isIntMinMaxRecurrenceKind(Kind) ||
(HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
return isMinMaxPattern(I, Kind, Prev);
else if (isFMulAddIntrinsic(I))
if (isMinMaxRecurrenceKind(Kind))
return isMinMaxPattern(I, Prev, FuncFMF);
if (isFMulAddIntrinsic(I))
return InstDesc(Kind == RecurKind::FMulAdd, I,
I->hasAllowReassoc() ? nullptr : I);
return InstDesc(false, I);
Expand Down Expand Up @@ -961,22 +991,14 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
}
if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a SMAX reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a SMIN reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a UMAX reduction PHI." << *Phi << "\n");
LLVM_DEBUG(dbgs() << "Found an integral MinMax reduction PHI." << *Phi
<< "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, FMF, RedDes, DB, AC, DT,
if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
LLVM_DEBUG(dbgs() << "Found a floating-point MinMax reduction PHI." << *Phi
<< "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
Expand All @@ -1000,43 +1022,11 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a float MAX reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMaximum, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a float MAXIMUM reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMinimum, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a float MINIMUM reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMaximumNum, TheLoop, FMF, RedDes, DB, AC,
DT, SE)) {
LLVM_DEBUG(dbgs() << "Found a float MAXIMUMNUM reduction PHI." << *Phi
<< "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMinimumNum, TheLoop, FMF, RedDes, DB, AC,
DT, SE)) {
LLVM_DEBUG(dbgs() << "Found a float MINIMUMNUM reduction PHI." << *Phi
<< "\n");
return true;
}

// Not a reduction of known type.
return false;
Expand Down