@@ -1569,6 +1569,69 @@ void SCEVExpander::rememberInstruction(Value *I) {
15691569 DoInsert (I);
15701570}
15711571
1572+ void SCEVExpander::replaceCongruentIVInc (
1573+ PHINode *&Phi, PHINode *&OrigPhi, Loop *L, const DominatorTree *DT,
1574+ SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
1575+ BasicBlock *LatchBlock = L->getLoopLatch ();
1576+ if (!LatchBlock)
1577+ return ;
1578+
1579+ Instruction *OrigInc =
1580+ dyn_cast<Instruction>(OrigPhi->getIncomingValueForBlock (LatchBlock));
1581+ Instruction *IsomorphicInc =
1582+ dyn_cast<Instruction>(Phi->getIncomingValueForBlock (LatchBlock));
1583+ if (!OrigInc || !IsomorphicInc)
1584+ return ;
1585+
1586+ // If this phi has the same width but is more canonical, replace the
1587+ // original with it. As part of the "more canonical" determination,
1588+ // respect a prior decision to use an IV chain.
1589+ if (OrigPhi->getType () == Phi->getType () &&
1590+ !(ChainedPhis.count (Phi) ||
1591+ isExpandedAddRecExprPHI (OrigPhi, OrigInc, L)) &&
1592+ (ChainedPhis.count (Phi) ||
1593+ isExpandedAddRecExprPHI (Phi, IsomorphicInc, L))) {
1594+ std::swap (OrigPhi, Phi);
1595+ std::swap (OrigInc, IsomorphicInc);
1596+ }
1597+
1598+ // Replacing the congruent phi is sufficient because acyclic
1599+ // redundancy elimination, CSE/GVN, should handle the
1600+ // rest. However, once SCEV proves that a phi is congruent,
1601+ // it's often the head of an IV user cycle that is isomorphic
1602+ // with the original phi. It's worth eagerly cleaning up the
1603+ // common case of a single IV increment so that DeleteDeadPHIs
1604+ // can remove cycles that had postinc uses.
1605+ // Because we may potentially introduce a new use of OrigIV that didn't
1606+ // exist before at this point, its poison flags need readjustment.
1607+ const SCEV *TruncExpr =
1608+ SE.getTruncateOrNoop (SE.getSCEV (OrigInc), IsomorphicInc->getType ());
1609+ if (OrigInc == IsomorphicInc || TruncExpr != SE.getSCEV (IsomorphicInc) ||
1610+ !SE.LI .replacementPreservesLCSSAForm (IsomorphicInc, OrigInc) ||
1611+ !hoistIVInc (OrigInc, IsomorphicInc,
1612+ /* RecomputePoisonFlags*/ true ))
1613+ return ;
1614+
1615+ SCEV_DEBUG_WITH_TYPE (DebugType,
1616+ dbgs () << " INDVARS: Eliminated congruent iv.inc: "
1617+ << *IsomorphicInc << ' \n ' );
1618+ Value *NewInc = OrigInc;
1619+ if (OrigInc->getType () != IsomorphicInc->getType ()) {
1620+ BasicBlock::iterator IP;
1621+ if (PHINode *PN = dyn_cast<PHINode>(OrigInc))
1622+ IP = PN->getParent ()->getFirstInsertionPt ();
1623+ else
1624+ IP = OrigInc->getNextNonDebugInstruction ()->getIterator ();
1625+
1626+ IRBuilder<> Builder (IP->getParent (), IP);
1627+ Builder.SetCurrentDebugLocation (IsomorphicInc->getDebugLoc ());
1628+ NewInc =
1629+ Builder.CreateTruncOrBitCast (OrigInc, IsomorphicInc->getType (), IVName);
1630+ }
1631+ IsomorphicInc->replaceAllUsesWith (NewInc);
1632+ DeadInsts.emplace_back (IsomorphicInc);
1633+ }
1634+
15721635// / replaceCongruentIVs - Check for congruent phis in this loop header and
15731636// / replace them with their most canonical representative. Return the number of
15741637// / phis eliminated.
@@ -1654,60 +1717,7 @@ SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
16541717 if (OrigPhiRef->getType ()->isPointerTy () != Phi->getType ()->isPointerTy ())
16551718 continue ;
16561719
1657- if (BasicBlock *LatchBlock = L->getLoopLatch ()) {
1658- Instruction *OrigInc = dyn_cast<Instruction>(
1659- OrigPhiRef->getIncomingValueForBlock (LatchBlock));
1660- Instruction *IsomorphicInc =
1661- dyn_cast<Instruction>(Phi->getIncomingValueForBlock (LatchBlock));
1662-
1663- if (OrigInc && IsomorphicInc) {
1664- // If this phi has the same width but is more canonical, replace the
1665- // original with it. As part of the "more canonical" determination,
1666- // respect a prior decision to use an IV chain.
1667- if (OrigPhiRef->getType () == Phi->getType () &&
1668- !(ChainedPhis.count (Phi) ||
1669- isExpandedAddRecExprPHI (OrigPhiRef, OrigInc, L)) &&
1670- (ChainedPhis.count (Phi) ||
1671- isExpandedAddRecExprPHI (Phi, IsomorphicInc, L))) {
1672- std::swap (OrigPhiRef, Phi);
1673- std::swap (OrigInc, IsomorphicInc);
1674- }
1675- // Replacing the congruent phi is sufficient because acyclic
1676- // redundancy elimination, CSE/GVN, should handle the
1677- // rest. However, once SCEV proves that a phi is congruent,
1678- // it's often the head of an IV user cycle that is isomorphic
1679- // with the original phi. It's worth eagerly cleaning up the
1680- // common case of a single IV increment so that DeleteDeadPHIs
1681- // can remove cycles that had postinc uses.
1682- // Because we may potentially introduce a new use of OrigIV that didn't
1683- // exist before at this point, its poison flags need readjustment.
1684- const SCEV *TruncExpr =
1685- SE.getTruncateOrNoop (SE.getSCEV (OrigInc), IsomorphicInc->getType ());
1686- if (OrigInc != IsomorphicInc &&
1687- TruncExpr == SE.getSCEV (IsomorphicInc) &&
1688- SE.LI .replacementPreservesLCSSAForm (IsomorphicInc, OrigInc) &&
1689- hoistIVInc (OrigInc, IsomorphicInc, /* RecomputePoisonFlags*/ true )) {
1690- SCEV_DEBUG_WITH_TYPE (
1691- DebugType, dbgs () << " INDVARS: Eliminated congruent iv.inc: "
1692- << *IsomorphicInc << ' \n ' );
1693- Value *NewInc = OrigInc;
1694- if (OrigInc->getType () != IsomorphicInc->getType ()) {
1695- BasicBlock::iterator IP;
1696- if (PHINode *PN = dyn_cast<PHINode>(OrigInc))
1697- IP = PN->getParent ()->getFirstInsertionPt ();
1698- else
1699- IP = OrigInc->getNextNonDebugInstruction ()->getIterator ();
1700-
1701- IRBuilder<> Builder (IP->getParent (), IP);
1702- Builder.SetCurrentDebugLocation (IsomorphicInc->getDebugLoc ());
1703- NewInc = Builder.CreateTruncOrBitCast (
1704- OrigInc, IsomorphicInc->getType (), IVName);
1705- }
1706- IsomorphicInc->replaceAllUsesWith (NewInc);
1707- DeadInsts.emplace_back (IsomorphicInc);
1708- }
1709- }
1710- }
1720+ replaceCongruentIVInc (Phi, OrigPhiRef, L, DT, DeadInsts);
17111721 SCEV_DEBUG_WITH_TYPE (DebugType,
17121722 dbgs () << " INDVARS: Eliminated congruent iv: " << *Phi
17131723 << ' \n ' );
0 commit comments