Revert "[SelectionDAG] Expand CTTZ_ELTS[_ZERO_POISON] and handle splitting" (#188220)

Reverts llvm/llvm-project#185605

Buildbot failures caused by ISel crashes in
https://lab.llvm.org/buildbot/#/builders/157/builds/45416 and
https://lab.llvm.org/buildbot/#/builders/10/builds/25156

Author: lukel97

llvm/include/llvm/CodeGen/BasicTTIImpl.h

@@ -2133,8 +2133,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
         VScaleRange = getVScaleRange(I->getCaller(), 64);
 
       unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
-          getTLI()->getValueType(DL, RetTy), ArgType.getVectorElementCount(),
-          ZeroIsPoison, &VScaleRange);
+          RetTy, ArgType.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
       Type *NewEltTy = IntegerType::getIntNTy(RetTy->getContext(), EltWidth);
 
       // Create the new vector type & get the vector length

llvm/include/llvm/CodeGen/ISDOpcodes.h

@@ -1580,7 +1580,7 @@ enum NodeType {
   EXPERIMENTAL_VECTOR_HISTOGRAM,
 
   /// Returns the number of number of trailing (least significant) zero elements
-  /// in a vector. Has a single vector operand. The result is poison if the
+  /// in a vector. Has a single i1 vector operand. The result is poison if the
   /// return type isn't wide enough to hold the maximum number of elements in
   /// the input vector.
   CTTZ_ELTS,

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -498,7 +498,7 @@ class LLVM_ABI TargetLoweringBase {
 
   /// Return the minimum number of bits required to hold the maximum possible
   /// number of trailing zero vector elements.
-  unsigned getBitWidthForCttzElements(EVT RetVT, ElementCount EC,
+  unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC,
                                       bool ZeroIsPoison,
                                       const ConstantRange *VScaleRange) const;
 
@@ -5820,10 +5820,6 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
   /// temporarily, advance store position, before re-loading the final vector.
   SDValue expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const;
 
-  /// Expand a CTTZ_ELTS or CTTZ_ELTS_ZERO_POISON by calculating (VL - i) for
-  /// each active lane (i), getting the maximum and subtracting it from VL.
-  SDValue expandCttzElts(SDNode *Node, SelectionDAG &DAG) const;
-
   /// Expands PARTIAL_REDUCE_S/UMLA nodes to a series of simpler operations,
   /// consisting of zext/sext, extract_subvector, mul and add operations.
   SDValue expandPartialReduceMLA(SDNode *Node, SelectionDAG &DAG) const;

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

@@ -2167,10 +2167,6 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::VECTOR_FIND_LAST_ACTIVE:
     Res = PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(N, OpNo);
     break;
-  case ISD::CTTZ_ELTS:
-  case ISD::CTTZ_ELTS_ZERO_POISON:
-    Res = PromoteIntOp_CTTZ_ELTS(N);
-    break;
   case ISD::GET_ACTIVE_LANE_MASK:
     Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(N);
     break;
@@ -3004,11 +3000,6 @@ SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N,
   return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
 }
 
-SDValue DAGTypeLegalizer::PromoteIntOp_CTTZ_ELTS(SDNode *N) {
-  SDValue Op = GetPromotedInteger(N->getOperand(0));
-  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
-}
-
 SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) {
   SmallVector<SDValue, 1> NewOps(N->ops());
   NewOps[0] = ZExtPromotedInteger(N->getOperand(0));

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

@@ -419,7 +419,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N, unsigned OpNo);
-  SDValue PromoteIntOp_CTTZ_ELTS(SDNode *N);
   SDValue PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N);
   SDValue PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N);
   SDValue PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N, unsigned OpNo);
@@ -988,7 +987,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue SplitVecOp_FPOpDifferentTypes(SDNode *N);
   SDValue SplitVecOp_CMP(SDNode *N);
   SDValue SplitVecOp_FP_TO_XINT_SAT(SDNode *N);
-  SDValue SplitVecOp_CttzElts(SDNode *N);
   SDValue SplitVecOp_VP_CttzElements(SDNode *N);
   SDValue SplitVecOp_VECTOR_HISTOGRAM(SDNode *N);
   SDValue SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N);

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

@@ -517,8 +517,6 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::VECREDUCE_FMIN:
   case ISD::VECREDUCE_FMINIMUM:
   case ISD::VECREDUCE_FMUL:
-  case ISD::CTTZ_ELTS:
-  case ISD::CTTZ_ELTS_ZERO_POISON:
   case ISD::VECTOR_FIND_LAST_ACTIVE:
     Action = TLI.getOperationAction(Node->getOpcode(),
                                     Node->getOperand(0).getValueType());
@@ -1356,10 +1354,6 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
   case ISD::VECTOR_COMPRESS:
     Results.push_back(TLI.expandVECTOR_COMPRESS(Node, DAG));
     return;
-  case ISD::CTTZ_ELTS:
-  case ISD::CTTZ_ELTS_ZERO_POISON:
-    Results.push_back(TLI.expandCttzElts(Node, DAG));
-    return;
   case ISD::VECTOR_FIND_LAST_ACTIVE:
     Results.push_back(TLI.expandVectorFindLastActive(Node, DAG));
     return;

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

@@ -3742,10 +3742,6 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::VP_REDUCE_FMINIMUM:
     Res = SplitVecOp_VP_REDUCE(N, OpNo);
     break;
-  case ISD::CTTZ_ELTS:
-  case ISD::CTTZ_ELTS_ZERO_POISON:
-    Res = SplitVecOp_CttzElts(N);
-    break;
   case ISD::VP_CTTZ_ELTS:
   case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
     Res = SplitVecOp_VP_CttzElements(N);
@@ -4832,26 +4828,6 @@ SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
 }
 
-SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
-  SDLoc DL(N);
-  EVT ResVT = N->getValueType(0);
-
-  SDValue Lo, Hi;
-  SDValue VecOp = N->getOperand(0);
-  GetSplitVector(VecOp, Lo, Hi);
-
-  // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
-  // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
-  SDValue ResLo = DAG.getNode(ISD::CTTZ_ELTS, DL, ResVT, Lo);
-  SDValue VL =
-      DAG.getElementCount(DL, ResVT, Lo.getValueType().getVectorElementCount());
-  SDValue ResLoNotVL =
-      DAG.getSetCC(DL, getSetCCResultType(ResVT), ResLo, VL, ISD::SETNE);
-  SDValue ResHi = DAG.getNode(N->getOpcode(), DL, ResVT, Hi);
-  return DAG.getSelect(DL, ResVT, ResLoNotVL, ResLo,
-                       DAG.getNode(ISD::ADD, DL, ResVT, VL, ResHi));
-}
-
 SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
   SDLoc DL(N);
   EVT ResVT = N->getValueType(0);

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

@@ -8324,13 +8324,55 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     return;
   }
   case Intrinsic::experimental_cttz_elts: {
+    auto DL = getCurSDLoc();
     SDValue Op = getValue(I.getOperand(0));
+    EVT OpVT = Op.getValueType();
     EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
     bool ZeroIsPoison =
         !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
-    SDValue Ret =
-        DAG.getNode(ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON : ISD::CTTZ_ELTS,
-                    sdl, RetTy, Op);
+
+    if (!TLI.shouldExpandCttzElements(OpVT)) {
+      SDValue Ret = DAG.getNode(ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
+                                             : ISD::CTTZ_ELTS,
+                                sdl, RetTy, Op);
+      setValue(&I, Ret);
+      return;
+    }
+
+    if (OpVT.getScalarType() != MVT::i1) {
+      // Compare the input vector elements to zero & use to count trailing zeros
+      SDValue AllZero = DAG.getConstant(0, DL, OpVT);
+      OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
+                              OpVT.getVectorElementCount());
+      Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
+    }
+
+    // If the zero-is-poison flag is set, we can assume the upper limit
+    // of the result is VF-1.
+    ConstantRange VScaleRange(1, true); // Dummy value.
+    if (isa<ScalableVectorType>(I.getOperand(0)->getType()))
+      VScaleRange = getVScaleRange(I.getCaller(), 64);
+    unsigned EltWidth = TLI.getBitWidthForCttzElements(
+        I.getType(), OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
+
+    MVT NewEltTy = MVT::getIntegerVT(EltWidth);
+
+    // Create the new vector type & get the vector length
+    EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
+                                 OpVT.getVectorElementCount());
+
+    SDValue VL =
+        DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
+
+    SDValue StepVec = DAG.getStepVector(DL, NewVT);
+    SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
+    SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
+    SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, Op);
+    SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
+    SDValue Max = DAG.getNode(ISD::VECREDUCE_UMAX, DL, NewEltTy, And);
+    SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
+
+    SDValue Ret = DAG.getZExtOrTrunc(Sub, DL, RetTy);
 
     setValue(&I, Ret);
     return;

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

@@ -10078,7 +10078,8 @@ SDValue TargetLowering::expandVectorFindLastActive(SDNode *N,
     VScaleRange = getVScaleRange(&DAG.getMachineFunction().getFunction(), 64);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   uint64_t EltWidth = TLI.getBitWidthForCttzElements(
-      EVT(getVectorIdxTy(DAG.getDataLayout())), MaskVT.getVectorElementCount(),
+      EVT(getVectorIdxTy(DAG.getDataLayout())).getTypeForEVT(*DAG.getContext()),
+      MaskVT.getVectorElementCount(),
       /*ZeroIsPoison=*/true, &VScaleRange);
   // If the step vector element type is smaller than the mask element type,
   // use the mask type directly to avoid widening issues.
@@ -12540,64 +12541,6 @@ SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
   return DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo);
 }
 
-SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
-  SDLoc DL(Node);
-  EVT VT = Node->getValueType(0);
-  SDValue Op = Node->getOperand(0);
-  EVT OpVT = Op.getValueType();
-
-  if (OpVT.getVectorElementType() != MVT::i1) {
-    // Compare the input vector elements to zero & use to count trailing zeros.
-    SDValue AllZero = DAG.getConstant(0, DL, OpVT);
-    EVT I1OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
-                                  OpVT.getVectorElementCount());
-    // If cttz_elts is legal for the i1 type, use it instead of expanding.
-    if (isOperationLegalOrCustom(Node->getOpcode(), I1OpVT)) {
-      Op = DAG.getSetCC(DL, I1OpVT, Op, AllZero, ISD::SETNE);
-      return DAG.getNode(Node->getOpcode(), DL, VT, Op);
-    }
-
-    Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
-  }
-
-  // If the zero-is-poison flag is set, we can assume the upper limit
-  // of the result is VF-1.
-  bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
-  ConstantRange VScaleRange(1, true); // Dummy value.
-  if (OpVT.isScalableVector())
-    VScaleRange = getVScaleRange(&DAG.getMachineFunction().getFunction(), 64);
-  unsigned EltWidth = getBitWidthForCttzElements(
-      VT, OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
-
-  EVT NewEltVT = MVT::getIntegerVT(EltWidth);
-
-  // Create the new vector type & get the vector length
-  EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT,
-                               OpVT.getVectorElementCount());
-
-  // Promote types now to avoid redundant zexts.
-  if (getTypeAction(NewVT.getSimpleVT()) == TypePromoteInteger) {
-    NewVT = getTypeToTransformTo(*DAG.getContext(), NewVT);
-    NewEltVT = NewVT.getVectorElementType();
-  }
-  if (getTypeAction(NewEltVT.getSimpleVT()) == TypePromoteInteger)
-    NewEltVT = getTypeToTransformTo(*DAG.getContext(), NewEltVT);
-
-  SDValue VL = DAG.getElementCount(DL, NewEltVT, NewVT.getVectorElementCount());
-
-  SDValue StepVec = DAG.getStepVector(DL, NewVT);
-  SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
-  SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
-  SDValue Ext = DAG.getSExtOrTrunc(Op, DL, NewVT);
-  SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
-  SDValue Max =
-      DAG.getNode(ISD::VECREDUCE_UMAX, DL, NewVT.getVectorElementType(), And);
-  SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltVT, VL,
-                            DAG.getZExtOrTrunc(Max, DL, NewEltVT));
-
-  return DAG.getZExtOrTrunc(Sub, DL, VT);
-}
-
 SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
                                                SelectionDAG &DAG) const {
   SDLoc DL(N);

llvm/lib/CodeGen/TargetLoweringBase.cpp

@@ -1347,7 +1347,7 @@ bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS,
 }
 
 unsigned TargetLoweringBase::getBitWidthForCttzElements(
-    EVT RetVT, ElementCount EC, bool ZeroIsPoison,
+    Type *RetTy, ElementCount EC, bool ZeroIsPoison,
     const ConstantRange *VScaleRange) const {
   // Find the smallest "sensible" element type to use for the expansion.
   ConstantRange CR(APInt(64, EC.getKnownMinValue()));
@@ -1357,7 +1357,7 @@ unsigned TargetLoweringBase::getBitWidthForCttzElements(
   if (ZeroIsPoison)
     CR = CR.subtract(APInt(64, 1));
 
-  unsigned EltWidth = RetVT.getScalarSizeInBits();
+  unsigned EltWidth = RetTy->getScalarSizeInBits();
   EltWidth = std::min(EltWidth, CR.getActiveBits());
   EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);