Merged main:e072cffe3b63 into amd-gfx:c0b9c269979b

Local branch amd-gfx c0b9c26 Merged main:93c2577020fe into amd-gfx:fb65d18eb39e
Remote branch main e072cff [RISCV] Add immop_oneuse PatLeaf. Use it to replace some PatLeafs with ImmLeaf/IntImmLeaf. NFC (llvm#120804)

Author: SC llvm team

clang/include/clang/Sema/SemaConcept.h

@@ -135,31 +135,20 @@ struct NormalizedConstraint {
     return *this;
   }
 
-  bool isAtomic() const { return Constraint.is<AtomicConstraint *>(); }
+  bool isAtomic() const { return llvm::isa<AtomicConstraint *>(Constraint); }
   bool isFoldExpanded() const {
-    return Constraint.is<FoldExpandedConstraint *>();
+    return llvm::isa<FoldExpandedConstraint *>(Constraint);
   }
-  bool isCompound() const { return Constraint.is<CompoundConstraint>(); }
+  bool isCompound() const { return llvm::isa<CompoundConstraint>(Constraint); }
 
-  CompoundConstraintKind getCompoundKind() const {
-    assert(isCompound() && "getCompoundKind on a non-compound constraint..");
-    return Constraint.get<CompoundConstraint>().getInt();
-  }
+  CompoundConstraintKind getCompoundKind() const;
 
   NormalizedConstraint &getLHS() const;
   NormalizedConstraint &getRHS() const;
 
-  AtomicConstraint *getAtomicConstraint() const {
-    assert(isAtomic() &&
-           "getAtomicConstraint called on non-atomic constraint.");
-    return Constraint.get<AtomicConstraint *>();
-  }
+  AtomicConstraint *getAtomicConstraint() const;
 
-  FoldExpandedConstraint *getFoldExpandedConstraint() const {
-    assert(isFoldExpanded() &&
-           "getFoldExpandedConstraint called on non-fold-expanded constraint.");
-    return Constraint.get<FoldExpandedConstraint *>();
-  }
+  FoldExpandedConstraint *getFoldExpandedConstraint() const;
 
 private:
   static std::optional<NormalizedConstraint>

clang/lib/Sema/SemaConcept.cpp

@@ -1958,3 +1958,21 @@ concepts::TypeRequirement::TypeRequirement(TypeSourceInfo *T) :
     Value(T),
     Status(T->getType()->isInstantiationDependentType() ? SS_Dependent
                                                         : SS_Satisfied) {}
+
+NormalizedConstraint::CompoundConstraintKind
+NormalizedConstraint::getCompoundKind() const {
+  assert(isCompound() && "getCompoundKind on a non-compound constraint..");
+  return cast<CompoundConstraint>(Constraint).getInt();
+}
+
+AtomicConstraint *NormalizedConstraint::getAtomicConstraint() const {
+  assert(isAtomic() && "getAtomicConstraint called on non-atomic constraint.");
+  return cast<AtomicConstraint *>(Constraint);
+}
+
+FoldExpandedConstraint *
+NormalizedConstraint::getFoldExpandedConstraint() const {
+  assert(isFoldExpanded() &&
+         "getFoldExpandedConstraint called on non-fold-expanded constraint.");
+  return cast<FoldExpandedConstraint *>(Constraint);
+}

llvm/include/llvm/Config/llvm-config.h.cmake

@@ -16,7 +16,7 @@
 
 /* Indicate that this is LLVM compiled from the amd-gfx branch. */
 #define LLVM_HAVE_BRANCH_AMD_GFX
-#define LLVM_MAIN_REVISION 522209
+#define LLVM_MAIN_REVISION 522213
 
 /* Define if LLVM_ENABLE_DUMP is enabled */
 #cmakedefine LLVM_ENABLE_DUMP

llvm/lib/Target/RISCV/RISCVInstrInfo.td

@@ -440,20 +440,10 @@ def GIImmSubFrom32 : GICustomOperandRenderer<"renderImmSubFrom32">,
 // in which imm = imm0 + imm1 and both imm0 and imm1 are simm12. We make imm0
 // as large as possible and imm1 as small as possible so that we might be able
 // to use c.addi for the small immediate.
-def AddiPair : PatLeaf<(imm), [{
-  if (!N->hasOneUse())
-    return false;
+def AddiPair : ImmLeaf<XLenVT, [{
   // The immediate operand must be in range [-4096,-2049] or [2048,4094].
-  int64_t Imm = N->getSExtValue();
   return (-4096 <= Imm && Imm <= -2049) || (2048 <= Imm && Imm <= 4094);
-}]> {
-  let GISelPredicateCode = [{
-    if (!MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()))
-      return false;
-    int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
-    return (-4096 <= Imm && Imm <= -2049) || (2048 <= Imm && Imm <= 4094);
-  }];
-}
+}]>;
 
 // Return imm - (imm < 0 ? -2048 : 2047).
 def AddiPairImmSmall : SDNodeXForm<imm, [{
@@ -492,55 +482,23 @@ def GIXLenSubTrailingOnes : GICustomOperandRenderer<"renderXLenSubTrailingOnes">
 
 // Checks if this mask is a non-empty sequence of ones starting at the
 // most/least significant bit with the remainder zero and exceeds simm32/simm12.
-def LeadingOnesMask : PatLeaf<(imm), [{
-  if (!N->hasOneUse())
-    return false;
-  return !isInt<32>(N->getSExtValue()) && isMask_64(~N->getSExtValue());
-}], TrailingZeros> {
-  let GISelPredicateCode = [{
-    if (!MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()))
-      return false;
-    const auto &MO = MI.getOperand(1);
-    return !isInt<32>(MO.getCImm()->getSExtValue()) && 
-           isMask_64(~MO.getCImm()->getSExtValue());
-  }];
-}
+def LeadingOnesMask : ImmLeaf<XLenVT, [{
+  return !isInt<32>(Imm) && isMask_64(~Imm);
+}], TrailingZeros>;
 
-def TrailingOnesMask : PatLeaf<(imm), [{
-  if (!N->hasOneUse())
-    return false;
-  return !isInt<12>(N->getSExtValue()) && isMask_64(N->getZExtValue());
-}], XLenSubTrailingOnes> {
-  let GISelPredicateCode = [{
-    if (!MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()))
-      return false;
-    const auto &MO = MI.getOperand(1);
-    return !isInt<12>(MO.getCImm()->getSExtValue()) &&
-           isMask_64(MO.getCImm()->getZExtValue());
-  }];
-}
+def TrailingOnesMask : IntImmLeaf<XLenVT, [{
+  return !isInt<12>(Imm.getSExtValue()) && isMask_64(Imm.getZExtValue());
+}], XLenSubTrailingOnes>;
 
 // Similar to LeadingOnesMask, but only consider leading ones in the lower 32
 // bits.
-def LeadingOnesWMask : PatLeaf<(imm), [{
-  if (!N->hasOneUse())
-    return false;
+def LeadingOnesWMask : ImmLeaf<XLenVT, [{
   // If the value is a uint32 but not an int32, it must have bit 31 set and
   // bits 63:32 cleared. After that we're looking for a shifted mask but not
   // an all ones mask.
-  int64_t Imm = N->getSExtValue();
   return !isInt<32>(Imm) && isUInt<32>(Imm) && isShiftedMask_64(Imm) &&
          Imm != UINT64_C(0xffffffff);
-}], TrailingZeros> {
-  let GISelPredicateCode = [{
-    if (!MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()))
-      return false;
-    const auto &MO = MI.getOperand(1);
-    int64_t Imm = MO.getCImm()->getSExtValue();
-    return !isInt<32>(Imm) && isUInt<32>(Imm) && isShiftedMask_64(Imm) &&
-           Imm != UINT64_C(0xffffffff);
-  }];
-}
+}], TrailingZeros>;
 
 //===----------------------------------------------------------------------===//
 // Instruction Formats
@@ -1350,6 +1308,14 @@ def 33signbits_node : PatLeaf<(i64 GPR:$src), [{
   return CurDAG->ComputeNumSignBits(SDValue(N, 0)) > 32;
 }]>;
 
+class immop_oneuse<ImmLeaf leaf> : PatLeaf<(leaf), [{
+  return N->hasOneUse();
+}]> {
+  let GISelPredicateCode = [{
+    return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg());
+  }];
+}
+
 /// Simple arithmetic operations
 
 def : PatGprGpr<add, ADD>;
@@ -1387,10 +1353,12 @@ def : Pat<(XLenVT (sub 0, (and_oneuse GPR:$rs, 1))),
                 (ImmSubFromXLen (XLenVT 1)))>;
 
 // AND with leading/trailing ones mask exceeding simm32/simm12.
-def : Pat<(i64 (and GPR:$rs, LeadingOnesMask:$mask)),
-          (SLLI (i64 (SRLI $rs, LeadingOnesMask:$mask)), LeadingOnesMask:$mask)>;
-def : Pat<(XLenVT (and GPR:$rs, TrailingOnesMask:$mask)),
-          (SRLI (XLenVT (SLLI $rs, TrailingOnesMask:$mask)), TrailingOnesMask:$mask)>;
+def : Pat<(i64 (and GPR:$rs, immop_oneuse<LeadingOnesMask>:$mask)),
+          (SLLI (i64 (SRLI $rs, (TrailingZeros imm:$mask))),
+                (TrailingZeros imm:$mask))>;
+def : Pat<(XLenVT (and GPR:$rs, immop_oneuse<TrailingOnesMask>:$mask)),
+          (SRLI (XLenVT (SLLI $rs, (XLenSubTrailingOnes imm:$mask))),
+                (XLenSubTrailingOnes imm:$mask))>;
 
 // Match both a plain shift and one where the shift amount is masked (this is
 // typically introduced when the legalizer promotes the shift amount and
@@ -1981,8 +1949,9 @@ def u32simm12 : ImmLeaf<XLenVT, [{
 
 let Predicates = [IsRV64] in {
 
-def : Pat<(i64 (and GPR:$rs, LeadingOnesWMask:$mask)),
-          (SLLI (i64 (SRLIW $rs, LeadingOnesWMask:$mask)), LeadingOnesWMask:$mask)>;
+def : Pat<(i64 (and GPR:$rs, immop_oneuse<LeadingOnesWMask>:$mask)),
+          (SLLI (i64 (SRLIW $rs, (TrailingZeros imm:$mask))),
+                (TrailingZeros imm:$mask))>;
 
 /// sext and zext
 
@@ -2081,15 +2050,15 @@ def KCFI_CHECK
 }
 
 /// Simple optimization
-def : Pat<(XLenVT (add GPR:$rs1, (AddiPair:$rs2))),
-          (ADDI (XLenVT (ADDI GPR:$rs1, (AddiPairImmLarge AddiPair:$rs2))),
-                (AddiPairImmSmall GPR:$rs2))>;
+def : Pat<(XLenVT (add GPR:$rs1, immop_oneuse<AddiPair>:$rs2)),
+          (ADDI (XLenVT (ADDI GPR:$rs1, (AddiPairImmLarge imm:$rs2))),
+                (AddiPairImmSmall imm:$rs2))>;
 
 let Predicates = [IsRV64] in {
 // Select W instructions if only the lower 32-bits of the result are used.
-def : Pat<(binop_allwusers<add> GPR:$rs1, (AddiPair:$rs2)),
-          (ADDIW (i64 (ADDIW GPR:$rs1, (AddiPairImmLarge AddiPair:$rs2))),
-                 (AddiPairImmSmall AddiPair:$rs2))>;
+def : Pat<(binop_allwusers<add> GPR:$rs1, immop_oneuse<AddiPair>:$rs2),
+          (ADDIW (i64 (ADDIW GPR:$rs1, (AddiPairImmLarge imm:$rs2))),
+                 (AddiPairImmSmall imm:$rs2))>;
 }
 
 //===----------------------------------------------------------------------===//

llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

@@ -61,10 +61,16 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
   case Instruction::ICmp:
   case VPInstruction::ActiveLaneMask:
     return inferScalarType(R->getOperand(1));
+  case VPInstruction::ComputeReductionResult: {
+    auto *PhiR = cast<VPReductionPHIRecipe>(R->getOperand(0));
+    auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
+    return OrigPhi->getType();
+  }
   case VPInstruction::ExplicitVectorLength:
     return Type::getIntNTy(Ctx, 32);
   case VPInstruction::FirstOrderRecurrenceSplice:
   case VPInstruction::Not:
+  case VPInstruction::ResumePhi:
     return SetResultTyFromOp();
   case VPInstruction::ExtractFromEnd: {
     Type *BaseTy = inferScalarType(R->getOperand(0));

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -629,7 +629,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
     Value *IncomingFromOtherPreds =
         State.get(getOperand(1), /* IsScalar */ true);
     auto *NewPhi =
-        Builder.CreatePHI(IncomingFromOtherPreds->getType(), 2, Name);
+        Builder.CreatePHI(State.TypeAnalysis.inferScalarType(this), 2, Name);
     BasicBlock *VPlanPred =
         State.CFG
             .VPBB2IRBB[cast<VPBasicBlock>(getParent()->getPredecessors()[0])];

mlir/lib/Dialect/Arith/Utils/Utils.cpp

@@ -66,7 +66,7 @@ mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc,
     int64_t inputIndex = it.index();
     // Call get<Value>() under the assumption that we're not casting
     // dynamism.
-    Value indexGroupSize = inputShape[inputIndex].get<Value>();
+    Value indexGroupSize = cast<Value>(inputShape[inputIndex]);
     Value indexGroupStaticSizesProduct =
         b.create<arith::ConstantIndexOp>(loc, indexGroupStaticSizesProductInt);
     Value dynamicDimSize = b.createOrFold<arith::DivUIOp>(

mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp

@@ -174,7 +174,7 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
             resultDims[llvm::cast<OpResult>(shapedValue).getResultNumber()];
         for (const auto &dim : enumerate(tensorType.getShape()))
           if (ShapedType::isDynamic(dim.value()))
-            dynamicSizes.push_back(shape[dim.index()].get<Value>());
+            dynamicSizes.push_back(cast<Value>(shape[dim.index()]));
       }
     }
 

mlir/lib/Dialect/DLTI/DLTI.cpp

@@ -312,7 +312,7 @@ combineOneSpec(DataLayoutSpecInterface spec,
       continue;
     }
 
-    Type typeSample = kvp.second.front().getKey().get<Type>();
+    Type typeSample = cast<Type>(kvp.second.front().getKey());
     assert(&typeSample.getDialect() !=
                typeSample.getContext()->getLoadedDialect<BuiltinDialect>() &&
            "unexpected data layout entry for built-in type");
@@ -325,7 +325,7 @@ combineOneSpec(DataLayoutSpecInterface spec,
   }
 
   for (const auto &kvp : newEntriesForID) {
-    StringAttr id = kvp.second.getKey().get<StringAttr>();
+    StringAttr id = cast<StringAttr>(kvp.second.getKey());
     Dialect *dialect = id.getReferencedDialect();
     if (!entriesForID.count(id)) {
       entriesForID[id] = kvp.second;
@@ -574,7 +574,7 @@ class TargetDataLayoutInterface : public DataLayoutDialectInterface {
 
   LogicalResult verifyEntry(DataLayoutEntryInterface entry,
                             Location loc) const final {
-    StringRef entryName = entry.getKey().get<StringAttr>().strref();
+    StringRef entryName = cast<StringAttr>(entry.getKey()).strref();
     if (entryName == DLTIDialect::kDataLayoutEndiannessKey) {
       auto value = dyn_cast<StringAttr>(entry.getValue());
       if (value &&

mlir/lib/Dialect/GPU/TransformOps/Utils.cpp

@@ -113,16 +113,17 @@ static GpuIdBuilderFnType commonLinearIdBuilderFn(int64_t multiplicity = 1) {
     // clang-format on
 
     // Return n-D ids for indexing and 1-D size + id for predicate generation.
-    return IdBuilderResult{
-        /*mappingIdOps=*/ids,
-        /*availableMappingSizes=*/
-        SmallVector<int64_t>{computeProduct(originalBasis)},
-        // `forallMappingSizes` iterate in the scaled basis, they need to be
-        // scaled back into the original basis to provide tight
-        // activeMappingSizes quantities for predication.
-        /*activeMappingSizes=*/
-        SmallVector<int64_t>{computeProduct(forallMappingSizes) * multiplicity},
-        /*activeIdOps=*/SmallVector<Value>{linearId.get<Value>()}};
+      return IdBuilderResult{
+          /*mappingIdOps=*/ids,
+          /*availableMappingSizes=*/
+          SmallVector<int64_t>{computeProduct(originalBasis)},
+          // `forallMappingSizes` iterate in the scaled basis, they need to be
+          // scaled back into the original basis to provide tight
+          // activeMappingSizes quantities for predication.
+          /*activeMappingSizes=*/
+          SmallVector<int64_t>{computeProduct(forallMappingSizes) *
+                               multiplicity},
+          /*activeIdOps=*/SmallVector<Value>{cast<Value>(linearId)}};
   };
 
   return res;
@@ -144,9 +145,8 @@ static GpuIdBuilderFnType common3DIdBuilderFn(int64_t multiplicity = 1) {
     // In the 3-D mapping case, scale the first dimension by the multiplicity.
     SmallVector<Value> scaledIds = ids;
     AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
-    scaledIds[0] = affine::makeComposedFoldedAffineApply(
-                       rewriter, loc, d0.floorDiv(multiplicity), {scaledIds[0]})
-                       .get<Value>();
+    scaledIds[0] = cast<Value>(affine::makeComposedFoldedAffineApply(
+        rewriter, loc, d0.floorDiv(multiplicity), {scaledIds[0]}));
     // In the 3-D mapping case, unscale the first dimension by the multiplicity.
     SmallVector<int64_t> forallMappingSizeInOriginalBasis(forallMappingSizes);
     forallMappingSizeInOriginalBasis[0] *= multiplicity;