Merge commit '9a491048ed813410b8ea79216ee920580f83c3de'

whitneywhtsang · whitneywhtsang · commit d812777081e1 · 2025-01-31T03:53:51.000Z
diff --git a/cmake/llvm-hash.txt b/cmake/llvm-hash.txt
@@ -1 +1 @@
-e2402615a5a76d46a433dfcc1de10b38a1263c9d
+c118864223c6309378cd704f3406533474c2759f
diff --git a/include/triton/Conversion/MLIRTypes.h b/include/triton/Conversion/MLIRTypes.h
@@ -26,17 +26,15 @@ inline Type f32Ty(MLIRContext *ctx) { return Float32Type::get(ctx); }
 inline Type f64Ty(MLIRContext *ctx) { return Float64Type::get(ctx); }
 inline Type bf16Ty(MLIRContext *ctx) { return BFloat16Type::get(ctx); }
 
-inline bool isFloat(Type type) {
-  return type.isF32() || type.isF64() || type.isF16() || type.isF128() ||
-         type.isBF16() || type.isFloat8E4M3B11FNUZ() || type.isFloat8E4M3FN() ||
-         type.isFloat8E4M3FNUZ() || type.isFloat8E5M2() ||
-         type.isFloat8E5M2FNUZ();
+inline bool isFloat8(Type type) {
+  return isa<Float8E4M3B11FNUZType, Float8E4M3FNType, Float8E4M3FNUZType,
+             Float8E5M2Type, Float8E5M2FNUZType>(type);
 }
 
-inline bool isFloat8(Type type) {
-  return type.isFloat8E4M3B11FNUZ() || type.isFloat8E4M3FN() ||
-         type.isFloat8E4M3FNUZ() || type.isFloat8E5M2() ||
-         type.isFloat8E5M2FNUZ();
+inline bool isFloat(Type type) {
+  return type.isF32() || type.isF64() || type.isF16() || type.isF128() ||
+         type.isBF16() || llvm::isa<Float8E4M3B11FNUZType>(type) ||
+         isFloat8(type);
 }
 
 inline bool isInt(Type type) { return type.isIntOrFloat() && !isFloat(type); }
diff --git a/lib/Analysis/Utility.cpp b/lib/Analysis/Utility.cpp
@@ -756,14 +756,14 @@ bool supportMMA(triton::DotOp op, int version) {
       return false;
     if (!(numWarps % 4 == 0 && retShapePerCTA[rank - 2] % 64 == 0 &&
           retShapePerCTA[rank - 1] % 8 == 0 &&
-          (aElemTy.isFloat8E5M2() || aElemTy.isFloat8E4M3FN() ||
+          (llvm::isa<Float8E5M2Type, Float8E4M3FNType>(aElemTy) ||
            aElemTy.isInteger(8) || aElemTy.isF16() || aElemTy.isBF16() ||
            aElemTy.isF32()))) {
       return false;
     }
     // We cannot use MMA_V3 if we need to accumulate in F32 within the MMA op.
     if (op.getMaxNumImpreciseAcc() < 32 &&
-        (aElemTy.isFloat8E5M2() || aElemTy.isFloat8E4M3FN()) &&
+        (llvm::isa<Float8E5M2Type, Float8E4M3FNType>(aElemTy)) &&
         cast<RankedTensorType>(op.getType()).getElementType().isF32()) {
       return false;
     }
@@ -784,8 +784,8 @@ bool supportMMA(Value value, int version) {
       cast<triton::gpu::TensorOrMemDesc>(value.getType()).getElementType();
   // FP8 is not natively supported on all mma versions but it can always be
   // promoted to fp16 therefore we can always support it.
-  bool isFP8 = elemTy.isFloat8E5M2() || elemTy.isFloat8E4M3FN() ||
-               elemTy.isFloat8E5M2FNUZ() || elemTy.isFloat8E4M3FNUZ();
+  bool isFP8 = llvm::isa<Float8E5M2Type, Float8E4M3FNType, Float8E5M2FNUZType,
+                         Float8E4M3FNUZType>(elemTy);
   return isFP8 || elemTy.isF16() || elemTy.isBF16() ||
          (elemTy.isF32() && version >= 2) ||
          (elemTy.isInteger(8) && version >= 2);
diff --git a/lib/Dialect/TritonGPU/Transforms/AccelerateMatmul.cpp b/lib/Dialect/TritonGPU/Transforms/AccelerateMatmul.cpp
@@ -632,7 +632,7 @@ static void decomposeMixedModeDotOp(ModuleOp mod, int computeCapability) {
     NvidiaMmaEncodingAttr mmaLayout =
         dyn_cast<NvidiaMmaEncodingAttr>(D.getType().getEncoding());
     if (mmaLayout) {
-      bool isNativeFP8 = AElType.isFloat8E5M2() || AElType.isFloat8E4M3FN();
+      bool isNativeFP8 = llvm::isa<Float8E5M2Type, Float8E4M3FNType>(AElType);
       // promote operands for sm < 89 since fp8 mma is not natively supported
       // promote operands for sm >= 90 when mma is not v3
       if (!isNativeFP8 ||
diff --git a/lib/Dialect/TritonGPU/Transforms/Utility.cpp b/lib/Dialect/TritonGPU/Transforms/Utility.cpp
@@ -45,9 +45,9 @@ SmallVector<unsigned, 3> mmaVersionToInstrShape(int version,
     SmallVector<unsigned> validN;
 
     // MMAv3 with larger instruction shape is preferred.
-    if (eltType.isFloat8E5M2() || eltType.isFloat8E4M3FN() ||
-        eltType.isFloat8E4M3FNUZ() || eltType.isF16() || eltType.isBF16() ||
-        eltType.isF32()) {
+    if (llvm::isa<Float8E5M2Type, Float8E4M3FNType, Float8E4M3FNUZType>(
+            eltType) ||
+        eltType.isF16() || eltType.isBF16() || eltType.isF32()) {
       validN.assign({256, 248, 240, 232, 224, 216, 208, 200, 192, 184, 176,
                      168, 160, 152, 144, 136, 128, 120, 112, 104, 96,  88,
                      80,  72,  64,  56,  48,  40,  32,  24,  16,  8});
diff --git a/lib/Dialect/TritonNvidiaGPU/IR/Ops.cpp b/lib/Dialect/TritonNvidiaGPU/IR/Ops.cpp
@@ -77,8 +77,8 @@ bool WarpGroupDotOp::needsPartialAccumulator() {
   const auto &d = getD();
   auto aTensorTy = cast<triton::gpu::TensorOrMemDesc>(a.getType());
   auto aElTy = cast<triton::gpu::TensorOrMemDesc>(a.getType()).getElementType();
-  bool isFP8 = aElTy.isFloat8E5M2() || aElTy.isFloat8E4M3FN() ||
-               aElTy.isFloat8E5M2FNUZ() || aElTy.isFloat8E4M3FNUZ();
+  bool isFP8 = llvm::isa<Float8E5M2Type, Float8E4M3FNType, Float8E5M2FNUZType,
+                         Float8E4M3FNUZType>(aElTy);
   bool accFP32 =
       cast<triton::gpu::TensorOrMemDesc>(d.getType()).getElementType().isF32();
   uint32_t maxNumImpreciseAcc = getMaxNumImpreciseAcc();
diff --git a/third_party/amd/lib/TritonAMDGPUToLLVM/ElementwiseOpToLLVM.cpp b/third_party/amd/lib/TritonAMDGPUToLLVM/ElementwiseOpToLLVM.cpp
@@ -1043,17 +1043,16 @@ struct FpToFpOpConversion
       return outVals;
     }
     size_t numElements = 4;
-    if (srcElementType.isFloat8E4M3FN() || dstElementType.isFloat8E4M3FN() ||
-        srcElementType.isFloat8E4M3FNUZ() ||
-        dstElementType.isFloat8E4M3FNUZ() ||
-        srcElementType.isFloat8E5M2FNUZ() ||
-        dstElementType.isFloat8E5M2FNUZ()) {
+    if (llvm::isa<Float8E4M3FNType, Float8E4M3FNUZType, Float8E5M2FNUZType>(
+            srcElementType) ||
+        llvm::isa<Float8E4M3FNType, Float8E4M3FNUZType, Float8E5M2FNUZType>(
+            dstElementType)) {
       numElements = 2;
     }
     bool useFP16IntermediateSrc =
-        srcElementType.isF32() && !(isaFamily == AMD::ISAFamily::CDNA3 &&
-                                    (dstElementType.isFloat8E4M3FNUZ() ||
-                                     dstElementType.isFloat8E5M2FNUZ()));
+        srcElementType.isF32() &&
+        !(isaFamily == AMD::ISAFamily::CDNA3 &&
+          (llvm::isa<Float8E4M3FNUZType, Float8E5M2FNUZType>(dstElementType)));
     bool isDstFP32 = dstElementType.isF32();
     Type srcType = useFP16IntermediateSrc ? f16_ty : srcElementType;
     Type dstType = isDstFP32 ? f16_ty : dstElementType;
diff --git a/third_party/amd/lib/TritonAMDGPUTransforms/AccelerateAMDMatmul.cpp b/third_party/amd/lib/TritonAMDGPUTransforms/AccelerateAMDMatmul.cpp
@@ -416,7 +416,7 @@ class BlockedToMFMA : public OpRewritePattern<tt::DotOp> {
     // store instructions, except for fp8 matmul kernels due to regression
     // TODO (lixun): investigate the regression and enable this feature again
     auto aElemTy = mfmaInstr.getElementTypeA();
-    bool isFP8 = aElemTy.isFloat8E5M2FNUZ() || aElemTy.isFloat8E4M3FNUZ();
+    bool isFP8 = llvm::isa<Float8E5M2FNUZType, Float8E4M3FNUZType>(aElemTy);
     bool isTransposed = isChainDot(dotOp) || !isFP8;
     mfmaEnc = ttg::AMDMfmaEncodingAttr::get(
         oldRetType.getContext(),
diff --git a/third_party/amd/lib/TritonAMDGPUTransforms/MfmaGroup.cpp b/third_party/amd/lib/TritonAMDGPUTransforms/MfmaGroup.cpp
@@ -20,19 +20,24 @@ static MfmaTypeId chooseAppropriateMfmaId(mlir::Type dataTypeA,
   if (dataTypeA.isInteger(8) && dataTypeB.isInteger(8)) {
     return MfmaTypeId::I8TyId;
   }
-  if (dataTypeA.isFloat8E4M3FNUZ() && dataTypeB.isFloat8E4M3FNUZ()) {
+  if (llvm::isa<Float8E4M3FNUZType>(dataTypeA) &&
+      llvm::isa<Float8E4M3FNUZType>(dataTypeB)) {
     return MfmaTypeId::Fp8Fp8TyId;
   }
-  if (dataTypeA.isFloat8E4M3FNUZ() && dataTypeB.isFloat8E5M2FNUZ()) {
+  if (llvm::isa<Float8E4M3FNUZType>(dataTypeA) &&
+      llvm::isa<Float8E5M2FNUZType>(dataTypeB)) {
     return MfmaTypeId::Fp8Bf8TyId;
   }
-  if (dataTypeA.isFloat8E5M2FNUZ() && dataTypeB.isFloat8E4M3FNUZ()) {
+  if (llvm::isa<Float8E5M2FNUZType>(dataTypeA) &&
+      llvm::isa<Float8E4M3FNUZType>(dataTypeB)) {
     return MfmaTypeId::Bf8Fp8TyId;
   }
-  if (dataTypeA.isFloat8E5M2FNUZ() && dataTypeB.isFloat8E5M2FNUZ()) {
+  if (llvm::isa<Float8E5M2FNUZType>(dataTypeA) &&
+      llvm::isa<Float8E5M2FNUZType>(dataTypeB)) {
     return MfmaTypeId::Bf8Bf8TyId;
   }
-  if (dataTypeA.isFloat8E5M2() && dataTypeB.isFloat8E5M2()) {
+  if (llvm::isa<Float8E5M2Type>(dataTypeA) &&
+      llvm::isa<Float8E5M2Type>(dataTypeB)) {
     return MfmaTypeId::Fp16TyId;
   }
   llvm_unreachable("Unsupported input argument type.");
diff --git a/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/DotOpToLLVM/MMAv2.cpp b/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/DotOpToLLVM/MMAv2.cpp
@@ -303,17 +303,17 @@ TensorCoreType getMmaType(triton::DotOp op) {
       return TensorCoreType::FP32_FP16_FP16_FP32;
     if (aTy.getElementType().isBF16() && bTy.getElementType().isBF16())
       return TensorCoreType::FP32_BF16_BF16_FP32;
-    if (aTy.getElementType().isFloat8E5M2() &&
-        bTy.getElementType().isFloat8E5M2())
+    if (llvm::isa<Float8E5M2Type>(aTy.getElementType()) &&
+        llvm::isa<Float8E5M2Type>(bTy.getElementType()))
       return TensorCoreType::FP32_FP8E5M2_FP8E5M2_FP32;
-    if (aTy.getElementType().isFloat8E5M2() &&
-        bTy.getElementType().isFloat8E4M3FN())
+    if (llvm::isa<Float8E5M2Type>(aTy.getElementType()) &&
+        llvm::isa<Float8E4M3FNType>(bTy.getElementType()))
       return TensorCoreType::FP32_FP8E5M2_FP8E4M3FN_FP32;
-    if (aTy.getElementType().isFloat8E4M3FN() &&
-        bTy.getElementType().isFloat8E5M2())
+    if (llvm::isa<Float8E4M3FNType>(aTy.getElementType()) &&
+        llvm::isa<Float8E5M2Type>(bTy.getElementType()))
       return TensorCoreType::FP32_FP8E4M3FN_FP8E5M2_FP32;
-    if (aTy.getElementType().isFloat8E4M3FN() &&
-        bTy.getElementType().isFloat8E4M3FN())
+    if (llvm::isa<Float8E4M3FNType>(aTy.getElementType()) &&
+        llvm::isa<Float8E4M3FNType>(bTy.getElementType()))
       return TensorCoreType::FP32_FP8E4M3FN_FP8E4M3FN_FP32;
     if (aTy.getElementType().isF32() && bTy.getElementType().isF32() &&
         op.getInputPrecision() == InputPrecision::TF32)
diff --git a/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/DotOpToLLVM/MMAv5.cpp b/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/DotOpToLLVM/MMAv5.cpp
@@ -61,7 +61,8 @@ enum class mxfpKind { mxf8f6f4 = 0, mxf4 = 1, mxf4nvf4 = 2 };
 inline mxfpKind getMXFPKind(ScaleDotElemType typeA, ScaleDotElemType typeB,
                             Type scaleAType, Type scaleBType) {
   if (typeA == ScaleDotElemType::E2M1 && typeB == ScaleDotElemType::E2M1) {
-    if (scaleAType.isFloat8E4M3FN() && scaleBType.isFloat8E4M3FN()) {
+    if (llvm::isa<Float8E4M3FNType>(scaleAType) &&
+        llvm::isa<Float8E4M3FNType>(scaleBType)) {
       return mxfpKind::mxf4nvf4;
     }
     return mxfpKind::mxf4;
@@ -102,9 +103,9 @@ static Value createInstDescriptor(ConversionPatternRewriter &rewriter,
       return 1;
     if (type.isF32())
       return 2;
-    if (type.isFloat8E4M3FN())
+    if (llvm::isa<Float8E4M3FNType>(type))
       return 0;
-    if (type.isFloat8E5M2())
+    if (llvm::isa<Float8E5M2Type>(type))
       return 1;
     llvm_unreachable("Unsupported type.");
   };
@@ -227,7 +228,7 @@ static void createGen5MMA(ConversionPatternRewriter &rewriter, Location loc,
     opcode += "f16";
   else if (srcElementTy.isF32())
     opcode += "tf32";
-  else if (srcElementTy.isFloat8E4M3FN() || srcElementTy.isFloat8E5M2())
+  else if (llvm::isa<Float8E4M3FNType, Float8E5M2Type>(srcElementTy))
     opcode += "f8f6f4";
   else
     assert(0 && "Unsupported type.");
diff --git a/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/DotOpToLLVM/WGMMA.cpp b/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/DotOpToLLVM/WGMMA.cpp
@@ -59,9 +59,9 @@ triton::nvgpu::WGMMAEltType getMmaOperandType(Value a, bool allowTF32) {
     return triton::nvgpu::WGMMAEltType::tf32;
   } else if (aTy.isInteger(8)) {
     return triton::nvgpu::WGMMAEltType::s8;
-  } else if (aTy.isFloat8E5M2()) {
+  } else if (llvm::isa<Float8E5M2Type>(aTy)) {
     return triton::nvgpu::WGMMAEltType::e5m2;
-  } else if (aTy.isFloat8E4M3FN()) {
+  } else if (llvm::isa<Float8E4M3FNType>(aTy)) {
     return triton::nvgpu::WGMMAEltType::e4m3;
   } else {
     llvm::report_fatal_error("Unsupported mma operand type found");
diff --git a/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/ElementwiseOpToLLVM.cpp b/third_party/nvidia/lib/TritonNVIDIAGPUToLLVM/ElementwiseOpToLLVM.cpp
@@ -448,8 +448,8 @@ struct FpToFpOpConversion
       llvm::errs() << "\n";
       llvm::report_fatal_error("Unsupported rounding mode for conversion.");
     }
-    if (computeCapability < 89 &&
-        (srcTy.isFloat8E4M3FN() || dstTy.isFloat8E4M3FN())) {
+    if (computeCapability < 89 && (llvm::isa<Float8E4M3FNType>(srcTy) ||
+                                   llvm::isa<Float8E4M3FNType>(dstTy))) {
       llvm::errs() << "Conversion from/to f8e4m3nv is only supported on "
                       "compute capability >= 89"
                    << "\n";
@@ -472,7 +472,7 @@ struct FpToFpOpConversion
     auto dstElementType = getElementType(op.getResult());
     auto roundingMode = op.getRounding();
 
-    if (dstElementType.isFloat8E5M2() || dstElementType.isFloat8E4M3FN()) {
+    if (llvm::isa<Float8E5M2Type, Float8E4M3FNType>(dstElementType)) {
       assert(roundingMode.has_value() &&
              "Rounding mode must be specified for convertsions to fp8");
 
@@ -509,8 +509,8 @@ struct FpToFpOpConversion
 
     bool useFP16IntermediateSrc =
         srcElementType.isF32() &&
-        (!(computeCapability >= 90 && (dstElementType.isFloat8E4M3FN() ||
-                                       dstElementType.isFloat8E5M2())) ||
+        (!(computeCapability >= 90 &&
+           (llvm::isa<Float8E4M3FNType, Float8E5M2Type>(dstElementType))) ||
          roundingMode.value() == RoundingMode::RTZ);
     bool isDstFP32 = dstElementType.isF32();
     Type srcType = useFP16IntermediateSrc ? f16_ty : srcElementType;

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-e2402615a5a76d46a433dfcc1de10b38a1263c9d`
	`1`	`+c118864223c6309378cd704f3406533474c2759f`
Original file line number	Diff line number	Diff line change
`@@ -20,19 +20,24 @@ static MfmaTypeId chooseAppropriateMfmaId(mlir::Type dataTypeA,`
`20`	`20`	`if (dataTypeA.isInteger(8) && dataTypeB.isInteger(8)) {`
`21`	`21`	`return MfmaTypeId::I8TyId;`
`22`	`22`	`}`
`23`		`- if (dataTypeA.isFloat8E4M3FNUZ() && dataTypeB.isFloat8E4M3FNUZ()) {`
	`23`	`+ if (llvm::isa<Float8E4M3FNUZType>(dataTypeA) &&`
	`24`	`+ llvm::isa<Float8E4M3FNUZType>(dataTypeB)) {`
`24`	`25`	`return MfmaTypeId::Fp8Fp8TyId;`
`25`	`26`	`}`
`26`		`- if (dataTypeA.isFloat8E4M3FNUZ() && dataTypeB.isFloat8E5M2FNUZ()) {`
	`27`	`+ if (llvm::isa<Float8E4M3FNUZType>(dataTypeA) &&`
	`28`	`+ llvm::isa<Float8E5M2FNUZType>(dataTypeB)) {`
`27`	`29`	`return MfmaTypeId::Fp8Bf8TyId;`
`28`	`30`	`}`
`29`		`- if (dataTypeA.isFloat8E5M2FNUZ() && dataTypeB.isFloat8E4M3FNUZ()) {`
	`31`	`+ if (llvm::isa<Float8E5M2FNUZType>(dataTypeA) &&`
	`32`	`+ llvm::isa<Float8E4M3FNUZType>(dataTypeB)) {`
`30`	`33`	`return MfmaTypeId::Bf8Fp8TyId;`
`31`	`34`	`}`
`32`		`- if (dataTypeA.isFloat8E5M2FNUZ() && dataTypeB.isFloat8E5M2FNUZ()) {`
	`35`	`+ if (llvm::isa<Float8E5M2FNUZType>(dataTypeA) &&`
	`36`	`+ llvm::isa<Float8E5M2FNUZType>(dataTypeB)) {`
`33`	`37`	`return MfmaTypeId::Bf8Bf8TyId;`
`34`	`38`	`}`
`35`		`- if (dataTypeA.isFloat8E5M2() && dataTypeB.isFloat8E5M2()) {`
	`39`	`+ if (llvm::isa<Float8E5M2Type>(dataTypeA) &&`
	`40`	`+ llvm::isa<Float8E5M2Type>(dataTypeB)) {`
`36`	`41`	`return MfmaTypeId::Fp16TyId;`
`37`	`42`	`}`
`38`	`43`	`llvm_unreachable("Unsupported input argument type.");`