[AMD] Enable General Swizzling ConvertLayoutOp (triton-lang#7482)

- Enables ConvertLayoutOp general swizzling for AMD
- Introduces simple heuristic in OptimizeLDSUsage so we use padding
based swizzling only if it is required to reduce LDS consumption
- Adds AMD specific ttg->llvm conversion pattern to process forced
padding separately from general swizzling

---------

Co-authored-by: Alexander Efimov <[email protected]>

Author: alefimov-amd

lib/Conversion/TritonGPUToLLVM/ConvertLayoutOpToLLVM.cpp

@@ -278,9 +278,8 @@ struct ConvertLayoutOpUsingLinearLayoutsConversion
     assert(cvtNeedsSharedMemory(op.getSrc().getType(), op.getType()));
 
     // Try to use swizzling to implement the conversion
-    // HACK Remove once AMD tests pass for the swizzling path
-    if (targetInfo.isCuda() && succeeded(transferWithinBlockSwizzling(
-                                   op, adaptor.getSrc(), rewriter))) {
+    if (succeeded(
+            transferWithinBlockSwizzling(op, adaptor.getSrc(), rewriter))) {
       return success();
     }
 

test/Conversion/amd/convert_layout.mlir

@@ -0,0 +1,41 @@
+// RUN: triton-opt %s -split-input-file --allocate-shared-memory --convert-triton-amdgpu-to-llvm=arch=gfx942 --canonicalize| FileCheck %s
+
+#blocked0 = #ttg.blocked<{sizePerThread = [1, 4], threadsPerWarp = [16, 4], warpsPerCTA = [2, 2], order = [1, 0], CTAsPerCGA = [1, 1], CTASplitNum = [1, 1], CTAOrder = [1, 0]}>
+#blocked1 = #ttg.blocked<{sizePerThread = [4, 1], threadsPerWarp = [4, 16], warpsPerCTA = [2, 2], order = [0, 1], CTAsPerCGA = [1, 1], CTASplitNum = [1, 1], CTAOrder = [1, 0]}>
+module attributes {"ttg.num-ctas" = 1 : i32, "ttg.num-warps" = 4 : i32, ttg.target = "hip:gfx942", "ttg.threads-per-warp" = 64 : i32} {
+  // CHECK: llvm.mlir.global external @global_smem
+  tt.func @convert_layout_general_swizzling(%arg0: tensor<64x64xf32, #blocked0>, %arg1: tensor<64x64x!tt.ptr<f32>, #blocked1>) {
+
+    // verify that following convert layout uses general swizzling
+
+    // CHECK-NOT: llvm.lshr
+
+    %0 = ttg.convert_layout %arg0 : tensor<64x64xf32, #blocked0> -> tensor<64x64xf32, #blocked1>
+    tt.store %arg1, %0 : tensor<64x64x!tt.ptr<f32>, #blocked1>
+    tt.return
+  }
+}
+
+// -----
+
+#blocked0 = #ttg.blocked<{sizePerThread = [1, 4], threadsPerWarp = [16, 4], warpsPerCTA = [2, 2], order = [1, 0], CTAsPerCGA = [1, 1], CTASplitNum = [1, 1], CTAOrder = [1, 0]}>
+#blocked1 = #ttg.blocked<{sizePerThread = [4, 1], threadsPerWarp = [4, 16], warpsPerCTA = [2, 2], order = [0, 1], CTAsPerCGA = [1, 1], CTASplitNum = [1, 1], CTAOrder = [1, 0]}>
+module attributes {"ttg.num-ctas" = 1 : i32, "ttg.num-warps" = 4 : i32, ttg.target = "hip:gfx942", "ttg.threads-per-warp" = 64 : i32} {
+  // CHECK-LABEL: convert_layout_padding_swizzling
+  tt.func @convert_layout_padding_swizzling(%arg0: tensor<64x64xf32, #blocked0>, %arg1: tensor<64x64x!tt.ptr<f32>, #blocked1>) {
+
+    // verify that following convert layout uses padded path
+    // see getVecAddr lambda in transferWithinBlockImpl function
+
+    // CHECK-DAG: [[CST_0:%.*]] = llvm.mlir.constant(0 : i32) : i32
+    // CHECK-DAG: [[CST_5:%.*]] = llvm.mlir.constant(5 : i32) : i32
+    // CHECK-DAG: [[OFFSET_0:%.*]] = llvm.lshr {{.*}}, [[CST_5]] : i32
+    // CHECK: [[OFFSET_1:%.*]] = llvm.shl [[OFFSET_0]], [[CST_0]] : i32
+    // CHECK: [[OFFSET_2:%.*]] = llvm.add [[OFFSET_1]], {{.*}} : i32
+    // CHECK: llvm.getelementptr inbounds {{.*}}{{\[}}[[OFFSET_2]]{{\]}}
+
+    %0 = ttg.convert_layout %arg0 {amdgpu.use_padded_scratch_shmem} : tensor<64x64xf32, #blocked0> -> tensor<64x64xf32, #blocked1>
+    tt.store %arg1, %0 : tensor<64x64x!tt.ptr<f32>, #blocked1>
+    tt.return
+  }
+}

third_party/amd/lib/TritonAMDGPUToLLVM/ConvertLayoutOpToLLVM.cpp

@@ -1,11 +1,14 @@
+#include "Analysis/AMDGPUAllocation.h"
 #include "PatternTritonGPUOpToLLVM.h"
 #include "Utility.h"
 #include "triton/Conversion/TritonGPUToLLVM/PatternTritonGPUOpToLLVM.h"
 #include "triton/Conversion/TritonGPUToLLVM/Utility.h"
 #include "triton/Dialect/TritonGPU/Transforms/Utility.h"
 
+using ::mlir::transferWithinBlockPadding;
 using ::mlir::triton::gpu::AMDMfmaEncodingAttr;
 using ::mlir::triton::gpu::AMDWmmaEncodingAttr;
+using ::mlir::triton::gpu::ConvertLayoutOp;
 using ::mlir::triton::gpu::DotOperandEncodingAttr;
 using ::mlir::triton::gpu::MemDescType;
 using ::triton::gpu::LinearEncodingAttr;
@@ -287,6 +290,36 @@ struct ConvertLayoutOpMFMAToLinearConversion
 protected:
   const TargetInfoBase &targetInfo;
 };
+
+struct ConvertLayoutForcedPadding
+    : public ConvertOpToLLVMPattern<ConvertLayoutOp> {
+
+  explicit ConvertLayoutForcedPadding(LLVMTypeConverter &typeConverter,
+                                      const TargetInfoBase &targetInfo,
+                                      PatternBenefit benefit)
+      : ConvertOpToLLVMPattern<ConvertLayoutOp>(typeConverter, benefit),
+        targetInfo(targetInfo) {}
+
+  LogicalResult
+  matchAndRewrite(ConvertLayoutOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    if (!op->hasAttr(mlir::triton::AMD::AttrSharedMemPadded))
+      return failure();
+    auto srcType = op.getSrc().getType();
+    auto dstType = op.getType();
+    if (!cvtNeedsSharedMemory(srcType, dstType))
+      return failure();
+
+    auto result = transferWithinBlockPadding(op, adaptor.getSrc(), targetInfo,
+                                             getTypeConverter(), rewriter);
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+
+protected:
+  const TargetInfoBase &targetInfo;
+};
+
 } // namespace
 
 void mlir::triton::AMD::populateConvertLayoutOpToLLVMPatterns(
@@ -296,4 +329,5 @@ void mlir::triton::AMD::populateConvertLayoutOpToLLVMPatterns(
                                                      benefit);
   patterns.add<ConvertLayoutOpMFMAToLinearConversion>(typeConverter, targetInfo,
                                                       benefit);
+  patterns.add<ConvertLayoutForcedPadding>(typeConverter, targetInfo, benefit);
 }

third_party/amd/lib/TritonAMDGPUToLLVM/OptimizeLDSUsage.cpp

@@ -94,15 +94,26 @@ class OptimizeAMDLDSUsage
     LDBG("Trying fit " << cvtOp << " into " << targetLDSSize << " bytes");
     OpBuilder builder(cvtOp);
 
+    auto ctx = builder.getContext();
     auto srcType = cvtOp.getSrc().getType();
     auto dstType = cvtOp.getType();
 
+    if (!cvtOp->hasAttr(triton::AMD::AttrSharedMemPadded)) {
+      auto emptyAttribute = UnitAttr::get(ctx);
+      // Padded conversion seems more friendly with this optimization
+      // use it instead of general swizzling.
+      cvtOp->setAttr(triton::AMD::AttrSharedMemPadded, emptyAttribute);
+      // if padded layout drops LDS usage on itself, we are done, return
+      if (triton::AMD::getConvertLayoutScratchInBytes(
+              srcType, dstType, /*usePadding*/ true) <= targetLDSSize)
+        return;
+    }
+
     auto srcEnc =
         cast<triton::gpu::DistributedEncodingTrait>(srcType.getEncoding());
     auto dstEnc =
         cast<triton::gpu::DistributedEncodingTrait>(dstType.getEncoding());
 
-    auto ctx = srcEnc.getContext();
     auto rank = srcType.getRank();
 
     unsigned numWarps = triton::gpu::lookupNumWarps(cvtOp);
@@ -244,13 +255,6 @@ class OptimizeAMDLDSUsage
       LDSLimit = targetInfo.getSharedMemorySize();
     }
 
-    auto context = mod.getContext();
-    auto emptyAttribute = UnitAttr::get(context);
-    // TODO choose between padded and swizzled memory patterns
-    mod.walk([emptyAttribute](triton::gpu::ConvertLayoutOp op) -> void {
-      op->setAttr(mlir::triton::AMD::AttrSharedMemPadded, emptyAttribute);
-    });
-
     ModuleAllocation allocAnalysis(
         mod, mlir::triton::AMD::AMDAllocationAnalysisScratchSizeFn);
     if (allocAnalysis.getSharedMemorySize() <= LDSLimit)