[LinearLayout] Fix crash with debug (triton-lang#6272)

This PR introduces stronger check for empty layouts in invertAndCompose
to fix compiler crashes while running in debug mode.

Author: binarman

lib/Tools/LinearLayout.cpp

@@ -837,6 +837,8 @@ LinearLayout lstsq(const LinearLayout &A, const LinearLayout &B) {
 
   // We need names for the in/out dim of the flattened layout we're going to
   // read off from `m`.  These could be anything, doesn't matter.
+  assert(!A.getInDimNames().empty() &&
+         "attempt to solve lstsq for empty layout");
   StringAttr inDim1D = *A.getInDimNames().begin();
   StringAttr outDim1D = *A.getOutDimNames().begin();
 
@@ -927,9 +929,8 @@ LinearLayout LinearLayout::invertAndCompose(const LinearLayout &outer) const {
   auto BReduced = B.sublayout(BNonIdentityInDims, outDims);
 
   // If one is empty, the other must be empty as well
-  assert((AReduced == LinearLayout::empty()) ==
-         (BReduced == LinearLayout::empty()));
-  bool isEmpty = AReduced == LinearLayout::empty();
+  assert((ANonIdentityInDims.empty()) == (BNonIdentityInDims.empty()));
+  bool isEmpty = ANonIdentityInDims.empty();
 
   auto ret = isEmpty ? LinearLayout::empty() : lstsq(AReduced, BReduced);
 

test/Conversion/tritongpu_to_llvm_debug.mlir

@@ -0,0 +1,11 @@
+// RUN: triton-opt %s -split-input-file --allocate-shared-memory --convert-triton-gpu-to-llvm --debug| FileCheck %s
+
+// CHECK-LABEL: convert_identity
+#blocked = #ttg.blocked<{sizePerThread = [8, 1], threadsPerWarp = [16, 4], warpsPerCTA = [1, 8], order = [0, 1]}>
+#smem = #ttg.shared_memory
+module attributes {"ttg.num-warps" = 8 : i32, "ttg.threads-per-warp" = 64 : i32} {
+  tt.func public @convert_identity(%arg0: tensor<128x128xf16, #blocked>) attributes {noinline = false} {
+    %1 = ttg.convert_layout %arg0 : tensor<128x128xf16, #blocked> -> tensor<128x128xf16, #blocked>
+    tt.return
+  }
+}