Merge branch 'main' into fortify_strcat

Author: venk-ks

clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp

@@ -432,7 +432,7 @@ dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
 }
 
 std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>, std::vector<std::string>>
-dependencies::initVFSForTUBuferScanning(
+dependencies::initVFSForTUBufferScanning(
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
     ArrayRef<std::string> CommandLine, StringRef WorkingDirectory,
     llvm::MemoryBufferRef TUBuffer) {

clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h

@@ -113,10 +113,10 @@ createCompilerInvocation(ArrayRef<std::string> CommandLine,
                          DiagnosticsEngine &Diags);
 
 std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>, std::vector<std::string>>
-initVFSForTUBuferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
-                          ArrayRef<std::string> CommandLine,
-                          StringRef WorkingDirectory,
-                          llvm::MemoryBufferRef TUBuffer);
+initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+                           ArrayRef<std::string> CommandLine,
+                           StringRef WorkingDirectory,
+                           llvm::MemoryBufferRef TUBuffer);
 
 std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
           std::vector<std::string>>

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

@@ -157,7 +157,7 @@ bool DependencyScanningWorker::computeDependencies(
     DependencyConsumer &Consumer, DependencyActionController &Controller,
     DiagnosticConsumer &DC, std::optional<llvm::MemoryBufferRef> TUBuffer) {
   if (TUBuffer) {
-    auto [FinalFS, FinalCommandLine] = initVFSForTUBuferScanning(
+    auto [FinalFS, FinalCommandLine] = initVFSForTUBufferScanning(
         BaseFS, CommandLine, WorkingDirectory, *TUBuffer);
     return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer,
                             Controller, DC, FinalFS);

lldb/docs/CMakeLists.txt

@@ -34,7 +34,7 @@ if (LLDB_ENABLE_PYTHON AND SPHINX_FOUND)
       COMMAND "${CMAKE_COMMAND}" -E copy "${LLDB_SOURCE_DIR}/examples/python/templates/scripted_thread_plan.py" "${CMAKE_CURRENT_BINARY_DIR}/lldb/plugins/"
       COMMENT "Copying lldb.py to pretend its a Python package.")
 
-    add_dependencies(lldb-python-doc-package swig_wrapper_python lldb-python)
+    add_dependencies(lldb-python-doc-package swig_wrapper_python)
 
     # FIXME: Don't treat Sphinx warnings as errors. The files generated by
     # automodapi are full of warnings (partly caused by SWIG, our documentation

llvm/lib/Target/RISCV/RISCVInstrInfoP.td

@@ -1461,7 +1461,7 @@ let Predicates = [HasStdExtP, IsRV32] in {
 // Codegen patterns
 //===----------------------------------------------------------------------===//
 
-def riscv_absw : RVSDNode<"ABSW", SDTIntUnaryOp>;
+def riscv_absw : RVSDNode<"ABSW", SDT_RISCVIntUnaryOpW>;
 
 def SDT_RISCVPASUB : SDTypeProfile<1, 2, [SDTCisVec<0>,
                                           SDTCisInt<0>,

llvm/lib/Target/RISCV/RISCVRegisterInfo.td

@@ -228,6 +228,8 @@ def XLenVecI8VT : ValueTypeByHwMode<[RV32, RV64],
                                     [v4i8, v8i8]>;
 def XLenVecI16VT : ValueTypeByHwMode<[RV32,  RV64],
                                      [v2i16, v4i16]>;
+def XLenVecI32VT : ValueTypeByHwMode<[RV64],
+                                     [v2i32]>;
 def XLenRI : RegInfoByHwMode<
       [RV32,              RV64],
       [RegInfo<32,32,32>, RegInfo<64,64,64>]>;
@@ -246,7 +248,7 @@ class RISCVRegisterClass<list<ValueType> regTypes, int align, dag regList>
 class GPRRegisterClass<dag regList>
     : RISCVRegisterClass<[XLenVT, XLenFVT,
                           // P extension packed vector types:
-                          XLenVecI8VT, XLenVecI16VT, v2i32], 32, regList> {
+                          XLenVecI8VT, XLenVecI16VT, XLenVecI32VT], 32, regList> {
   let RegInfos = XLenRI;
 }
 

llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll

@@ -8,6 +8,15 @@ target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 ; CHECK: %j.1 = phi
 ; CHECK: %j.2 = phi
 ; CHECK: %j.3 = phi
+;
+; fore_aft_less SHOULD be unroll-and-jammed (count=4) as it's safe.
+; Memory accesses:
+;   - Fore block: A[i] = 1      (write in outer loop before inner)
+;   - Aft block:  A[i-1] = sum  (write in outer loop after inner)
+; No dependency conflict: The fore block write A[i] and aft block write A[i-1]
+; access different array elements, so unrolling the outer loop and jamming the
+; inner loop is safe. The backward dependency (i-1) doesn't create conflicts
+; between different unrolled iterations.
 define void @fore_aft_less(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -48,6 +57,15 @@ cleanup:
 ; CHECK: %j.1 = phi
 ; CHECK: %j.2 = phi
 ; CHECK: %j.3 = phi
+;
+; fore_aft_eq SHOULD be unroll-and-jammed (count=4) as it's safe.
+; Memory accesses:
+;   - Fore block: A[i] = 1    (write in outer loop before inner)
+;   - Aft block:  A[i] = sum  (write in outer loop after inner)
+; Dependency conflict: Both fore and aft blocks write to A[i], creating a
+; write-after-write (WAW) dependency. However, this is safe for unroll-and-jam
+; because the aft block write always happens after the fore block write in
+; the same iteration, preserving the original execution order.
 define void @fore_aft_eq(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -86,6 +104,15 @@ cleanup:
 ; CHECK-LABEL: fore_aft_more
 ; CHECK: %j = phi
 ; CHECK-NOT: %j.1 = phi
+;
+; fore_aft_more should NOT be unroll-and-jammed due to a dependency violation.
+; Memory accesses:
+;   - Fore block: A[i] = 1      (write in outer loop before inner)
+;   - Aft block:  A[i+1] = sum  (write in outer loop after inner)
+; Dependency conflict: The fore block writes A[i] and aft block writes A[i+1].
+; When unroll-and-jamming, iteration i's aft block writes A[i+1] which conflicts
+; with iteration i+1's fore block write to A[i+1], creating a write-after-write
+; race condition that violates the original sequential semantics.
 define void @fore_aft_more(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -126,6 +153,14 @@ cleanup:
 ; CHECK: %j.1 = phi
 ; CHECK: %j.2 = phi
 ; CHECK: %j.3 = phi
+;
+; fore_sub_less SHOULD be unroll-and-jammed (count=4) as it's safe.
+; Memory accesses:
+;   - Fore block: A[i] = 1      (write in outer loop before inner)
+;   - Sub block:  A[i-1] = sum  (write inside inner loop)
+; No dependency conflict: The fore block writes A[i] and sub block writes A[i-1].
+; These access different array elements, so unroll-and-jam is safe. The backward
+; dependency pattern doesn't create conflicts between unrolled iterations.
 define void @fore_sub_less(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -166,6 +201,15 @@ cleanup:
 ; CHECK: %j.1 = phi
 ; CHECK: %j.2 = phi
 ; CHECK: %j.3 = phi
+;
+; fore_sub_eq SHOULD be unroll-and-jammed (count=4) as it's safe.
+; Memory accesses:
+;   - Fore block: A[i] = 1    (write in outer loop before inner)
+;   - Sub block:  A[i] = sum  (write inside inner loop)
+; Dependency conflict: Both fore and sub blocks write to A[i], creating a
+; write-after-write (WAW) dependency. However, this is safe for unroll-and-jam
+; because the execution order is preserved: fore block executes first, then
+; the entire inner loop (sub block) executes, maintaining the original semantics.
 define void @fore_sub_eq(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -204,6 +248,15 @@ cleanup:
 ; CHECK-LABEL: fore_sub_more
 ; CHECK: %j = phi
 ; CHECK-NOT: %j.1 = phi
+;
+; fore_sub_more should NOT be unroll-and-jammed due to a dependency violation.
+; Memory accesses:
+;   - Fore block: A[i] = 1      (write in outer loop before inner)
+;   - Sub block:  A[i+1] = sum  (write inside inner loop)
+; Dependency conflict: The fore block writes A[i] and sub block writes A[i+1].
+; When unroll-and-jamming, iteration i's fore block writes A[i] but iteration i's
+; sub block writes A[i+1]. This conflicts with iteration i+1's fore block write
+; to A[i+1], creating a write-after-write race condition.
 define void @fore_sub_more(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -244,6 +297,14 @@ cleanup:
 ; CHECK: %j.1 = phi
 ; CHECK: %j.2 = phi
 ; CHECK: %j.3 = phi
+;
+; sub_aft_less SHOULD be unroll-and-jammed (count=4) as it's safe.
+; Memory accesses:
+;   - Sub block: A[i] = 1      (write inside inner loop)
+;   - Aft block: A[i-1] = sum  (write in outer loop after inner)
+; No dependency conflict: The sub block writes A[i] and aft block writes A[i-1].
+; These access different array elements, so unroll-and-jam is safe. The backward
+; dependency pattern doesn't create conflicts between unrolled iterations.
 define void @sub_aft_less(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -284,6 +345,15 @@ cleanup:
 ; CHECK: %j.1 = phi
 ; CHECK: %j.2 = phi
 ; CHECK: %j.3 = phi
+;
+; sub_aft_eq SHOULD be unroll-and-jammed (count=4) as it's safe.
+; Memory accesses:
+;   - Sub block: A[i] = 1    (write inside inner loop)
+;   - Aft block: A[i] = sum  (write in outer loop after inner)
+; Dependency conflict: Both sub and aft blocks write to A[i], creating a
+; write-after-write (WAW) dependency. However, this is safe for unroll-and-jam
+; because the execution order is preserved: the entire inner loop (sub block)
+; executes first, then the aft block executes, maintaining original semantics.
 define void @sub_aft_eq(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -322,6 +392,15 @@ cleanup:
 ; CHECK-LABEL: sub_aft_more
 ; CHECK: %j = phi
 ; CHECK-NOT: %j.1 = phi
+;
+; sub_aft_more should NOT be unroll-and-jammed due to a dependency violation.
+; Memory accesses:
+;   - Sub block: A[i] = 1      (write inside inner loop)
+;   - Aft block: A[i+1] = sum  (write in outer loop after inner)
+; Dependency conflict: The sub block writes A[i] and aft block writes A[i+1].
+; When unroll-and-jamming, iteration i's aft block writes A[i+1] which conflicts
+; with iteration i+1's sub block write to A[i+1], creating a write-after-write
+; race condition that violates the original sequential semantics.
 define void @sub_aft_more(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -360,6 +439,15 @@ cleanup:
 ; CHECK-LABEL: sub_sub_less
 ; CHECK: %j = phi
 ; CHECK-NOT: %j.1 = phi
+;
+; sub_sub_less should NOT be unroll-and-jammed due to a dependency violation.
+; Memory accesses:
+;   - Sub block: A[i] = 1      (write inside inner loop)
+;   - Sub block: A[i-1] = sum  (write inside inner loop)
+; Dependency conflict: Both writes are in the sub block (inner loop), accessing
+; A[i] and A[i-1]. When unroll-and-jamming, the inner loop is jammed, meaning
+; iterations of the inner loop from different outer iterations execute together.
+; This creates a backward dependency that can cause race conditions.
 define void @sub_sub_less(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -400,6 +488,15 @@ cleanup:
 ; CHECK: %j.1 = phi
 ; CHECK: %j.2 = phi
 ; CHECK: %j.3 = phi
+;
+; sub_sub_eq SHOULD be unroll-and-jammed (count=4) as it's safe.
+; Memory accesses:
+;   - Sub block: A[i] = 1    (write inside inner loop)
+;   - Sub block: A[i] = sum  (write inside inner loop)
+; Dependency conflict: Both writes are to A[i] within the sub block, creating a
+; write-after-write (WAW) dependency. However, this is safe for unroll-and-jam
+; because both writes are in the same basic block and maintain their relative
+; order: A[i] = 1 always executes before A[i] = sum in each iteration.
 define void @sub_sub_eq(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0
@@ -438,6 +535,15 @@ cleanup:
 ; CHECK-LABEL: sub_sub_more
 ; CHECK: %j = phi
 ; CHECK-NOT: %j.1 = phi
+;
+; sub_sub_more should NOT be unroll-and-jammed due to a dependency violation.
+; Memory accesses:
+;   - Sub block: A[i] = 1      (write inside inner loop)
+;   - Sub block: A[i+1] = sum  (write inside inner loop)
+; Dependency conflict: Both writes are in the sub block, accessing A[i] and A[i+1].
+; When unroll-and-jamming, iteration i's sub block writes A[i+1] which conflicts
+; with iteration i+1's sub block write to A[i+1]. This creates a forward
+; dependency that causes write-after-write race conditions.
 define void @sub_sub_more(ptr noalias nocapture %A, i32 %N, ptr noalias nocapture readonly %B) {
 entry:
   %cmp = icmp sgt i32 %N, 0

mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp

@@ -1285,6 +1285,71 @@ struct WgToSgVectorTransposeOp
   }
 };
 
+// This pattern distributes the vector.constant_mask ops to work at subgroup
+// level.
+struct WgToSgVectorConstantMaskOp
+    : public OpConversionPattern<vector::ConstantMaskOp> {
+  using OpConversionPattern<vector::ConstantMaskOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(vector::ConstantMaskOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    xegpu::DistributeLayoutAttr layout =
+        xegpu::getDistributeLayoutAttr(op.getResult());
+    if (!layout || !layout.isForWorkgroup())
+      return failure();
+
+    Location loc = op.getLoc();
+    VectorType type = op.getResult().getType();
+    auto wgShape = type.getShape();
+
+    ArrayRef<int64_t> wgMaskDimSizes = op.getMaskDimSizes();
+
+    // Get subgroup ID.
+    Value sgId =
+        gpu::SubgroupIdOp::create(rewriter, loc, /*upper_bound=*/nullptr);
+    auto sgOffsets =
+        layout.computeDistributedCoords(rewriter, loc, sgId, wgShape);
+    if (failed(sgOffsets))
+      return failure();
+
+    SmallVector<int64_t> sgShape = getSgShapeAndCount(wgShape, layout).first;
+    VectorType resultType = VectorType::get(sgShape, type.getElementType());
+
+    // In each dimension, each subgroup computes its local mask size as:
+    // min(max(wgMaskSize[d] - offset[d], 0), sgDimSize[d])
+    SmallVector<Value> newCreateMaskOps;
+    for (auto offsetSet : *sgOffsets) {
+      SmallVector<Value> maskOperands;
+
+      for (auto [i, wgMaskSize] : llvm::enumerate(wgMaskDimSizes)) {
+        Value wgMaskSizeVal =
+            arith::ConstantIndexOp::create(rewriter, loc, wgMaskSize);
+        Value dimSizeVal =
+            arith::ConstantIndexOp::create(rewriter, loc, sgShape[i]);
+        Value offset = offsetSet[i];
+        Value adjustedMaskSize =
+            arith::SubIOp::create(rewriter, loc, wgMaskSizeVal, offset);
+        Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
+        Value nonNegative =
+            arith::MaxSIOp::create(rewriter, loc, adjustedMaskSize, zero);
+        Value sgMaskSize =
+            arith::MinSIOp::create(rewriter, loc, nonNegative, dimSizeVal);
+        maskOperands.push_back(sgMaskSize);
+      }
+
+      auto newCreateMaskOp =
+          vector::CreateMaskOp::create(rewriter, loc, resultType, maskOperands);
+      xegpu::setDistributeLayoutAttr(newCreateMaskOp->getResult(0),
+                                     layout.dropSgLayoutAndData());
+      newCreateMaskOps.push_back(newCreateMaskOp.getResult());
+    }
+
+    rewriter.replaceOpWithMultiple(op, {newCreateMaskOps});
+    return success();
+  }
+};
+
 } // namespace
 
 namespace mlir {
@@ -1299,8 +1364,8 @@ void populateXeGPUWgToSgDistributePatterns(RewritePatternSet &patterns) {
            WgToSgArithConstantOp, WgToSgLoadGatherOpWithOffset,
            WgToSgStoreScatterOpWithOffset, WgToSgLoadMatrixOp,
            WgToSgStoreMatrixOp, WgToSgVectorStepOp, WgToSgVectorShapeCastOp,
-           WgToSgMultiDimReductionOp, WgToSgVectorTransposeOp>(
-          patterns.getContext());
+           WgToSgMultiDimReductionOp, WgToSgVectorTransposeOp,
+           WgToSgVectorConstantMaskOp>(patterns.getContext());
 }
 } // namespace xegpu
 } // namespace mlir
@@ -1427,9 +1492,9 @@ void XeGPUWgToSgDistributePass::runOnOperation() {
         return isLegal(layout);
       });
 
-  target.addDynamicallyLegalOp<vector::ShapeCastOp, vector::StepOp,
-                               vector::TransposeOp, vector::BroadcastOp,
-                               vector::MultiDimReductionOp>(
+  target.addDynamicallyLegalOp<
+      vector::ShapeCastOp, vector::StepOp, vector::TransposeOp,
+      vector::BroadcastOp, vector::MultiDimReductionOp, vector::ConstantMaskOp>(
       [=](Operation *op) -> bool {
         // Check for either a SliceAttr or LayoutAttr on the result.
         auto layout = xegpu::getDistributeLayoutAttr(op->getResult(0));

mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-unify-ops-rr.mlir

@@ -130,5 +130,13 @@ gpu.module @test_distribution {
     %trans = vector.transpose %load, [1, 0] {layout_result_0 = #xegpu.layout<sg_layout = [4, 8], sg_data = [16, 32], lane_layout = [1, 16], lane_data = [1, 1], order =[1, 0]>} : vector<256x128xf32> to vector<128x256xf32>
       gpu.return
   }
+
+  // CHECK-LABEL: vector_mask_2D
+  gpu.func @vector_mask_2D() {
+    // CHECK-COUNT-4: vector.create_mask {{.*}}, {{.*}} : vector<16x16xi1>
+    // CHECK-NOT: vector.create_mask
+    %constant_mask = vector.constant_mask [16, 16] {layout_result_0 = #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16]>} : vector<256x128xi1>
+    gpu.return
+  }
 }