Merge remote-tracking branch 'origin/main' into vplan-resume-phi-using-PHI

Author: fhahn

clang/lib/AST/Expr.cpp

@@ -1470,8 +1470,8 @@ static unsigned SizeOfCallExprInstance(Expr::StmtClass SC) {
 
 // changing the size of SourceLocation, CallExpr, and
 // subclasses requires careful considerations
-static_assert(sizeof(SourceLocation) == 4 && sizeof(CXXOperatorCallExpr) == 32,
-              "we assume CXXOperatorCallExpr is 32 bytes");
+static_assert(sizeof(SourceLocation) == 4 && sizeof(CXXOperatorCallExpr) <= 32,
+              "we assume CXXOperatorCallExpr is at most 32 bytes");
 
 CallExpr::CallExpr(StmtClass SC, Expr *Fn, ArrayRef<Expr *> PreArgs,
                    ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,

clang/lib/CodeGen/CGBuiltin.cpp

@@ -29,6 +29,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/Instruction.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/MatrixBuilder.h"
@@ -4190,6 +4191,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         Matrix, Dst.emitRawPointer(*this),
         Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
         MatrixTy->getNumRows(), MatrixTy->getNumColumns());
+    addInstToNewSourceAtom(cast<Instruction>(Result), Matrix);
     return RValue::get(Result);
   }
 
@@ -4350,7 +4352,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *SizeVal = EmitScalarExpr(E->getArg(1));
     EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
                         E->getArg(0)->getExprLoc(), FD, 0);
-    Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
+    auto *I = Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(nullptr);
   }
 
@@ -4365,7 +4368,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)),
                         E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
                         0);
-    Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    auto *I = Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(nullptr);
   }
 
@@ -4378,7 +4382,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
     EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
     EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
-    Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+    auto *I = Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+    addInstToNewSourceAtom(I, nullptr);
     if (BuiltinID == Builtin::BImempcpy ||
         BuiltinID == Builtin::BI__builtin_mempcpy)
       return RValue::get(Builder.CreateInBoundsGEP(
@@ -4394,7 +4399,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
     EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
     EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
-    Builder.CreateMemCpyInline(Dest, Src, Size);
+    auto *I = Builder.CreateMemCpyInline(Dest, Src, Size);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(nullptr);
   }
 
@@ -4415,7 +4421,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+    auto *I = Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(Dest, *this);
   }
 
@@ -4441,7 +4448,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    auto *I = Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(Dest, *this);
   }
 
@@ -4461,7 +4469,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                   .getQuantity()));
     EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
     EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
-    Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    auto *I = Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(Dest, *this);
   }
   case Builtin::BImemset:
@@ -4472,7 +4481,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
     EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
                         E->getArg(0)->getExprLoc(), FD, 0);
-    Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
+    auto *I = Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
+    addInstToNewSourceAtom(I, ByteVal);
     return RValue::get(Dest, *this);
   }
   case Builtin::BI__builtin_memset_inline: {
@@ -4484,7 +4494,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)),
                         E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
                         0);
-    Builder.CreateMemSetInline(Dest, ByteVal, Size);
+    auto *I = Builder.CreateMemSetInline(Dest, ByteVal, Size);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(nullptr);
   }
   case Builtin::BI__builtin___memset_chk: {
@@ -4501,7 +4512,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
                                          Builder.getInt8Ty());
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
+    auto *I = Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
+    addInstToNewSourceAtom(I, nullptr);
     return RValue::get(Dest, *this);
   }
   case Builtin::BI__builtin_wmemchr: {

clang/test/DebugInfo/KeyInstructions/builtin.c

@@ -0,0 +1,88 @@
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c++ %s -debug-info-kind=line-tables-only -gno-column-info -emit-llvm -o - -ftrivial-auto-var-init=zero -fenable-matrix -disable-llvm-passes \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c %s -debug-info-kind=line-tables-only -gno-column-info -emit-llvm -o - -ftrivial-auto-var-init=zero -fenable-matrix -disable-llvm-passes \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+typedef float m2x2 __attribute__((matrix_type(2, 2)));
+m2x2 mat;
+float f4[4];
+float f8[8];
+int v = 3;
+
+void fun() {
+// CHECK: %a = alloca ptr, align 8
+// CHECK: %0 = alloca i8, i64 4{{.*}}, !dbg [[G1R2:!.*]]
+// CHECK: call void @llvm.memset{{.*}}, !dbg [[G1R1:!.*]], !annotation
+// CHECK: store ptr %0, ptr %a{{.*}}, !dbg [[G1R1:!.*]]
+    void *a = __builtin_alloca(4);
+
+// CHECK: %1 = alloca i8, i64 4{{.*}}, !dbg [[G2R2:!.*]]
+// CHECK: call void @llvm.memset{{.*}}, !dbg [[G2R1:!.*]], !annotation
+// CHECK: store ptr %1, ptr %b{{.*}}, !dbg [[G2R1:!.*]]
+    void *b = __builtin_alloca_with_align(4, 8);
+
+// CHECK: %2 = load <4 x float>, ptr @mat{{.*}}, !dbg [[G3R2:!.*]]
+// CHECK: call void @llvm.matrix.column.major.store.v4f32{{.*}}, !dbg [[G3R1:!.*]]
+    __builtin_matrix_column_major_store(mat, f4, sizeof(float) * 2);
+
+// CHECK: call void @llvm.memset{{.*}}, !dbg [[G4R1:!.*]]
+    __builtin_bzero(f4, sizeof(float) * 2);
+
+// CHECK: call void @llvm.memmove{{.*}}, !dbg [[G5R1:!.*]]
+    __builtin_bcopy(f4, f8, sizeof(float) * 4);
+
+// CHECK: call void @llvm.memcpy{{.*}}, !dbg [[G6R1:!.*]]
+    __builtin_memcpy(f4, f8, sizeof(float) * 4);
+
+// CHECK: call void @llvm.memcpy{{.*}}, !dbg [[G7R1:!.*]]
+    __builtin_mempcpy(f4, f8, sizeof(float) * 4);
+
+// CHECK: call void @llvm.memcpy{{.*}}, !dbg [[G8R1:!.*]]
+    __builtin_memcpy_inline(f4, f8, sizeof(float) * 4);
+
+// CHECK: call void @llvm.memcpy{{.*}}, !dbg [[G9R1:!.*]]
+    __builtin___memcpy_chk(f4, f8, sizeof(float) * 4, -1);
+
+// CHECK: call void @llvm.memmove{{.*}}, !dbg [[G10R1:!.*]]
+    __builtin___memmove_chk(f4, f8, sizeof(float) * 4, -1);
+
+// CHECK: call void @llvm.memmove{{.*}}, !dbg [[G11R1:!.*]]
+    __builtin_memmove(f4, f8, sizeof(float) * 4);
+
+// CHECK: call void @llvm.memset{{.*}}, !dbg [[G12R1:!.*]]
+    __builtin_memset(f4, 0, sizeof(float) * 4);
+
+// CHECK: call void @llvm.memset{{.*}}, !dbg [[G13R1:!.*]]
+    __builtin_memset_inline(f4, 0, sizeof(float) * 4);
+
+// CHECK: call void @llvm.memset{{.*}}, !dbg [[G14R1:!.*]]
+    __builtin___memset_chk(f4, 0, sizeof(float), -1);
+
+// CHECK: %3 = load i32, ptr @v{{.*}}, !dbg [[G15R3:!.*]]
+// CHECK-NEXT: %4 = trunc i32 %3 to i8, !dbg [[G15R2:!.*]]
+// CHECK-NEXT: call void @llvm.memset{{.*}}, !dbg [[G15R1:!.*]]
+    __builtin_memset(f4, v, sizeof(float) * 4);
+}
+
+// CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
+// CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
+// CHECK: [[G2R2]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 2)
+// CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
+// CHECK: [[G3R2]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 2)
+// CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
+// CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
+// CHECK: [[G5R1]] = !DILocation({{.*}}, atomGroup: 5, atomRank: 1)
+// CHECK: [[G6R1]] = !DILocation({{.*}}, atomGroup: 6, atomRank: 1)
+// CHECK: [[G7R1]] = !DILocation({{.*}}, atomGroup: 7, atomRank: 1)
+// CHECK: [[G8R1]] = !DILocation({{.*}}, atomGroup: 8, atomRank: 1)
+// CHECK: [[G9R1]] = !DILocation({{.*}}, atomGroup: 9, atomRank: 1)
+// CHECK: [[G10R1]] = !DILocation({{.*}}, atomGroup: 10, atomRank: 1)
+// CHECK: [[G11R1]] = !DILocation({{.*}}, atomGroup: 11, atomRank: 1)
+// CHECK: [[G12R1]] = !DILocation({{.*}}, atomGroup: 12, atomRank: 1)
+// CHECK: [[G13R1]] = !DILocation({{.*}}, atomGroup: 13, atomRank: 1)
+// CHECK: [[G14R1]] = !DILocation({{.*}}, atomGroup: 14, atomRank: 1)
+// CHECK: [[G15R3]] = !DILocation({{.*}}, atomGroup: 15, atomRank: 3)
+// CHECK: [[G15R2]] = !DILocation({{.*}}, atomGroup: 15, atomRank: 2)
+// CHECK: [[G15R1]] = !DILocation({{.*}}, atomGroup: 15, atomRank: 1)

flang/include/flang/Lower/AbstractConverter.h

@@ -348,6 +348,10 @@ class AbstractConverter {
   virtual Fortran::lower::SymbolBox
   lookupOneLevelUpSymbol(const Fortran::semantics::Symbol &sym) = 0;
 
+  /// Find the symbol in the inner-most level of the local map or return null.
+  virtual Fortran::lower::SymbolBox
+  shallowLookupSymbol(const Fortran::semantics::Symbol &sym) = 0;
+
   /// Return the mlir::SymbolTable associated to the ModuleOp.
   /// Look-ups are faster using it than using module.lookup<>,
   /// but the module op should be queried in case of failure

flang/include/flang/Optimizer/Dialect/FIROps.h

@@ -147,6 +147,10 @@ class CoordinateIndicesAdaptor {
   mlir::ValueRange values;
 };
 
+struct LocalitySpecifierOperands {
+  llvm::SmallVector<::mlir::Value> privateVars;
+  llvm::SmallVector<::mlir::Attribute> privateSyms;
+};
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_DIALECT_FIROPS_H

flang/include/flang/Optimizer/Dialect/FIROps.td

@@ -3605,6 +3605,21 @@ def fir_LocalitySpecifierOp : fir_Op<"local", [IsolatedFromAbove]> {
   ];
 
   let extraClassDeclaration = [{
+    mlir::BlockArgument getInitMoldArg() {
+      auto &region = getInitRegion();
+      return region.empty() ? nullptr : region.getArgument(0);
+    }
+    mlir::BlockArgument getInitPrivateArg() {
+      auto &region = getInitRegion();
+      return region.empty() ? nullptr : region.getArgument(1);
+    }
+
+    /// Returns true if the init region might read from the mold argument
+    bool initReadsFromMold() {
+      mlir::BlockArgument moldArg = getInitMoldArg();
+      return moldArg && !moldArg.use_empty();
+    }
+
     /// Get the type for arguments to nested regions. This should
     /// generally be either the same as getType() or some pointer
     /// type (pointing to the type allocated by this op).

flang/lib/Lower/Bridge.cpp

@@ -12,6 +12,8 @@
 
 #include "flang/Lower/Bridge.h"
 
+#include "OpenMP/DataSharingProcessor.h"
+#include "OpenMP/Utils.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -1142,6 +1144,14 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     return name;
   }
 
+  /// Find the symbol in the inner-most level of the local map or return null.
+  Fortran::lower::SymbolBox
+  shallowLookupSymbol(const Fortran::semantics::Symbol &sym) override {
+    if (Fortran::lower::SymbolBox v = localSymbols.shallowLookupSymbol(sym))
+      return v;
+    return {};
+  }
+
 private:
   FirConverter() = delete;
   FirConverter(const FirConverter &) = delete;
@@ -1216,14 +1226,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     return {};
   }
 
-  /// Find the symbol in the inner-most level of the local map or return null.
-  Fortran::lower::SymbolBox
-  shallowLookupSymbol(const Fortran::semantics::Symbol &sym) {
-    if (Fortran::lower::SymbolBox v = localSymbols.shallowLookupSymbol(sym))
-      return v;
-    return {};
-  }
-
   /// Find the symbol in one level up of symbol map such as for host-association
   /// in OpenMP code or return null.
   Fortran::lower::SymbolBox
@@ -2027,9 +2029,34 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   void handleLocalitySpecs(const IncrementLoopInfo &info) {
     Fortran::semantics::SemanticsContext &semanticsContext =
         bridge.getSemanticsContext();
-    for (const Fortran::semantics::Symbol *sym : info.localSymList)
+    // TODO Extract `DataSharingProcessor` from omp to a more general location.
+    Fortran::lower::omp::DataSharingProcessor dsp(
+        *this, semanticsContext, getEval(),
+        /*useDelayedPrivatization=*/true, localSymbols);
+    fir::LocalitySpecifierOperands privateClauseOps;
+    auto doConcurrentLoopOp =
+        mlir::dyn_cast_if_present<fir::DoConcurrentLoopOp>(info.loopOp);
+    // TODO Promote to using `enableDelayedPrivatization` (which is enabled by
+    // default unlike the staging flag) once the implementation of this is more
+    // complete.
+    bool useDelayedPriv =
+        enableDelayedPrivatizationStaging && doConcurrentLoopOp;
+
+    for (const Fortran::semantics::Symbol *sym : info.localSymList) {
+      if (useDelayedPriv) {
+        dsp.privatizeSymbol<fir::LocalitySpecifierOp>(sym, &privateClauseOps);
+        continue;
+      }
+
       createHostAssociateVarClone(*sym, /*skipDefaultInit=*/false);
+    }
+
     for (const Fortran::semantics::Symbol *sym : info.localInitSymList) {
+      if (useDelayedPriv) {
+        dsp.privatizeSymbol<fir::LocalitySpecifierOp>(sym, &privateClauseOps);
+        continue;
+      }
+
       createHostAssociateVarClone(*sym, /*skipDefaultInit=*/true);
       const auto *hostDetails =
           sym->detailsIf<Fortran::semantics::HostAssocDetails>();
@@ -2048,6 +2075,24 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           sym->detailsIf<Fortran::semantics::HostAssocDetails>();
       copySymbolBinding(hostDetails->symbol(), *sym);
     }
+
+    if (useDelayedPriv) {
+      doConcurrentLoopOp.getLocalVarsMutable().assign(
+          privateClauseOps.privateVars);
+      doConcurrentLoopOp.setLocalSymsAttr(
+          builder->getArrayAttr(privateClauseOps.privateSyms));
+
+      for (auto [sym, privateVar] : llvm::zip_equal(
+               dsp.getAllSymbolsToPrivatize(), privateClauseOps.privateVars)) {
+        auto arg = doConcurrentLoopOp.getRegion().begin()->addArgument(
+            privateVar.getType(), doConcurrentLoopOp.getLoc());
+        bindSymbol(*sym, hlfir::translateToExtendedValue(
+                             privateVar.getLoc(), *builder, hlfir::Entity{arg},
+                             /*contiguousHint=*/true)
+                             .first);
+      }
+    }
+
     // Note that allocatable, types with ultimate components, and type
     // requiring finalization are forbidden in LOCAL/LOCAL_INIT (F2023 C1130),
     // so no clean-up needs to be generated for these entities.