[CIR] Emit copy for aggregate initialization (#155697)

This adds the implementation of aggEmitFinalDestCopy for the case where
the destination value is not ignored. This requires adding the cir.copy
operation and associated interface code.

Author: andykaylor

clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h

@@ -245,6 +245,11 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return createGetGlobal(global.getLoc(), global);
   }
 
+  /// Create a copy with inferred length.
+  cir::CopyOp createCopy(mlir::Value dst, mlir::Value src) {
+    return cir::CopyOp::create(*this, dst.getLoc(), dst, src);
+  }
+
   cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
                            bool isVolatile = false,
                            mlir::IntegerAttr align = {},

clang/include/clang/CIR/Dialect/IR/CIROps.td

@@ -2451,6 +2451,52 @@ def CIR_CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// CopyOp
+//===----------------------------------------------------------------------===//
+
+def CIR_CopyOp : CIR_Op<"copy",[
+  SameTypeOperands,
+  DeclareOpInterfaceMethods<PromotableMemOpInterface>
+]> {
+  let summary = "Copies contents from a CIR pointer to another";
+  let description = [{
+    Given two CIR pointers, `src` and `dst`, `cir.copy` will copy the memory
+    pointed by `src` to the memory pointed by `dst`.
+
+    The number of bytes copied is inferred from the pointee type. The pointee
+    type of `src` and `dst` must match and both must implement the
+    `DataLayoutTypeInterface`.
+
+    Examples:
+
+    ```mlir
+      // Copying contents from one record to another:
+      cir.copy %0 to %1 : !cir.ptr<!record_ty>
+    ```
+  }];
+
+  let arguments = (ins
+      Arg<CIR_PointerType, "", [MemWrite]>:$dst,
+      Arg<CIR_PointerType, "", [MemRead]>:$src
+  );
+
+  let assemblyFormat = [{$src `to` $dst
+                        attr-dict `:` qualified(type($dst))
+  }];
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    /// Returns the pointer type being copied.
+    cir::PointerType getType() { return getSrc().getType(); }
+
+    /// Returns the number of bytes to be copied.
+    unsigned getLength(const mlir::DataLayout &dt) {
+      return dt.getTypeSize(getType().getPointee());
+    }    
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // ReturnAddrOp and FrameAddrOp
 //===----------------------------------------------------------------------===//

clang/include/clang/CIR/MissingFeatures.h

@@ -168,6 +168,7 @@ struct MissingFeatures {
   // Misc
   static bool abiArgInfo() { return false; }
   static bool addHeapAllocSiteMetadata() { return false; }
+  static bool aggEmitFinalDestCopyRValue() { return false; }
   static bool aggValueSlot() { return false; }
   static bool aggValueSlotAlias() { return false; }
   static bool aggValueSlotDestructedFlag() { return false; }

clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp

@@ -62,12 +62,19 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   /// Perform the final copy to DestPtr, if desired.
   void emitFinalDestCopy(QualType type, const LValue &src);
 
+  void emitCopy(QualType type, const AggValueSlot &dest,
+                const AggValueSlot &src);
+
   void emitInitializationToLValue(Expr *e, LValue lv);
 
   void emitNullInitializationToLValue(mlir::Location loc, LValue lv);
 
   void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(e); }
 
+  void VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
+    emitAggLoadOfLValue(e);
+  }
+
   void VisitCallExpr(const CallExpr *e);
   void VisitStmtExpr(const StmtExpr *e) {
     CIRGenFunction::StmtExprEvaluation eval(cgf);
@@ -91,13 +98,6 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   }
 
   // Stubs -- These should be moved up when they are implemented.
-  void VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *e) {
-    // We shouldn't really get here, but we do because of missing handling for
-    // emitting constant aggregate initializers. If we just ignore this, a
-    // fallback handler will do the right thing.
-    assert(!cir::MissingFeatures::constEmitterAggILE());
-    return;
-  }
   void VisitCastExpr(CastExpr *e) {
     switch (e->getCastKind()) {
     case CK_LValueToRValue:
@@ -167,10 +167,6 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
     cgf.cgm.errorNYI(e->getSourceRange(),
                      "AggExprEmitter: VisitCompoundLiteralExpr");
   }
-  void VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
-    cgf.cgm.errorNYI(e->getSourceRange(),
-                     "AggExprEmitter: VisitArraySubscriptExpr");
-  }
   void VisitPredefinedExpr(const PredefinedExpr *e) {
     cgf.cgm.errorNYI(e->getSourceRange(),
                      "AggExprEmitter: VisitPredefinedExpr");
@@ -460,7 +456,31 @@ void AggExprEmitter::emitFinalDestCopy(QualType type, const LValue &src) {
   if (dest.isIgnored())
     return;
 
-  cgf.cgm.errorNYI("emitFinalDestCopy: non-ignored dest is NYI");
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+  assert(!cir::MissingFeatures::aggEmitFinalDestCopyRValue());
+  assert(!cir::MissingFeatures::aggValueSlotGC());
+
+  AggValueSlot srcAgg = AggValueSlot::forLValue(src, AggValueSlot::IsDestructed,
+                                                AggValueSlot::IsAliased,
+                                                AggValueSlot::MayOverlap);
+  emitCopy(type, dest, srcAgg);
+}
+
+/// Perform a copy from the source into the destination.
+///
+/// \param type - the type of the aggregate being copied; qualifiers are
+///   ignored
+void AggExprEmitter::emitCopy(QualType type, const AggValueSlot &dest,
+                              const AggValueSlot &src) {
+  assert(!cir::MissingFeatures::aggValueSlotGC());
+
+  // If the result of the assignment is used, copy the LHS there also.
+  // It's volatile if either side is.  Use the minimum alignment of
+  // the two sides.
+  LValue destLV = cgf.makeAddrLValue(dest.getAddress(), type);
+  LValue srcLV = cgf.makeAddrLValue(src.getAddress(), type);
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+  cgf.emitAggregateCopy(destLV, srcLV, type, dest.mayOverlap());
 }
 
 void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
@@ -712,6 +732,68 @@ void CIRGenFunction::emitAggExpr(const Expr *e, AggValueSlot slot) {
   AggExprEmitter(*this, slot).Visit(const_cast<Expr *>(e));
 }
 
+void CIRGenFunction::emitAggregateCopy(LValue dest, LValue src, QualType ty,
+                                       AggValueSlot::Overlap_t mayOverlap) {
+  // TODO(cir): this function needs improvements, commented code for now since
+  // this will be touched again soon.
+  assert(!ty->isAnyComplexType() && "Unexpected copy of complex");
+
+  Address destPtr = dest.getAddress();
+  Address srcPtr = src.getAddress();
+
+  if (getLangOpts().CPlusPlus) {
+    if (auto *record = ty->getAsCXXRecordDecl()) {
+      assert((record->hasTrivialCopyConstructor() ||
+              record->hasTrivialCopyAssignment() ||
+              record->hasTrivialMoveConstructor() ||
+              record->hasTrivialMoveAssignment() ||
+              record->hasAttr<TrivialABIAttr>() || record->isUnion()) &&
+             "Trying to aggregate-copy a type without a trivial copy/move "
+             "constructor or assignment operator");
+      // Ignore empty classes in C++.
+      if (record->isEmpty())
+        return;
+    }
+  }
+
+  assert(!cir::MissingFeatures::cudaSupport());
+
+  // Aggregate assignment turns into llvm.memcpy.  This is almost valid per
+  // C99 6.5.16.1p3, which states "If the value being stored in an object is
+  // read from another object that overlaps in anyway the storage of the first
+  // object, then the overlap shall be exact and the two objects shall have
+  // qualified or unqualified versions of a compatible type."
+  //
+  // memcpy is not defined if the source and destination pointers are exactly
+  // equal, but other compilers do this optimization, and almost every memcpy
+  // implementation handles this case safely.  If there is a libc that does not
+  // safely handle this, we can add a target hook.
+
+  // Get data size info for this aggregate. Don't copy the tail padding if this
+  // might be a potentially-overlapping subobject, since the tail padding might
+  // be occupied by a different object. Otherwise, copying it is fine.
+  TypeInfoChars typeInfo;
+  if (mayOverlap)
+    typeInfo = getContext().getTypeInfoDataSizeInChars(ty);
+  else
+    typeInfo = getContext().getTypeInfoInChars(ty);
+
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+
+  // NOTE(cir): original codegen would normally convert destPtr and srcPtr to
+  // i8* since memcpy operates on bytes. We don't need that in CIR because
+  // cir.copy will operate on any CIR pointer that points to a sized type.
+
+  // Don't do any of the memmove_collectable tests if GC isn't set.
+  if (cgm.getLangOpts().getGC() != LangOptions::NonGC)
+    cgm.errorNYI("emitAggregateCopy: GC");
+
+  [[maybe_unused]] cir::CopyOp copyOp =
+      builder.createCopy(destPtr.getPointer(), srcPtr.getPointer());
+
+  assert(!cir::MissingFeatures::opTBAA());
+}
+
 LValue CIRGenFunction::emitAggExprToLValue(const Expr *e) {
   assert(hasAggregateEvaluationKind(e->getType()) && "Invalid argument!");
   Address temp = createMemTemp(e->getType(), getLoc(e->getSourceRange()));

clang/lib/CIR/CodeGen/CIRGenFunction.h

@@ -965,6 +965,16 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   LValue emitAggExprToLValue(const Expr *e);
 
+  /// Emit an aggregate copy.
+  ///
+  /// \param isVolatile \c true iff either the source or the destination is
+  ///        volatile.
+  /// \param MayOverlap Whether the tail padding of the destination might be
+  ///        occupied by some other object. More efficient code can often be
+  ///        generated if not.
+  void emitAggregateCopy(LValue dest, LValue src, QualType eltTy,
+                         AggValueSlot::Overlap_t mayOverlap);
+
   /// Emit code to compute the specified expression which can have any type. The
   /// result is returned as an RValue struct. If this is an aggregate
   /// expression, the aggloc/agglocvolatile arguments indicate where the result

clang/lib/CIR/CodeGen/CIRGenValue.h

@@ -379,6 +379,8 @@ class AggValueSlot {
 
   mlir::Value getPointer() const { return addr.getPointer(); }
 
+  Overlap_t mayOverlap() const { return Overlap_t(overlapFlag); }
+
   IsZeroed_t isZeroed() const { return IsZeroed_t(zeroedFlag); }
 
   RValue asRValue() const {

clang/lib/CIR/Dialect/IR/CIRDialect.cpp

@@ -1919,6 +1919,21 @@ OpFoldResult cir::UnaryOp::fold(FoldAdaptor adaptor) {
   return {};
 }
 
+//===----------------------------------------------------------------------===//
+// CopyOp Definitions
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::CopyOp::verify() {
+  // A data layout is required for us to know the number of bytes to be copied.
+  if (!getType().getPointee().hasTrait<DataLayoutTypeInterface::Trait>())
+    return emitError() << "missing data layout for pointee type";
+
+  if (getSrc() == getDst())
+    return emitError() << "source and destination are the same";
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // GetMemberOp Definitions
 //===----------------------------------------------------------------------===//

clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp

@@ -118,6 +118,45 @@ DeletionKind cir::StoreOp::removeBlockingUses(
   return DeletionKind::Delete;
 }
 
+//===----------------------------------------------------------------------===//
+// Interfaces for CopyOp
+//===----------------------------------------------------------------------===//
+
+bool cir::CopyOp::loadsFrom(const MemorySlot &slot) {
+  return getSrc() == slot.ptr;
+}
+
+bool cir::CopyOp::storesTo(const MemorySlot &slot) {
+  return getDst() == slot.ptr;
+}
+
+Value cir::CopyOp::getStored(const MemorySlot &slot, OpBuilder &builder,
+                             Value reachingDef, const DataLayout &dataLayout) {
+  return cir::LoadOp::create(builder, getLoc(), slot.elemType, getSrc());
+}
+
+DeletionKind cir::CopyOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder, mlir::Value reachingDefinition,
+    const DataLayout &dataLayout) {
+  if (loadsFrom(slot))
+    cir::StoreOp::create(builder, getLoc(), reachingDefinition, getDst(),
+                         /*isVolatile=*/false,
+                         /*alignment=*/mlir::IntegerAttr{},
+                         /*mem-order=*/cir::MemOrderAttr());
+  return DeletionKind::Delete;
+}
+
+bool cir::CopyOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses,
+    const DataLayout &dataLayout) {
+  if (getDst() == getSrc())
+    return false;
+
+  return getLength(dataLayout) == dataLayout.getTypeSize(slot.elemType);
+}
+
 //===----------------------------------------------------------------------===//
 // Interfaces for CastOp
 //===----------------------------------------------------------------------===//

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp

@@ -173,6 +173,18 @@ mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {
   llvm_unreachable("Unknown CIR linkage type");
 }
 
+mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite(
+    cir::CopyOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
+  const mlir::Value length = mlir::LLVM::ConstantOp::create(
+      rewriter, op.getLoc(), rewriter.getI32Type(), op.getLength(layout));
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+  rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
+      op, adaptor.getDst(), adaptor.getSrc(), length, /*isVolatile=*/false);
+  return mlir::success();
+}
+
 static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter,
                                   mlir::Value llvmSrc, mlir::Type llvmDstIntTy,
                                   bool isUnsigned, uint64_t cirSrcWidth,
@@ -2430,6 +2442,7 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMComplexRealOpLowering,
                CIRToLLVMComplexRealPtrOpLowering,
                CIRToLLVMComplexSubOpLowering,
+               CIRToLLVMCopyOpLowering,
                CIRToLLVMConstantOpLowering,
                CIRToLLVMExpectOpLowering,
                CIRToLLVMFAbsOpLowering,

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h

@@ -170,6 +170,15 @@ class CIRToLLVMCastOpLowering : public mlir::OpConversionPattern<cir::CastOp> {
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMCopyOpLowering : public mlir::OpConversionPattern<cir::CopyOp> {
+public:
+  using mlir::OpConversionPattern<cir::CopyOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::CopyOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMExpectOpLowering
     : public mlir::OpConversionPattern<cir::ExpectOp> {
 public: