[CIR] Emit copy for aggregate initialization

This adds the implementation of aggEmitFinalDestCopy for the case where
the destination value is not ignored. This requires adding the cir.copy
operation and associated interface code.

Author: andykaylor

clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h

@@ -245,6 +245,11 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return createGetGlobal(global.getLoc(), global);
   }
 
+  /// Create a copy with inferred length.
+  cir::CopyOp createCopy(mlir::Value dst, mlir::Value src) {
+    return cir::CopyOp::create(*this, dst.getLoc(), dst, src);
+  }
+
   cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
                            bool isVolatile = false,
                            mlir::IntegerAttr align = {},

clang/include/clang/CIR/Dialect/IR/CIROps.td

@@ -2451,6 +2451,51 @@ def CIR_CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// CopyOp
+//===----------------------------------------------------------------------===//
+
+def CIR_CopyOp : CIR_Op<"copy",[
+  SameTypeOperands,
+  DeclareOpInterfaceMethods<PromotableMemOpInterface>
+]> {
+  let arguments = (ins
+      Arg<CIR_PointerType, "", [MemWrite]>:$dst,
+      Arg<CIR_PointerType, "", [MemRead]>:$src
+  );
+  let summary = "Copies contents from a CIR pointer to another";
+  let description = [{
+    Given two CIR pointers, `src` and `dst`, `cir.copy` will copy the memory
+    pointed by `src` to the memory pointed by `dst`.
+
+    The number of bytes copied is inferred from the pointee type. The pointee
+    type of `src` and `dst` must match and both must implement the
+    `DataLayoutTypeInterface`.
+
+    Examples:
+
+    ```mlir
+      // Copying contents from one record to another:
+      cir.copy %0 to %1 : !cir.ptr<!record_ty>
+    ```
+  }];
+
+  let assemblyFormat = [{$src `to` $dst
+                        attr-dict `:` qualified(type($dst))
+  }];
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    /// Returns the pointer type being copied.
+    cir::PointerType getType() { return getSrc().getType(); }
+
+    /// Returns the number of bytes to be copied.
+    unsigned getLength() {
+      return mlir::DataLayout::closest(*this).getTypeSize(getType().getPointee());
+    }
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // ReturnAddrOp and FrameAddrOp
 //===----------------------------------------------------------------------===//

clang/include/clang/CIR/MissingFeatures.h

@@ -168,6 +168,7 @@ struct MissingFeatures {
   // Misc
   static bool abiArgInfo() { return false; }
   static bool addHeapAllocSiteMetadata() { return false; }
+  static bool aggEmitFinalDestCopyRValue() { return false; }
   static bool aggValueSlot() { return false; }
   static bool aggValueSlotAlias() { return false; }
   static bool aggValueSlotDestructedFlag() { return false; }

clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp

@@ -62,12 +62,19 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   /// Perform the final copy to DestPtr, if desired.
   void emitFinalDestCopy(QualType type, const LValue &src);
 
+  void emitCopy(QualType type, const AggValueSlot &dest,
+                const AggValueSlot &src);
+
   void emitInitializationToLValue(Expr *e, LValue lv);
 
   void emitNullInitializationToLValue(mlir::Location loc, LValue lv);
 
   void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(e); }
 
+  void VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
+    emitAggLoadOfLValue(e);
+  }
+
   void VisitCallExpr(const CallExpr *e);
   void VisitStmtExpr(const StmtExpr *e) {
     CIRGenFunction::StmtExprEvaluation eval(cgf);
@@ -91,13 +98,6 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   }
 
   // Stubs -- These should be moved up when they are implemented.
-  void VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *e) {
-    // We shouldn't really get here, but we do because of missing handling for
-    // emitting constant aggregate initializers. If we just ignore this, a
-    // fallback handler will do the right thing.
-    assert(!cir::MissingFeatures::constEmitterAggILE());
-    return;
-  }
   void VisitCastExpr(CastExpr *e) {
     switch (e->getCastKind()) {
     case CK_LValueToRValue:
@@ -167,10 +167,6 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
     cgf.cgm.errorNYI(e->getSourceRange(),
                      "AggExprEmitter: VisitCompoundLiteralExpr");
   }
-  void VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
-    cgf.cgm.errorNYI(e->getSourceRange(),
-                     "AggExprEmitter: VisitArraySubscriptExpr");
-  }
   void VisitPredefinedExpr(const PredefinedExpr *e) {
     cgf.cgm.errorNYI(e->getSourceRange(),
                      "AggExprEmitter: VisitPredefinedExpr");
@@ -460,7 +456,31 @@ void AggExprEmitter::emitFinalDestCopy(QualType type, const LValue &src) {
   if (dest.isIgnored())
     return;
 
-  cgf.cgm.errorNYI("emitFinalDestCopy: non-ignored dest is NYI");
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+  assert(!cir::MissingFeatures::aggEmitFinalDestCopyRValue());
+  assert(!cir::MissingFeatures::aggValueSlotGC());
+
+  AggValueSlot srcAgg = AggValueSlot::forLValue(src, AggValueSlot::IsDestructed,
+                                                AggValueSlot::IsAliased,
+                                                AggValueSlot::MayOverlap);
+  emitCopy(type, dest, srcAgg);
+}
+
+/// Perform a copy from the source into the destination.
+///
+/// \param type - the type of the aggregate being copied; qualifiers are
+///   ignored
+void AggExprEmitter::emitCopy(QualType type, const AggValueSlot &dest,
+                              const AggValueSlot &src) {
+  assert(!cir::MissingFeatures::aggValueSlotGC());
+
+  // If the result of the assignment is used, copy the LHS there also.
+  // It's volatile if either side is.  Use the minimum alignment of
+  // the two sides.
+  LValue destLV = cgf.makeAddrLValue(dest.getAddress(), type);
+  LValue srcLV = cgf.makeAddrLValue(src.getAddress(), type);
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+  cgf.emitAggregateCopy(destLV, srcLV, type, dest.mayOverlap());
 }
 
 void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
@@ -712,6 +732,68 @@ void CIRGenFunction::emitAggExpr(const Expr *e, AggValueSlot slot) {
   AggExprEmitter(*this, slot).Visit(const_cast<Expr *>(e));
 }
 
+void CIRGenFunction::emitAggregateCopy(LValue dest, LValue src, QualType ty,
+                                       AggValueSlot::Overlap_t mayOverlap) {
+  // TODO(cir): this function needs improvements, commented code for now since
+  // this will be touched again soon.
+  assert(!ty->isAnyComplexType() && "Unexpected copy of complex");
+
+  Address destPtr = dest.getAddress();
+  Address srcPtr = src.getAddress();
+
+  if (getLangOpts().CPlusPlus) {
+    if (auto *record = ty->getAsCXXRecordDecl()) {
+      assert((record->hasTrivialCopyConstructor() ||
+              record->hasTrivialCopyAssignment() ||
+              record->hasTrivialMoveConstructor() ||
+              record->hasTrivialMoveAssignment() ||
+              record->hasAttr<TrivialABIAttr>() || record->isUnion()) &&
+             "Trying to aggregate-copy a type without a trivial copy/move "
+             "constructor or assignment operator");
+      // Ignore empty classes in C++.
+      if (record->isEmpty())
+        return;
+    }
+  }
+
+  assert(!cir::MissingFeatures::cudaSupport());
+
+  // Aggregate assignment turns into llvm.memcpy.  This is almost valid per
+  // C99 6.5.16.1p3, which states "If the value being stored in an object is
+  // read from another object that overlaps in anyway the storage of the first
+  // object, then the overlap shall be exact and the two objects shall have
+  // qualified or unqualified versions of a compatible type."
+  //
+  // memcpy is not defined if the source and destination pointers are exactly
+  // equal, but other compilers do this optimization, and almost every memcpy
+  // implementation handles this case safely.  If there is a libc that does not
+  // safely handle this, we can add a target hook.
+
+  // Get data size info for this aggregate. Don't copy the tail padding if this
+  // might be a potentially-overlapping subobject, since the tail padding might
+  // be occupied by a different object. Otherwise, copying it is fine.
+  TypeInfoChars typeInfo;
+  if (mayOverlap)
+    typeInfo = getContext().getTypeInfoDataSizeInChars(ty);
+  else
+    typeInfo = getContext().getTypeInfoInChars(ty);
+
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+
+  // NOTE(cir): original codegen would normally convert destPtr and srcPtr to
+  // i8* since memcpy operates on bytes. We don't need that in CIR because
+  // cir.copy will operate on any CIR pointer that points to a sized type.
+
+  // Don't do any of the memmove_collectable tests if GC isn't set.
+  if (cgm.getLangOpts().getGC() != LangOptions::NonGC)
+    cgm.errorNYI("emitAggregateCopy: GC");
+
+  [[maybe_unused]] cir::CopyOp copyOp =
+      builder.createCopy(destPtr.getPointer(), srcPtr.getPointer());
+
+  assert(!cir::MissingFeatures::opTBAA());
+}
+
 LValue CIRGenFunction::emitAggExprToLValue(const Expr *e) {
   assert(hasAggregateEvaluationKind(e->getType()) && "Invalid argument!");
   Address temp = createMemTemp(e->getType(), getLoc(e->getSourceRange()));

clang/lib/CIR/CodeGen/CIRGenFunction.h

@@ -965,6 +965,16 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   LValue emitAggExprToLValue(const Expr *e);
 
+  /// Emit an aggregate copy.
+  ///
+  /// \param isVolatile \c true iff either the source or the destination is
+  ///        volatile.
+  /// \param MayOverlap Whether the tail padding of the destination might be
+  ///        occupied by some other object. More efficient code can often be
+  ///        generated if not.
+  void emitAggregateCopy(LValue dest, LValue src, QualType eltTy,
+                         AggValueSlot::Overlap_t mayOverlap);
+
   /// Emit code to compute the specified expression which can have any type. The
   /// result is returned as an RValue struct. If this is an aggregate
   /// expression, the aggloc/agglocvolatile arguments indicate where the result

clang/lib/CIR/CodeGen/CIRGenValue.h

@@ -379,6 +379,8 @@ class AggValueSlot {
 
   mlir::Value getPointer() const { return addr.getPointer(); }
 
+  Overlap_t mayOverlap() const { return Overlap_t(overlapFlag); }
+
   IsZeroed_t isZeroed() const { return IsZeroed_t(zeroedFlag); }
 
   RValue asRValue() const {

clang/lib/CIR/Dialect/IR/CIRDialect.cpp

@@ -1919,6 +1919,21 @@ OpFoldResult cir::UnaryOp::fold(FoldAdaptor adaptor) {
   return {};
 }
 
+//===----------------------------------------------------------------------===//
+// CopyOp Definitions
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::CopyOp::verify() {
+  // A data layout is required for us to know the number of bytes to be copied.
+  if (!getType().getPointee().hasTrait<DataLayoutTypeInterface::Trait>())
+    return emitError() << "missing data layout for pointee type";
+
+  if (getSrc() == getDst())
+    return emitError() << "source and destination are the same";
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // GetMemberOp Definitions
 //===----------------------------------------------------------------------===//

clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp

@@ -118,6 +118,44 @@ DeletionKind cir::StoreOp::removeBlockingUses(
   return DeletionKind::Delete;
 }
 
+//===----------------------------------------------------------------------===//
+// Interfaces for CopyOp
+//===----------------------------------------------------------------------===//
+
+bool cir::CopyOp::loadsFrom(const MemorySlot &slot) {
+  return getSrc() == slot.ptr;
+}
+
+bool cir::CopyOp::storesTo(const MemorySlot &slot) {
+  return getDst() == slot.ptr;
+}
+
+Value cir::CopyOp::getStored(const MemorySlot &slot, OpBuilder &builder,
+                             Value reachingDef, const DataLayout &dataLayout) {
+  return cir::LoadOp::create(builder, getLoc(), slot.elemType, getSrc());
+}
+
+DeletionKind cir::CopyOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder, mlir::Value reachingDefinition,
+    const DataLayout &dataLayout) {
+  if (loadsFrom(slot))
+    cir::StoreOp::create(builder, getLoc(), reachingDefinition, getDst(),
+                         /*alignment=*/mlir::IntegerAttr{},
+                         /*mem-order=*/cir::MemOrderAttr());
+  return DeletionKind::Delete;
+}
+
+bool cir::CopyOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses,
+    const DataLayout &dataLayout) {
+  if (getDst() == getSrc())
+    return false;
+
+  return getLength() == dataLayout.getTypeSize(slot.elemType);
+}
+
 //===----------------------------------------------------------------------===//
 // Interfaces for CastOp
 //===----------------------------------------------------------------------===//

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp

@@ -173,6 +173,17 @@ mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {
   llvm_unreachable("Unknown CIR linkage type");
 }
 
+mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite(
+    cir::CopyOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  const mlir::Value length = mlir::LLVM::ConstantOp::create(
+      rewriter, op.getLoc(), rewriter.getI32Type(), op.getLength());
+  assert(!cir::MissingFeatures::aggValueSlotVolatile());
+  rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
+      op, adaptor.getDst(), adaptor.getSrc(), length, /*isVolatile=*/false);
+  return mlir::success();
+}
+
 static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter,
                                   mlir::Value llvmSrc, mlir::Type llvmDstIntTy,
                                   bool isUnsigned, uint64_t cirSrcWidth,
@@ -2430,6 +2441,7 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMComplexRealOpLowering,
                CIRToLLVMComplexRealPtrOpLowering,
                CIRToLLVMComplexSubOpLowering,
+               CIRToLLVMCopyOpLowering,
                CIRToLLVMConstantOpLowering,
                CIRToLLVMExpectOpLowering,
                CIRToLLVMFAbsOpLowering,

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h

@@ -170,6 +170,15 @@ class CIRToLLVMCastOpLowering : public mlir::OpConversionPattern<cir::CastOp> {
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMCopyOpLowering : public mlir::OpConversionPattern<cir::CopyOp> {
+public:
+  using mlir::OpConversionPattern<cir::CopyOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::CopyOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMExpectOpLowering
     : public mlir::OpConversionPattern<cir::ExpectOp> {
 public: