[CIR] Implement NRVO variable cleanup

This implements the cleanup handling for C++ NRVO variables. Because
exception handling is still incomplete, this doesn't behave any differently
with exceptions enabled yet, but the code for that case is trivial.

Author: andykaylor

clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h

@@ -331,6 +331,14 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return cir::StoreOp::create(*this, loc, val, dst, isVolatile, align, order);
   }
 
+  /// Emit a load from an boolean flag variable.
+  cir::LoadOp createFlagLoad(mlir::Location loc, mlir::Value addr) {
+    mlir::Type boolTy = getBoolTy();
+    if (boolTy != mlir::cast<cir::PointerType>(addr.getType()).getPointee())
+      addr = createPtrBitcast(addr, boolTy);
+    return createLoad(loc, addr, /*isVolatile=*/false, /*alignment=*/1);
+  }
+
   cir::StoreOp createFlagStore(mlir::Location loc, bool val, mlir::Value dst) {
     mlir::Value flag = getBool(val, loc);
     return CIRBaseBuilderTy::createStore(loc, flag, dst);

clang/include/clang/CIR/MissingFeatures.h

@@ -227,7 +227,6 @@ struct MissingFeatures {
   static bool countedBySize() { return false; }
   static bool cgFPOptionsRAII() { return false; }
   static bool checkBitfieldClipping() { return false; }
-  static bool cleanupDestroyNRVOVariable() { return false; }
   static bool cirgenABIInfo() { return false; }
   static bool cleanupAfterErrorDiags() { return false; }
   static bool cleanupAppendInsts() { return false; }

clang/lib/CIR/CodeGen/CIRGenDecl.cpp

@@ -106,7 +106,7 @@ CIRGenFunction::emitAutoVarAlloca(const VarDecl &d,
           cir::ConstantOp falseNVRO = builder.getFalse(loc);
           Address nrvoFlag = createTempAlloca(falseNVRO.getType(),
                                               CharUnits::One(), loc, "nrvo",
-                                              /*arraySize=*/nullptr, &address);
+                                              /*arraySize=*/nullptr);
           assert(builder.getInsertionBlock());
           builder.createStore(loc, falseNVRO, nrvoFlag);
 
@@ -835,7 +835,24 @@ template <class Derived> struct DestroyNRVOVariable : EHScopeStack::Cleanup {
   QualType ty;
 
   void emit(CIRGenFunction &cgf, Flags flags) override {
-    assert(!cir::MissingFeatures::cleanupDestroyNRVOVariable());
+    // Along the exceptions path we always execute the dtor.
+    bool nrvo = flags.isForNormalCleanup() && nrvoFlag;
+
+    CIRGenBuilderTy &builder = cgf.getBuilder();
+    mlir::OpBuilder::InsertionGuard guard(builder);
+    mlir::Location loc = addr.getPointer().getLoc();
+    if (nrvo) {
+      // If we exited via NRVO, we skip the destructor call.
+      mlir::Value didNRVO = builder.createFlagLoad(loc, nrvoFlag);
+      mlir::Value notNRVO = builder.createNot(didNRVO);
+      cir::IfOp::create(builder, loc, notNRVO, /*withElseRegion=*/false,
+                        [&](mlir::OpBuilder &b, mlir::Location) {
+                          static_cast<Derived *>(this)->emitDestructorCall(cgf);
+                          builder.createYield(loc);
+                        });
+    } else {
+      static_cast<Derived *>(this)->emitDestructorCall(cgf);
+    }
   }
 
   virtual ~DestroyNRVOVariable() = default;
@@ -851,7 +868,9 @@ struct DestroyNRVOVariableCXX final
   const CXXDestructorDecl *dtor;
 
   void emitDestructorCall(CIRGenFunction &cgf) {
-    assert(!cir::MissingFeatures::cleanupDestroyNRVOVariable());
+    cgf.emitCXXDestructorCall(dtor, Dtor_Complete,
+                              /*forVirtualBase=*/false,
+                              /*delegating=*/false, addr, ty);
   }
 };
 

clang/test/CIR/CodeGen/nrvo.cpp

@@ -72,6 +72,11 @@ NonTrivial test_nrvo() {
 // CIR:   cir.call @_Z10maybeThrowv() : () -> ()
 // CIR:   %[[TRUE:.*]] = cir.const #true
 // CIR:   cir.store{{.*}} %[[TRUE]], %[[NRVO_FLAG]]
+// CIR:   %[[NRVO_FLAG_VAL:.*]] = cir.load{{.*}} %[[NRVO_FLAG]]
+// CIR:   %[[NOT_NRVO_VAL:.*]] = cir.unary(not, %[[NRVO_FLAG_VAL]])
+// CIR:   cir.if %[[NOT_NRVO_VAL]] {
+// CIR:     cir.call @_ZN10NonTrivialD1Ev(%[[RESULT]])
+// CIR:   }
 // CIR:   %[[RET:.*]] = cir.load %[[RESULT]]
 // CIR:   cir.return %[[RET]]
 
@@ -81,6 +86,14 @@ NonTrivial test_nrvo() {
 // LLVM:   store i8 0, ptr %[[NRVO_FLAG]]
 // LLVM:   call void @_Z10maybeThrowv()
 // LLVM:   store i8 1, ptr %[[NRVO_FLAG]]
+// LLVM:   %[[NRVO_VAL:.*]] = load i8, ptr %[[NRVO_FLAG]]
+// LLVM:   %[[NRVO_VAL_TRUNC:.*]] = trunc i8 %[[NRVO_VAL]] to i1
+// LLVM:   %[[NOT_NRVO_VAL:.*]] = xor i1 %[[NRVO_VAL_TRUNC]], true
+// LLVM:   br i1 %[[NOT_NRVO_VAL]], label %[[NRVO_UNUSED:.*]], label %[[NRVO_USED:.*]]
+// LLVM: [[NRVO_UNUSED]]:
+// LLVM:   call void @_ZN10NonTrivialD1Ev(ptr %[[RESULT]])
+// LLVM:   br label %[[NRVO_USED]]
+// LLVM: [[NRVO_USED]]:
 // LLVM:   %[[RET:.*]] = load %struct.NonTrivial, ptr %[[RESULT]]
 // LLVM:   ret %struct.NonTrivial %[[RET]]