[CIR] Upstream framework for NRVO cleanup

clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h

@@ -331,6 +331,11 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return cir::StoreOp::create(*this, loc, val, dst, isVolatile, align, order);
   }
 
+  cir::StoreOp createFlagStore(mlir::Location loc, bool val, mlir::Value dst) {
+    mlir::Value flag = getBool(val, loc);
+    return CIRBaseBuilderTy::createStore(loc, flag, dst);
+  }
+
   [[nodiscard]] cir::GlobalOp createGlobal(mlir::ModuleOp mlirModule,
                                            mlir::Location loc,
                                            mlir::StringRef name,

clang/include/clang/CIR/MissingFeatures.h

@@ -227,6 +227,7 @@ struct MissingFeatures {
   static bool countedBySize() { return false; }
   static bool cgFPOptionsRAII() { return false; }
   static bool checkBitfieldClipping() { return false; }
+  static bool cleanupDestroyNRVOVariable() { return false; }
   static bool cirgenABIInfo() { return false; }
   static bool cleanupAfterErrorDiags() { return false; }
   static bool cleanupAppendInsts() { return false; }

clang/lib/CIR/CodeGen/CIRGenDecl.cpp

@@ -98,8 +98,22 @@ CIRGenFunction::emitAutoVarAlloca(const VarDecl &d,
       if (const RecordDecl *rd = ty->getAsRecordDecl()) {
         if (const auto *cxxrd = dyn_cast<CXXRecordDecl>(rd);
             (cxxrd && !cxxrd->hasTrivialDestructor()) ||
-            rd->isNonTrivialToPrimitiveDestroy())
-          cgm.errorNYI(d.getSourceRange(), "emitAutoVarAlloca: set NRVO flag");
+            rd->isNonTrivialToPrimitiveDestroy()) {
+          // In LLVM: Create a flag that is used to indicate when the NRVO was
+          // applied to this variable. Set it to zero to indicate that NRVO was
+          // not applied. For now, use the same approach for CIRGen until we can
+          // be sure it's worth doing something more aggressive.
+          cir::ConstantOp falseNVRO = builder.getFalse(loc);
+          Address nrvoFlag = createTempAlloca(falseNVRO.getType(),
+                                              CharUnits::One(), loc, "nrvo",
+                                              /*arraySize=*/nullptr, &address);
+          assert(builder.getInsertionBlock());
+          builder.createStore(loc, falseNVRO, nrvoFlag);
+
+          // Record the NRVO flag for this variable.
+          nrvoFlags[&d] = nrvoFlag.getPointer();
+          emission.nrvoFlag = nrvoFlag.getPointer();
+        }
       }
     } else {
       // A normal fixed sized variable becomes an alloca in the entry block,
@@ -809,6 +823,35 @@ struct DestroyObject final : EHScopeStack::Cleanup {
   }
 };
 
+template <class Derived> struct DestroyNRVOVariable : EHScopeStack::Cleanup {
+  DestroyNRVOVariable(Address addr, QualType type, mlir::Value nrvoFlag)
+      : nrvoFlag(nrvoFlag), addr(addr), ty(type) {}
+
+  mlir::Value nrvoFlag;
+  Address addr;
+  QualType ty;
+
+  void emit(CIRGenFunction &cgf) override {
+    assert(!cir::MissingFeatures::cleanupDestroyNRVOVariable());
+  }
+
+  virtual ~DestroyNRVOVariable() = default;
+};
+
+struct DestroyNRVOVariableCXX final
+    : DestroyNRVOVariable<DestroyNRVOVariableCXX> {
+  DestroyNRVOVariableCXX(Address addr, QualType type,
+                         const CXXDestructorDecl *dtor, mlir::Value nrvoFlag)
+      : DestroyNRVOVariable<DestroyNRVOVariableCXX>(addr, type, nrvoFlag),
+        dtor(dtor) {}
+
+  const CXXDestructorDecl *dtor;
+
+  void emitDestructorCall(CIRGenFunction &cgf) {
+    assert(!cir::MissingFeatures::cleanupDestroyNRVOVariable());
+  }
+};
+
 struct CallStackRestore final : EHScopeStack::Cleanup {
   Address stack;
   CallStackRestore(Address stack) : stack(stack) {}
@@ -965,7 +1008,10 @@ void CIRGenFunction::emitAutoVarTypeCleanup(
     // If there's an NRVO flag on the emission, we need a different
     // cleanup.
     if (emission.nrvoFlag) {
-      cgm.errorNYI(var->getSourceRange(), "emitAutoVarTypeCleanup: NRVO");
+      assert(!type->isArrayType());
+      CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor();
+      ehStack.pushCleanup<DestroyNRVOVariableCXX>(cleanupKind, addr, type, dtor,
+                                                  emission.nrvoFlag);
       return;
     }
     // Otherwise, this is handled below.

clang/lib/CIR/CodeGen/CIRGenFunction.h

@@ -123,6 +123,10 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   GlobalDecl curSEHParent;
 
+  /// A mapping from NRVO variables to the flags used to indicate
+  /// when the NRVO has been applied to this variable.
+  llvm::DenseMap<const VarDecl *, mlir::Value> nrvoFlags;
+
   llvm::DenseMap<const clang::ValueDecl *, clang::FieldDecl *>
       lambdaCaptureFields;
   clang::FieldDecl *lambdaThisCaptureField = nullptr;

clang/lib/CIR/CodeGen/CIRGenStmt.cpp

@@ -458,7 +458,14 @@ mlir::LogicalResult CIRGenFunction::emitReturnStmt(const ReturnStmt &s) {
     if (getContext().getLangOpts().ElideConstructors && s.getNRVOCandidate() &&
         s.getNRVOCandidate()->isNRVOVariable()) {
       assert(!cir::MissingFeatures::openMP());
-      assert(!cir::MissingFeatures::nrvo());
+      // Apply the named return value optimization for this return statement,
+      // which means doing nothing: the appropriate result has already been
+      // constructed into the NRVO variable.
+
+      // If there is an NRVO flag for this variable, set it to 1 into indicate
+      // that the cleanup code should not destroy the variable.
+      if (auto nrvoFlag = nrvoFlags[s.getNRVOCandidate()])
+        builder.createFlagStore(loc, true, nrvoFlag);
     } else if (!rv) {
       // No return expression. Do nothing.
     } else if (rv->getType()->isVoidType()) {

clang/test/CIR/CodeGen/nrvo.cpp

@@ -49,3 +49,52 @@ struct S f1() {
 // OGCG-NEXT:   call void @_ZN1SC1Ev(ptr {{.*}} %[[RETVAL]])
 // OGCG-NEXT:   %[[RET:.*]] = load i64, ptr %[[RETVAL]]
 // OGCG-NEXT:   ret i64 %[[RET]]
+
+struct NonTrivial {
+  ~NonTrivial();
+};
+
+void maybeThrow();
+
+NonTrivial test_nrvo() {
+  NonTrivial result;
+  maybeThrow();
+  return result;
+}
+
+// TODO(cir): Handle normal cleanup properly.
+
+// CIR: cir.func {{.*}} @_Z9test_nrvov()
+// CIR:   %[[RESULT:.*]] = cir.alloca !rec_NonTrivial, !cir.ptr<!rec_NonTrivial>, ["__retval"]
+// CIR:   %[[NRVO_FLAG:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["nrvo"]
+// CIR:   %[[FALSE:.*]] = cir.const #false
+// CIR:   cir.store{{.*}} %[[FALSE]], %[[NRVO_FLAG]]
+// CIR:   cir.call @_Z10maybeThrowv() : () -> ()
+// CIR:   %[[TRUE:.*]] = cir.const #true
+// CIR:   cir.store{{.*}} %[[TRUE]], %[[NRVO_FLAG]]
+// CIR:   %[[RET:.*]] = cir.load %[[RESULT]]
+// CIR:   cir.return %[[RET]]
+
+// LLVM: define {{.*}} %struct.NonTrivial @_Z9test_nrvov()
+// LLVM:   %[[RESULT:.*]] = alloca %struct.NonTrivial
+// LLVM:   %[[NRVO_FLAG:.*]] = alloca i8
+// LLVM:   store i8 0, ptr %[[NRVO_FLAG]]
+// LLVM:   call void @_Z10maybeThrowv()
+// LLVM:   store i8 1, ptr %[[NRVO_FLAG]]
+// LLVM:   %[[RET:.*]] = load %struct.NonTrivial, ptr %[[RESULT]]
+// LLVM:   ret %struct.NonTrivial %[[RET]]
+
+// OGCG: define {{.*}} void @_Z9test_nrvov(ptr {{.*}} sret(%struct.NonTrivial) {{.*}} %[[RESULT:.*]])
+// OGCG:   %[[RESULT_ADDR:.*]] = alloca ptr
+// OGCG:   %[[NRVO_FLAG:.*]] = alloca i1, align 1
+// OGCG:   store ptr %[[RESULT]], ptr %[[RESULT_ADDR]]
+// OGCG:   store i1 false, ptr %[[NRVO_FLAG]]
+// OGCG:   call void @_Z10maybeThrowv()
+// OGCG:   store i1 true, ptr %[[NRVO_FLAG]]
+// OGCG:   %[[NRVO_VAL:.*]] = load i1, ptr %[[NRVO_FLAG]]
+// OGCG:   br i1 %[[NRVO_VAL]], label %[[SKIPDTOR:.*]], label %[[NRVO_UNUSED:.*]]
+// OGCG: [[NRVO_UNUSED]]:
+// OGCG:   call void @_ZN10NonTrivialD1Ev(ptr {{.*}} %[[RESULT]])
+// OGCG:   br label %[[SKIPDTOR]]
+// OGCG: [[SKIPDTOR]]:
+// OGCG:   ret void