[CIR] Upstream framework for NRVO cleanup

This upstreams the framework code to handle NRVO cleanup. At this point
the implementation follows the incubator in not performing the actual
cleanup, but it populates the EH stack so that the cleanup can be done.

This inserts a flag variable that tracks whether we are performing an NRVO
return. Classic codegen checks this variable and calls a destructor if
it is not set during normal cleanup. The check and destructor call are
not yet implemented here.

Author: andykaylor

clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h

@@ -331,6 +331,11 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
     return cir::StoreOp::create(*this, loc, val, dst, isVolatile, align, order);
   }
 
+  cir::StoreOp createFlagStore(mlir::Location loc, bool val, mlir::Value dst) {
+    mlir::Value flag = getBool(val, loc);
+    return CIRBaseBuilderTy::createStore(loc, flag, dst);
+  }
+
   [[nodiscard]] cir::GlobalOp createGlobal(mlir::ModuleOp mlirModule,
                                            mlir::Location loc,
                                            mlir::StringRef name,

clang/include/clang/CIR/MissingFeatures.h

@@ -227,6 +227,7 @@ struct MissingFeatures {
   static bool countedBySize() { return false; }
   static bool cgFPOptionsRAII() { return false; }
   static bool checkBitfieldClipping() { return false; }
+  static bool cleanupDestroyNRVOVariable() { return false; }
   static bool cirgenABIInfo() { return false; }
   static bool cleanupAfterErrorDiags() { return false; }
   static bool cleanupAppendInsts() { return false; }

clang/lib/CIR/CodeGen/CIRGenDecl.cpp

@@ -98,8 +98,22 @@ CIRGenFunction::emitAutoVarAlloca(const VarDecl &d,
       if (const RecordDecl *rd = ty->getAsRecordDecl()) {
         if (const auto *cxxrd = dyn_cast<CXXRecordDecl>(rd);
             (cxxrd && !cxxrd->hasTrivialDestructor()) ||
-            rd->isNonTrivialToPrimitiveDestroy())
-          cgm.errorNYI(d.getSourceRange(), "emitAutoVarAlloca: set NRVO flag");
+            rd->isNonTrivialToPrimitiveDestroy()) {
+          // In LLVM: Create a flag that is used to indicate when the NRVO was
+          // applied to this variable. Set it to zero to indicate that NRVO was
+          // not applied. For now, use the same approach for CIRGen until we can
+          // be sure it's worth doing something more aggressive.
+          cir::ConstantOp falseNVRO = builder.getFalse(loc);
+          Address nrvoFlag = createTempAlloca(falseNVRO.getType(),
+                                              CharUnits::One(), loc, "nrvo",
+                                              /*ArraySize=*/nullptr, &address);
+          assert(builder.getInsertionBlock());
+          builder.createStore(loc, falseNVRO, nrvoFlag);
+
+          // Record the NRVO flag for this variable.
+          nrvoFlags[&d] = nrvoFlag.getPointer();
+          emission.nrvoFlag = nrvoFlag.getPointer();
+        }
       }
     } else {
       // A normal fixed sized variable becomes an alloca in the entry block,
@@ -809,6 +823,35 @@ struct DestroyObject final : EHScopeStack::Cleanup {
   }
 };
 
+template <class Derived> struct DestroyNRVOVariable : EHScopeStack::Cleanup {
+  DestroyNRVOVariable(Address addr, QualType type, mlir::Value nrvoFlag)
+      : nrvoFlag(nrvoFlag), addr(addr), ty(type) {}
+
+  mlir::Value nrvoFlag;
+  Address addr;
+  QualType ty;
+
+  void emit(CIRGenFunction &cgf) override {
+    assert(!cir::MissingFeatures::cleanupDestroyNRVOVariable());
+  }
+
+  virtual ~DestroyNRVOVariable() = default;
+};
+
+struct DestroyNRVOVariableCXX final
+    : DestroyNRVOVariable<DestroyNRVOVariableCXX> {
+  DestroyNRVOVariableCXX(Address addr, QualType type,
+                         const CXXDestructorDecl *dtor, mlir::Value nrvoFlag)
+      : DestroyNRVOVariable<DestroyNRVOVariableCXX>(addr, type, nrvoFlag),
+        dtor(dtor) {}
+
+  const CXXDestructorDecl *dtor;
+
+  void emitDestructorCall(CIRGenFunction &cgf) {
+    assert(!cir::MissingFeatures::cleanupDestroyNRVOVariable());
+  }
+};
+
 struct CallStackRestore final : EHScopeStack::Cleanup {
   Address stack;
   CallStackRestore(Address stack) : stack(stack) {}
@@ -965,7 +1008,10 @@ void CIRGenFunction::emitAutoVarTypeCleanup(
     // If there's an NRVO flag on the emission, we need a different
     // cleanup.
     if (emission.nrvoFlag) {
-      cgm.errorNYI(var->getSourceRange(), "emitAutoVarTypeCleanup: NRVO");
+      assert(!type->isArrayType());
+      CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor();
+      ehStack.pushCleanup<DestroyNRVOVariableCXX>(cleanupKind, addr, type, dtor,
+                                                  emission.nrvoFlag);
       return;
     }
     // Otherwise, this is handled below.

clang/lib/CIR/CodeGen/CIRGenFunction.h

@@ -123,6 +123,10 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   GlobalDecl curSEHParent;
 
+  /// A mapping from NRVO variables to the flags used to indicate
+  /// when the NRVO has been applied to this variable.
+  llvm::DenseMap<const VarDecl *, mlir::Value> nrvoFlags;
+
   llvm::DenseMap<const clang::ValueDecl *, clang::FieldDecl *>
       lambdaCaptureFields;
   clang::FieldDecl *lambdaThisCaptureField = nullptr;

clang/lib/CIR/CodeGen/CIRGenStmt.cpp

@@ -458,7 +458,14 @@ mlir::LogicalResult CIRGenFunction::emitReturnStmt(const ReturnStmt &s) {
     if (getContext().getLangOpts().ElideConstructors && s.getNRVOCandidate() &&
         s.getNRVOCandidate()->isNRVOVariable()) {
       assert(!cir::MissingFeatures::openMP());
-      assert(!cir::MissingFeatures::nrvo());
+      // Apply the named return value optimization for this return statement,
+      // which means doing nothing: the appropriate result has already been
+      // constructed into the NRVO variable.
+
+      // If there is an NRVO flag for this variable, set it to 1 into indicate
+      // that the cleanup code should not destroy the variable.
+      if (auto nrvoFlag = nrvoFlags[s.getNRVOCandidate()])
+        builder.createFlagStore(loc, true, nrvoFlag);
     } else if (!rv) {
       // No return expression. Do nothing.
     } else if (rv->getType()->isVoidType()) {

clang/test/CIR/CodeGen/nrvo.cpp

@@ -49,3 +49,53 @@ struct S f1() {
 // OGCG-NEXT:   call void @_ZN1SC1Ev(ptr {{.*}} %[[RETVAL]])
 // OGCG-NEXT:   %[[RET:.*]] = load i64, ptr %[[RETVAL]]
 // OGCG-NEXT:   ret i64 %[[RET]]
+
+struct NonTrivial {
+  ~NonTrivial();
+};
+
+void maybeThrow();
+
+NonTrivial test_nrvo() {
+  NonTrivial result;
+  maybeThrow();
+  return result;
+}
+
+// TODO(cir): Handle normal cleanup properly.
+
+// CIR: cir.func {{.*}} @_Z9test_nrvov()
+// CIR:   %[[RESULT:.*]] = cir.alloca !rec_NonTrivial, !cir.ptr<!rec_NonTrivial>, ["__retval"]
+// CIR:   %[[NRVO_FLAG:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["nrvo"]
+// CIR:   %[[FALSE:.*]] = cir.const #false
+// CIR:   cir.store{{.*}} %[[FALSE]], %[[NRVO_FLAG]]
+// CIR:   cir.call @_Z10maybeThrowv() : () -> ()
+// CIR:   %[[TRUE:.*]] = cir.const #true
+// CIR:   cir.store{{.*}} %[[TRUE]], %[[NRVO_FLAG]]
+// CIR:   %[[RET:.*]] = cir.load %[[RESULT]]
+// CIR:   cir.return %[[RET]]
+
+// LLVM: define {{.*}} %struct.NonTrivial @_Z9test_nrvov()
+// LLVM:   %[[RESULT:.*]] = alloca %struct.NonTrivial
+// LLVM:   %[[NRVO_FLAG:.*]] = alloca i8
+// LLVM:   store i8 0, ptr %[[NRVO_FLAG]]
+// LLVM:   call void @_Z10maybeThrowv()
+// LLVM:   store i8 1, ptr %[[NRVO_FLAG]]
+// LLVM:   %[[RET:.*]] = load %struct.NonTrivial, ptr %[[RESULT]]
+// LLVM:   ret %struct.NonTrivial %[[RET]]
+
+// OGCG: define {{.*}} void @_Z9test_nrvov(ptr {{.*}} sret(%struct.NonTrivial) {{.*}} %[[RESULT:.*]])
+// OGCG:   %[[RESULT_ADDR:.*]] = alloca ptr
+// OGCG:   %[[NRVO_FLAG:.*]] = alloca i1, align 1
+// OGCG:   store ptr %[[RESULT]], ptr %[[RESULT_ADDR]]
+// OGCG:   store i1 false, ptr %[[NRVO_FLAG]]
+// OGCG:   call void @_Z10maybeThrowv()
+// OGCG:   store i1 true, ptr %[[NRVO_FLAG]]
+// OGCG:   %[[NRVO_VAL:.*]] = load i1, ptr %[[NRVO_FLAG]]
+// OGCG:   br i1 %[[NRVO_VAL]], label %[[SKIPDTOR:.*]], label %[[NRVO_UNUSED:.*]]
+// OGCG: [[NRVO_UNUSED]]:
+// OGCG:   call void @_ZN10NonTrivialD1Ev(ptr {{.*}} %[[RESULT]])
+// OGCG:   br label %[[SKIPDTOR]]
+// OGCG: [[SKIPDTOR]]:
+// OGCG:   ret void
+