Elide suspension points via [[clang::coro_await_suspend_destroy]]

clang/docs/ReleaseNotes.rst

@@ -159,6 +159,14 @@ Removed Compiler Flags
 Attribute Changes in Clang
 --------------------------
 
+- Introduced a new attribute ``[[clang::coro_await_suspend_destroy]]``.  When
+  applied to an ``await_suspend(std::coroutine_handle<Promise>)`` member of a
+  coroutine awaiter, it causes suspensions into this awaiter to use a new
+  ``await_suspend_destroy(Promise&)`` method.  The coroutine is then immediately
+  destroyed.  This flow bypasses the original ``await_suspend()`` (though it
+  must contain a compatibility stub), and omits suspend intrinsics.  The net
+  effect is improved code speed & size for "short-circuiting" coroutines.
+
 Improvements to Clang's diagnostics
 -----------------------------------
 - Added a separate diagnostic group ``-Wfunction-effect-redeclarations``, for the more pedantic

clang/include/clang/AST/ExprCXX.h

@@ -5266,6 +5266,7 @@ class CoroutineSuspendExpr : public Expr {
       : Expr(SC, Resume->getType(), Resume->getValueKind(),
              Resume->getObjectKind()),
         KeywordLoc(KeywordLoc), OpaqueValue(OpaqueValue) {
+    CoroutineSuspendExprBits.UseAwaitSuspendDestroy = false;
     SubExprs[SubExpr::Operand] = Operand;
     SubExprs[SubExpr::Common] = Common;
     SubExprs[SubExpr::Ready] = Ready;
@@ -5279,6 +5280,7 @@ class CoroutineSuspendExpr : public Expr {
       : Expr(SC, Ty, VK_PRValue, OK_Ordinary), KeywordLoc(KeywordLoc) {
     assert(Common->isTypeDependent() && Ty->isDependentType() &&
            "wrong constructor for non-dependent co_await/co_yield expression");
+    CoroutineSuspendExprBits.UseAwaitSuspendDestroy = false;
     SubExprs[SubExpr::Operand] = Operand;
     SubExprs[SubExpr::Common] = Common;
     SubExprs[SubExpr::Ready] = nullptr;
@@ -5288,13 +5290,22 @@ class CoroutineSuspendExpr : public Expr {
   }
 
   CoroutineSuspendExpr(StmtClass SC, EmptyShell Empty) : Expr(SC, Empty) {
+    CoroutineSuspendExprBits.UseAwaitSuspendDestroy = false;
     SubExprs[SubExpr::Operand] = nullptr;
     SubExprs[SubExpr::Common] = nullptr;
     SubExprs[SubExpr::Ready] = nullptr;
     SubExprs[SubExpr::Suspend] = nullptr;
     SubExprs[SubExpr::Resume] = nullptr;
   }
 
+  bool useAwaitSuspendDestroy() const {
+    return CoroutineSuspendExprBits.UseAwaitSuspendDestroy;
+  }
+
+  void setUseAwaitSuspendDestroy(bool Use) {
+    CoroutineSuspendExprBits.UseAwaitSuspendDestroy = Use;
+  }
+
   Expr *getCommonExpr() const {
     return static_cast<Expr*>(SubExprs[SubExpr::Common]);
   }

clang/include/clang/AST/Stmt.h

@@ -1258,12 +1258,23 @@ class alignas(void *) Stmt {
 
   //===--- C++ Coroutines bitfields classes ---===//
 
-  class CoawaitExprBitfields {
-    friend class CoawaitExpr;
+  class CoroutineSuspendExprBitfields {
+    friend class CoroutineSuspendExpr;
 
     LLVM_PREFERRED_TYPE(ExprBitfields)
     unsigned : NumExprBits;
 
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned UseAwaitSuspendDestroy : 1;
+  };
+  enum { NumCoroutineSuspendExprBits = NumExprBits + 1 };
+
+  class CoawaitExprBitfields {
+    friend class CoawaitExpr;
+
+    LLVM_PREFERRED_TYPE(CoroutineSuspendExprBitfields)
+    unsigned : NumCoroutineSuspendExprBits;
+
     LLVM_PREFERRED_TYPE(bool)
     unsigned IsImplicit : 1;
   };
@@ -1388,6 +1399,7 @@ class alignas(void *) Stmt {
     PackIndexingExprBitfields PackIndexingExprBits;
 
     // C++ Coroutines expressions
+    CoroutineSuspendExprBitfields CoroutineSuspendExprBits;
     CoawaitExprBitfields CoawaitBits;
 
     // Obj-C Expressions

clang/include/clang/Basic/Attr.td

@@ -1352,6 +1352,14 @@ def CoroAwaitElidableArgument : InheritableAttr {
   let SimpleHandler = 1;
 }
 
+def CoroAwaitSuspendDestroy: InheritableAttr {
+  let Spellings = [Clang<"coro_await_suspend_destroy">];
+  let Subjects = SubjectList<[CXXMethod]>;
+  let LangOpts = [CPlusPlus];
+  let Documentation = [CoroAwaitSuspendDestroyDoc];
+  let SimpleHandler = 1;
+}
+
 // OSObject-based attributes.
 def OSConsumed : InheritableParamAttr {
   let Spellings = [Clang<"os_consumed">];

clang/include/clang/Basic/AttrDocs.td

@@ -9363,6 +9363,119 @@ Example:
 }];
 }
 
+def CoroAwaitSuspendDestroyDoc : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+
+The ``[[clang::coro_await_suspend_destroy]]`` attribute applies to an
+``await_suspend(std::coroutine_handle<Promise>)`` member function of a
+coroutine awaiter.  When applied, suspensions into the awaiter use an optimized
+call path that bypasses standard suspend intrinsics, and immediately destroys
+the suspending coro.
+
+Each annotated ``await_suspend`` member must contain a compatibility stub:
+
+.. code-block:: c++
+
+  [[clang::coro_await_suspend_destroy]]
+  void await_suspend(std::coroutine_handle<Promise> handle) {
+    await_suspend_destroy(handle.promise());
+    handle.destroy();
+  }
+
+An awaiter type may provide both annotated and non-annotated overloads of
+``await_suspend()``, as long as each invocation of an annotated overload has a
+corresponding ``await_suspend_destroy(Promise&)`` overload.
+
+Instead of calling the annotated ``await_suspend()``, the coroutine calls
+``await_suspend_destroy(Promise&)`` and immediately destroys the coroutine
+``await_suspend_destroy()`` must return ``void`` (Note: if desired, it
+would be straightforward to also support the "symmetric transfer"
+``std::coroutine_handle`` return type.)
+
+This optimization improves code speed and size for "short-circuiting"
+coroutines — those that use coroutine syntax **exclusively** for early returns
+and control flow rather than true asynchronous operations.
+
+Specifically, a short-circuiting awaiter is one that either proceeds
+immediately (``await_ready()`` returns ``true``, skipping to
+``await_resume()``) or terminates the coroutine execution.
+
+Then, a short-circuiting coroutine is one where **all** the awaiters (including
+``co_await``, ``co_yield``, initial, and final suspend) are short-circuiting.
+
+The short-circuiting coroutine concept introduced above has close analogs in
+other languages:
+
+- Rust has ``Result<T>`` and a ``?`` operator to unpack it, while
+  ``folly::result<T>`` is a C++ short-circuiting coroutine, within which
+  ``co_await or_unwind(someResult())`` acts just like ``someResult()?``.
+
+- Haskell has ``Maybe`` & ``Error`` monads.  A short-circuiting ``co_await``
+  loosely corresponds to the monadic ``>>=``, whereas a short-circuiting
+  ``std::optional`` coro would be an exact analog of ``Maybe``.
+
+Returning to C++, even non-short-circuiting coroutines, including asynchronous
+ones that suspend, may contain short-circuiting awaiters, and those might still
+see some performance benefit if annotated.
+
+Marking your ``await_suspend_destroy`` as ``noexcept`` can sometimes further
+improve optimization.
+
+However, if **all** awaiters within a coroutine are short-circuiting, then the
+coro frame **can reliably be allocated on-stack**, making short-circuiting
+coros behave qualitatively more like plain functions -- with better
+optimization & more predictable behavior under memory pressure.
+
+Technical aside: Heap elision becomes reliable because LLVM is allowed to elide
+heap allocations whenever it can prove that the handle doesn't "escape" from
+the coroutine.  User code can only access the handle via suspend intrinsics,
+and annotated short-circuiting awaiters simply don't use any.
+
+Note that a short-circuiting coroutine differs in one important way from a
+function that replaced each ``co_await awaiter`` with explicit control flow:
+
+.. code-block:: c++
+
+  T value;
+  if (awaiter.await_ready()) {
+    value = awaiter.await_resume();
+  } else {
+    // ... content of `await_suspend_destroy` ...
+    return /* early-termination return object */;
+  }
+
+That key difference is that ``unhandled_exception()`` lets the promise type
+wrap the function body in an implicit try-catch.  This automatic exception
+boundary behavior can be desirable in robust, return-value-oriented programs
+that benefit from short-circuiting coroutines.  If not, the promise can
+re-throw.
+
+Here is an example of a short-circuiting awaiter for a hypothetical
+``std::optional`` coroutine:
+
+.. code-block:: c++
+
+  template <typename T>
+  struct optional_awaiter {
+    std::optional<T> opt_;
+    bool await_ready() const noexcept { return opt_.has_value(); }
+    T await_resume() { return std::move(opt_).value(); }
+    void await_suspend_destroy(auto& promise) {
+      // The return object of `promise`'s coro should default to "empty".
+      assert(!promise.returned_optional_ptr_->has_value());
+    }
+    [[clang::coro_await_suspend_destroy]]
+    void await_suspend(auto handle) {
+      // Fallback for when `coro_await_suspend_destroy` is unavailable.
+      await_suspend_destroy(handle.promise());
+      handle.destroy();
+    }
+  };
+
+}];
+}
+
 def CountedByDocs : Documentation {
   let Category = DocCatField;
   let Content = [{

clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -12513,6 +12513,12 @@ def note_coroutine_promise_call_implicitly_required : Note<
 def err_await_suspend_invalid_return_type : Error<
   "return type of 'await_suspend' is required to be 'void' or 'bool' (have %0)"
 >;
+def err_await_suspend_destroy_invalid_return_type : Error<
+  "return type of 'await_suspend_destroy' is required to be 'void' (have %0)"
+>;
+def err_await_suspend_suspend_destroy_return_type_mismatch : Error<
+  "return type of 'await_suspend' (%1) must match return type of 'await_suspend_destroy' (%0)"
+>;
 def note_await_ready_no_bool_conversion : Note<
   "return type of 'await_ready' is required to be contextually convertible to 'bool'"
 >;