[libc++] Inline fast path forexception_ptr copy constructor & destructor

libcxx/docs/ReleaseNotes/22.rst

@@ -82,6 +82,9 @@ Improvements and New Features
   iterators, resulting in a performance improvement for ``std::deque<short>`` and
   ``std::join_view<vector<vector<short>>>`` iterators.
 
+- ``std::exception_ptr`` was optimized, allowing the compiler to generate better code especially for empty
+  ``std::exception_ptr`` values.
+
 Deprecations and Removals
 -------------------------
 

libcxx/include/__exception/exception_ptr.h

@@ -30,6 +30,28 @@ _LIBCPP_PUSH_MACROS
 
 #ifndef _LIBCPP_ABI_MICROSOFT
 
+// Previously, parts of exception_ptr were defined out-of-line, which prevented
+// useful compiler optimizations. Changing the out-of-line definitions to inline
+// definitions is an ABI break, however. To prevent this, we have to make sure
+// the symbols remain available in the libc++ library, in addition to being
+// defined inline here in this header.
+// To this end, we use _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE macro:
+// The macro is defined as empty for src/exception.cpp, forcing the definitions of
+// the functions to be emitted and included in the library. When users of libc++
+// compile their code, the __gnu_inline__ attribute will suppress generation of
+// these functions while making their definitions available for inlining.
+#  ifdef _LIBCPP_EMIT_CODE_FOR_EXCEPTION_PTR
+#    define _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE _LIBCPP_EXPORTED_FROM_ABI
+#  else
+#    if !__has_cpp_attribute(__gnu__::__gnu_inline__)
+#      error "GNU inline attribute is not supported"
+#    endif
+#    define _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE [[__gnu__::__gnu_inline__]] inline
+#  endif
+
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wgnu-inline-cpp-without-extern")
+
 #  if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
 
 namespace __cxxabiv1 {
@@ -67,6 +89,17 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _
 class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   void* __ptr_;
 
+  static void __do_increment_refcount(void* __ptr) _NOEXCEPT;
+  static void __do_decrement_refcount(void* __ptr) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI static void __increment_refcount(void* __ptr) _NOEXCEPT {
+    if (__ptr)
+      __do_increment_refcount(__ptr);
+  }
+  _LIBCPP_HIDE_FROM_ABI static void __decrement_refcount(void* __ptr) _NOEXCEPT {
+    if (__ptr)
+      __do_decrement_refcount(__ptr);
+  }
+
   static exception_ptr __from_native_exception_pointer(void*) _NOEXCEPT;
 
   template <class _Ep>
@@ -81,17 +114,18 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   _LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {}
   _LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
 
-  exception_ptr(const exception_ptr&) _NOEXCEPT;
+  _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr(const exception_ptr&) _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI exception_ptr(exception_ptr&& __other) _NOEXCEPT : __ptr_(__other.__ptr_) {
     __other.__ptr_ = nullptr;
   }
-  exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
+  _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI exception_ptr& operator=(exception_ptr&& __other) _NOEXCEPT {
-    exception_ptr __tmp(std::move(__other));
-    std::swap(__tmp, *this);
+    __decrement_refcount(__ptr_);
+    __ptr_         = __other.__ptr_;
+    __other.__ptr_ = nullptr;
     return *this;
   }
-  ~exception_ptr() _NOEXCEPT;
+  _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE ~exception_ptr() _NOEXCEPT;
 
   _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __ptr_ != nullptr; }
 
@@ -109,6 +143,25 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   friend _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
 };
 
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr::exception_ptr(const exception_ptr& __other) _NOEXCEPT
+    : __ptr_(__other.__ptr_) {
+  __increment_refcount(__ptr_);
+}
+
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr& exception_ptr::operator=(const exception_ptr& __other) _NOEXCEPT {
+  if (__ptr_ != __other.__ptr_) {
+    __increment_refcount(__other.__ptr_);
+    __decrement_refcount(__ptr_);
+    __ptr_ = __other.__ptr_;
+  }
+  return *this;
+}
+
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr::~exception_ptr() _NOEXCEPT { __decrement_refcount(__ptr_); }
+
 inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT {
   std::swap(__x.__ptr_, __y.__ptr_);
 }
@@ -178,6 +231,8 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep) _NOEXCEPT {
 }
 #  endif // _LIBCPP_HAS_EXCEPTIONS
 
+_LIBCPP_DIAGNOSTIC_POP
+
 #else // _LIBCPP_ABI_MICROSOFT
 
 class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {

libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -858,6 +858,8 @@
 {'is_defined': True, 'name': '__ZNSt13bad_exceptionD0Ev', 'type': 'I'}
 {'is_defined': True, 'name': '__ZNSt13bad_exceptionD1Ev', 'type': 'I'}
 {'is_defined': True, 'name': '__ZNSt13bad_exceptionD2Ev', 'type': 'I'}
+{'is_defined': True, 'name': '__ZNSt13exception_ptr23__do_decrement_refcountEPv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt13exception_ptr23__do_increment_refcountEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}

libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -494,6 +494,8 @@
 {'is_defined': True, 'name': '_ZNSt13bad_exceptionD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13bad_exceptionD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13bad_exceptionD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_decrement_refcountEPv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_increment_refcountEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}

libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -250,6 +250,8 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_decrement_refcountEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_increment_refcountEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptrC1ERKS_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptrC2ERKS_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}

libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -250,6 +250,8 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD0Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD1Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD2Ev', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_decrement_refcountEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_increment_refcountEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptrC1ERKS_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNSt13exception_ptrC2ERKS_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}

libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -857,6 +857,8 @@
 {'is_defined': True, 'name': '__ZNSt13bad_exceptionD0Ev', 'type': 'I'}
 {'is_defined': True, 'name': '__ZNSt13bad_exceptionD1Ev', 'type': 'I'}
 {'is_defined': True, 'name': '__ZNSt13bad_exceptionD2Ev', 'type': 'I'}
+{'is_defined': True, 'name': '__ZNSt13exception_ptr23__do_decrement_refcountEPv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt13exception_ptr23__do_increment_refcountEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}

libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -494,6 +494,8 @@
 {'is_defined': True, 'name': '_ZNSt13bad_exceptionD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13bad_exceptionD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13bad_exceptionD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_decrement_refcountEPv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_increment_refcountEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}

libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -525,6 +525,8 @@
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_decrement_refcountEPv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_increment_refcountEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}

libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist

@@ -523,6 +523,8 @@
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_decrement_refcountEPv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_increment_refcountEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}

libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist

@@ -494,6 +494,8 @@
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt12experimental19bad_optional_accessD2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_decrement_refcountEPv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt13exception_ptr23__do_increment_refcountEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptr31__from_native_exception_pointerEPv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC1ERKS_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt13exception_ptrC2ERKS_', 'type': 'FUNC'}

libcxx/src/exception.cpp

@@ -8,6 +8,7 @@
 
 #define _LIBCPP_ENABLE_CXX20_REMOVED_UNCAUGHT_EXCEPTION
 #define _LIBCPP_DISABLE_DEPRECATION_WARNINGS
+#define _LIBCPP_EMIT_CODE_FOR_EXCEPTION_PTR
 
 #include <exception>
 #include <new>

libcxx/src/support/runtime/exception_pointer_cxxabi.ipp

@@ -13,19 +13,12 @@
 
 namespace std {
 
-exception_ptr::~exception_ptr() noexcept { __cxa_decrement_exception_refcount(__ptr_); }
-
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
-  __cxa_increment_exception_refcount(__ptr_);
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
+  __cxa_increment_exception_refcount(__ptr);
 }
 
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
-  if (__ptr_ != other.__ptr_) {
-    __cxa_increment_exception_refcount(other.__ptr_);
-    __cxa_decrement_exception_refcount(__ptr_);
-    __ptr_ = other.__ptr_;
-  }
-  return *this;
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
+  __cxa_decrement_exception_refcount(__ptr);
 }
 
 exception_ptr exception_ptr::__from_native_exception_pointer(void* __e) noexcept {

libcxx/src/support/runtime/exception_pointer_glibcxx.ipp

@@ -7,14 +7,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+
 // libsupc++ does not implement the dependent EH ABI and the functionality
 // it uses to implement std::exception_ptr (which it declares as an alias of
 // std::__exception_ptr::exception_ptr) is not directly exported to clients. So
 // we have little choice but to hijack std::__exception_ptr::exception_ptr's
-// (which fortunately has the same layout as our std::exception_ptr) copy
-// constructor, assignment operator and destructor (which are part of its
-// stable ABI), and its rethrow_exception(std::__exception_ptr::exception_ptr)
-// function.
+// _M_addref and _M_release and its rethrow_exception function. Fortunately,
+// glibcxx's exception_ptr has the same layout as our exception_ptr and we can
+// reinterpret_cast between the two.
 
 namespace std {
 
@@ -23,27 +23,20 @@ namespace __exception_ptr {
 struct exception_ptr {
   void* __ptr_;
 
-  explicit exception_ptr(void*) noexcept;
-  exception_ptr(const exception_ptr&) noexcept;
-  exception_ptr& operator=(const exception_ptr&) noexcept;
-  ~exception_ptr() noexcept;
+  void _M_addref() noexcept;
+  void _M_release() noexcept;
 };
 
 } // namespace __exception_ptr
 
 [[noreturn]] void rethrow_exception(__exception_ptr::exception_ptr);
 
-exception_ptr::~exception_ptr() noexcept { reinterpret_cast<__exception_ptr::exception_ptr*>(this)->~exception_ptr(); }
-
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
-  new (reinterpret_cast<void*>(this))
-      __exception_ptr::exception_ptr(reinterpret_cast<const __exception_ptr::exception_ptr&>(other));
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
+  reinterpret_cast<__exception_ptr::exception_ptr*>(this)->_M_addref();
 }
 
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
-  *reinterpret_cast<__exception_ptr::exception_ptr*>(this) =
-      reinterpret_cast<const __exception_ptr::exception_ptr&>(other);
-  return *this;
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
+  reinterpret_cast<__exception_ptr::exception_ptr*>(this)->_M_release();
 }
 
 exception_ptr exception_ptr::__from_native_exception_pointer(void* __e) noexcept {

libcxx/src/support/runtime/exception_pointer_unimplemented.ipp

@@ -11,17 +11,12 @@
 
 namespace std {
 
-exception_ptr::~exception_ptr() noexcept {
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
 #warning exception_ptr not yet implemented
   __libcpp_verbose_abort("exception_ptr not yet implemented\n");
 }
 
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
-#warning exception_ptr not yet implemented
-  __libcpp_verbose_abort("exception_ptr not yet implemented\n");
-}
-
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
 #warning exception_ptr not yet implemented
   __libcpp_verbose_abort("exception_ptr not yet implemented\n");
 }

libcxx/test/tools/clang_tidy_checks/hide_from_abi.cpp

@@ -23,7 +23,8 @@ hide_from_abi::hide_from_abi(llvm::StringRef name, clang::tidy::ClangTidyContext
 void hide_from_abi::registerMatchers(clang::ast_matchers::MatchFinder* finder) {
   using namespace clang::ast_matchers;
 
-  auto has_hide_from_abi_attr = anyOf(hasAttr(clang::attr::Visibility), hasAttr(clang::attr::AbiTag));
+  auto has_hide_from_abi_attr =
+      anyOf(hasAttr(clang::attr::Visibility), hasAttr(clang::attr::AbiTag), hasAttr(clang::attr::GNUInline));
 
   finder->addMatcher(
       functionDecl(