[libc++] Inline copy constructor & destructor for std::exception_ptr

Author: vogelsgesang

libcxx/include/__exception/exception_ptr.h

@@ -30,6 +30,28 @@ _LIBCPP_PUSH_MACROS
 
 #ifndef _LIBCPP_ABI_MICROSOFT
 
+// Previously, parts of exception_ptr were defined out-of-line, which prevented
+// useful compiler optimizations. Changing the out-of-line definitions to inline
+// definitions is an ABI break, however. To prevent this, we have to make sure
+// the symbols remain available in the libc++ library, in addition to being
+// defined inline here in this header.
+// To this end, we use _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE macro:
+// The macro is defined as empty for src/exception.cpp, forcing the definitions of
+// the functions to be emitted and included in the library. When users of libc++
+// compile their code, the __gnu_inline__ attribute will suppress generation of
+// these functions while making their definitions available for inlining.
+#ifdef _LIBCPP_EMIT_CODE_FOR_EXCEPTION_PTR
+#  define _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE _LIBCPP_EXPORTED_FROM_ABI
+#else
+#  if !__has_cpp_attribute(__gnu__::__gnu_inline__)
+#    error "GNU inline attribute is not supported"
+#  endif
+#  define _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE [[__gnu__::__gnu_inline__]] inline
+#endif
+
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wgnu-inline-cpp-without-extern")
+
 #  if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
 
 namespace __cxxabiv1 {
@@ -67,6 +89,11 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _
 class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   void* __ptr_;
 
+  static void __do_increment_refcount(void* __ptr) _NOEXCEPT;
+  static void __do_decrement_refcount(void* __ptr) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI static void __increment_refcount(void* __ptr) _NOEXCEPT { if (__ptr) __do_increment_refcount(__ptr); }
+  _LIBCPP_HIDE_FROM_ABI static void __decrement_refcount(void* __ptr) _NOEXCEPT { if (__ptr) __do_decrement_refcount(__ptr); }
+
   static exception_ptr __from_native_exception_pointer(void*) _NOEXCEPT;
 
   template <class _Ep>
@@ -81,17 +108,18 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   _LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {}
   _LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
 
-  exception_ptr(const exception_ptr&) _NOEXCEPT;
+  _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr(const exception_ptr&) _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI exception_ptr(exception_ptr&& __other) _NOEXCEPT : __ptr_(__other.__ptr_) {
     __other.__ptr_ = nullptr;
   }
-  exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
+  _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI exception_ptr& operator=(exception_ptr&& __other) _NOEXCEPT {
-    exception_ptr __tmp(std::move(__other));
-    std::swap(__tmp, *this);
+    __decrement_refcount(__ptr_);
+    __ptr_         = __other.__ptr_;
+    __other.__ptr_ = nullptr;
     return *this;
   }
-  ~exception_ptr() _NOEXCEPT;
+  _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE ~exception_ptr() _NOEXCEPT;
 
   _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __ptr_ != nullptr; }
 
@@ -109,6 +137,25 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   friend _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
 };
 
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr::exception_ptr(const exception_ptr& __other) _NOEXCEPT
+    : __ptr_(__other.__ptr_) {
+  __increment_refcount(__ptr_);
+}
+
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr& exception_ptr::operator=(const exception_ptr& __other) _NOEXCEPT {
+  if (__ptr_ != __other.__ptr_) {
+    __increment_refcount(__other.__ptr_);
+    __decrement_refcount(__ptr_);
+    __ptr_ = __other.__ptr_;
+  }
+  return *this;
+}
+
+// Must be defined outside the class definition due to _LIBCPP_EXPORTED_FROM_LIB_INLINEABLE
+_LIBCPP_EXPORTED_FROM_LIB_INLINEABLE exception_ptr::~exception_ptr() _NOEXCEPT { __decrement_refcount(__ptr_); }
+
 inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT {
   std::swap(__x.__ptr_, __y.__ptr_);
 }
@@ -222,6 +269,8 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
 #endif // _LIBCPP_ABI_MICROSOFT
 _LIBCPP_END_UNVERSIONED_NAMESPACE_STD
 
+_LIBCPP_DIAGNOSTIC_POP
+
 _LIBCPP_POP_MACROS
 
 #endif // _LIBCPP___EXCEPTION_EXCEPTION_PTR_H

libcxx/src/exception.cpp

@@ -8,6 +8,7 @@
 
 #define _LIBCPP_ENABLE_CXX20_REMOVED_UNCAUGHT_EXCEPTION
 #define _LIBCPP_DISABLE_DEPRECATION_WARNINGS
+#define _LIBCPP_EMIT_CODE_FOR_EXCEPTION_PTR
 
 #include <exception>
 #include <new>

libcxx/src/support/runtime/exception_pointer_cxxabi.ipp

@@ -13,19 +13,12 @@
 
 namespace std {
 
-exception_ptr::~exception_ptr() noexcept { __cxa_decrement_exception_refcount(__ptr_); }
-
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
-  __cxa_increment_exception_refcount(__ptr_);
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
+  __cxa_increment_exception_refcount(__ptr);
 }
 
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
-  if (__ptr_ != other.__ptr_) {
-    __cxa_increment_exception_refcount(other.__ptr_);
-    __cxa_decrement_exception_refcount(__ptr_);
-    __ptr_ = other.__ptr_;
-  }
-  return *this;
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
+  __cxa_decrement_exception_refcount(__ptr);
 }
 
 exception_ptr exception_ptr::__from_native_exception_pointer(void* __e) noexcept {

libcxx/src/support/runtime/exception_pointer_glibcxx.ipp

@@ -7,14 +7,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+
 // libsupc++ does not implement the dependent EH ABI and the functionality
 // it uses to implement std::exception_ptr (which it declares as an alias of
 // std::__exception_ptr::exception_ptr) is not directly exported to clients. So
 // we have little choice but to hijack std::__exception_ptr::exception_ptr's
-// (which fortunately has the same layout as our std::exception_ptr) copy
-// constructor, assignment operator and destructor (which are part of its
-// stable ABI), and its rethrow_exception(std::__exception_ptr::exception_ptr)
-// function.
+// _M_addref and _M_release and its rethrow_exception function. Fortunately,
+// glibcxx's exception_ptr has the same layout as our exception_ptr and we can
+// reinterpret_cast between the two.
 
 namespace std {
 
@@ -23,27 +23,20 @@ namespace __exception_ptr {
 struct exception_ptr {
   void* __ptr_;
 
-  explicit exception_ptr(void*) noexcept;
-  exception_ptr(const exception_ptr&) noexcept;
-  exception_ptr& operator=(const exception_ptr&) noexcept;
-  ~exception_ptr() noexcept;
+  void _M_addref() noexcept;
+  void _M_release() noexcept;
 };
 
 } // namespace __exception_ptr
 
 [[noreturn]] void rethrow_exception(__exception_ptr::exception_ptr);
 
-exception_ptr::~exception_ptr() noexcept { reinterpret_cast<__exception_ptr::exception_ptr*>(this)->~exception_ptr(); }
-
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
-  new (reinterpret_cast<void*>(this))
-      __exception_ptr::exception_ptr(reinterpret_cast<const __exception_ptr::exception_ptr&>(other));
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
+  reinterpret_cast<__exception_ptr::exception_ptr*>(this)->_M_addref();
 }
 
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
-  *reinterpret_cast<__exception_ptr::exception_ptr*>(this) =
-      reinterpret_cast<const __exception_ptr::exception_ptr&>(other);
-  return *this;
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
+  reinterpret_cast<__exception_ptr::exception_ptr*>(this)->_M_release();
 }
 
 exception_ptr exception_ptr::__from_native_exception_pointer(void* __e) noexcept {

libcxx/src/support/runtime/exception_pointer_unimplemented.ipp

@@ -11,17 +11,12 @@
 
 namespace std {
 
-exception_ptr::~exception_ptr() noexcept {
+void exception_ptr::__do_increment_refcount(void* __ptr) noexcept {
 #warning exception_ptr not yet implemented
   __libcpp_verbose_abort("exception_ptr not yet implemented\n");
 }
 
-exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
-#warning exception_ptr not yet implemented
-  __libcpp_verbose_abort("exception_ptr not yet implemented\n");
-}
-
-exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
+void exception_ptr::__do_decrement_refcount(void* __ptr) noexcept {
 #warning exception_ptr not yet implemented
   __libcpp_verbose_abort("exception_ptr not yet implemented\n");
 }