[libc++] Add move constructor & assignment to exception_ptr (llvm#164281)

This commit adds move constructor, move assignment and `swap`
to `exception_ptr`. Adding those operators allows us to avoid
unnecessary calls to `__cxa_{inc,dec}rement_refcount`.

Performance results (from libc++'s CI):

```
Benchmark                               Baseline    Candidate    Difference    % Difference
------------------------------------  ----------  -----------  ------------  --------------
bm_exception_ptr_copy_assign_nonnull        9.77         9.94          0.18           1.79%
bm_exception_ptr_copy_assign_null          10.29        10.65          0.35           3.42%
bm_exception_ptr_copy_ctor_nonnull          7.02         7.01         -0.01          -0.13%
bm_exception_ptr_copy_ctor_null            10.54        10.60          0.06           0.56%
bm_exception_ptr_move_assign_nonnull       16.92        13.76         -3.16         -18.70%
bm_exception_ptr_move_assign_null          10.61        10.76          0.14           1.36%
bm_exception_ptr_move_ctor_nonnull         13.31        10.25         -3.06         -23.02%
bm_exception_ptr_move_ctor_null            10.28         7.30         -2.98         -28.95%
bm_exception_ptr_swap_nonnull              19.22         0.63        -18.59         -96.74%
bm_exception_ptr_swap_null                 20.02         7.79        -12.23         -61.07%
```

As expected, the `bm_exception_ptr_copy_*` benchmarks are not influenced by
this change. `bm_exception_ptr_move_*` benefits between 18% and 30%. The
`bm_exception_ptr_swap_*` tests show the biggest improvements since multiple
calls to the copy constructor are replaced by a simple pointer swap.

While `bm_exception_ptr_move_assign_null` did not show a regression in the CI
measurements, local measurements showed a regression from 3.98 to 4.71, i.e. by
18%. This is due to the additional `__tmp` inside `operator=`. The destructor
of `__other` is a no-op after the move because `__other.__ptr` will be a
nullptr. However, the compiler does not realize this, since the destructor is
not inlined and is lacking a fast-path. As such, the swap-based implementation
leads to an additional destructor call. `bm_exception_ptr_move_assign_nonnull`
still benefits because the swap-based move constructor avoids unnecessary
__cxa_{in,de}crement_refcount calls. As soon as we inline the destructor, this
regression should disappear again.

Works towards llvm#44892

Author: vogelsgesang

libcxx/include/__exception/exception_ptr.h

@@ -16,13 +16,18 @@
 #include <__memory/construct_at.h>
 #include <__type_traits/decay.h>
 #include <__type_traits/is_pointer.h>
+#include <__utility/move.h>
+#include <__utility/swap.h>
 #include <cstdlib>
 #include <typeinfo>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 #ifndef _LIBCPP_ABI_MICROSOFT
 
 #  if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
@@ -57,6 +62,8 @@ _LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
 
 #ifndef _LIBCPP_ABI_MICROSOFT
 
+inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT;
+
 class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   void* __ptr_;
 
@@ -75,7 +82,15 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   _LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
 
   exception_ptr(const exception_ptr&) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI exception_ptr(exception_ptr&& __other) _NOEXCEPT : __ptr_(__other.__ptr_) {
+    __other.__ptr_ = nullptr;
+  }
   exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI exception_ptr& operator=(exception_ptr&& __other) _NOEXCEPT {
+    exception_ptr __tmp(std::move(__other));
+    std::swap(__tmp, *this);
+    return *this;
+  }
   ~exception_ptr() _NOEXCEPT;
 
   _LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __ptr_ != nullptr; }
@@ -88,10 +103,16 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
     return !(__x == __y);
   }
 
+  friend _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT;
+
   friend _LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
   friend _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
 };
 
+inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT {
+  std::swap(__x.__ptr_, __y.__ptr_);
+}
+
 #  if _LIBCPP_HAS_EXCEPTIONS
 #    if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
 template <class _Ep>
@@ -201,4 +222,6 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
 #endif // _LIBCPP_ABI_MICROSOFT
 _LIBCPP_END_UNVERSIONED_NAMESPACE_STD
 
+_LIBCPP_POP_MACROS
+
 #endif // _LIBCPP___EXCEPTION_EXCEPTION_PTR_H

libcxx/modules/std/exception.inc

@@ -18,6 +18,7 @@ export namespace std {
   using std::rethrow_exception;
   using std::rethrow_if_nested;
   using std::set_terminate;
+  using std::swap;
   using std::terminate;
   using std::terminate_handler;
   using std::throw_with_nested;

libcxx/test/std/language.support/support.exception/propagation/exception_ptr.pass.cpp

@@ -14,7 +14,6 @@
 
 #include <exception>
 #include <cassert>
-#include <type_traits>
 
 #include "test_macros.h"
 

libcxx/test/std/language.support/support.exception/propagation/exception_ptr_move_assignment.pass.cpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: no-exceptions, c++03
+
+// <exception>
+
+// typedef unspecified exception_ptr;
+
+// Test the move assignment of exception_ptr
+
+#include <exception>
+#include <utility>
+#include <cassert>
+
+#include "test_macros.h"
+
+int main(int, char**) {
+  std::exception_ptr p = std::make_exception_ptr(42);
+  std::exception_ptr p2{p};
+  assert(p2 == p);
+  // Under test: the move assignment
+  std::exception_ptr p3;
+  p3 = std::move(p2);
+  assert(p3 == p);
+// `p2` was moved from. In libc++ it will be nullptr, but
+// this is not guaranteed by the standard.
+#if defined(_LIBCPP_VERSION) && !defined(_LIBCPP_ABI_MICROSOFT)
+  assert(p2 == nullptr);
+  assert(p2 == nullptr);
+#endif
+
+  try {
+    std::rethrow_exception(p3);
+  } catch (int e) {
+    assert(e == 42);
+  }
+
+  return 0;
+}

libcxx/test/std/language.support/support.exception/propagation/exception_ptr_move_ctr.pass.cpp

@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: no-exceptions, c++03
+
+// <exception>
+
+// typedef unspecified exception_ptr;
+
+// Test the move constructor of exception_ptr
+
+#include <exception>
+#include <utility>
+#include <cassert>
+
+#include "test_macros.h"
+
+int main(int, char**) {
+  std::exception_ptr p = std::make_exception_ptr(42);
+  std::exception_ptr p2{p};
+  assert(p2 == p);
+  // Under test: The move constructor
+  std::exception_ptr p3{std::move(p2)};
+  assert(p3 == p);
+// `p2` was moved from. In libc++ it will be nullptr, but
+// this is not guaranteed by the standard.
+#if defined(_LIBCPP_VERSION) && !defined(_LIBCPP_ABI_MICROSOFT)
+  assert(p2 == nullptr);
+#endif
+
+  try {
+    std::rethrow_exception(p3);
+  } catch (int e) {
+    assert(e == 42);
+  }
+
+  return 0;
+}

libcxx/test/std/language.support/support.exception/propagation/exception_ptr_swap.pass.cpp

@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: no-exceptions
+
+// <exception>
+
+// typedef unspecified exception_ptr;
+
+// Test swapping of exception_ptr
+
+#include <exception>
+#include <utility>
+#include <cassert>
+
+#include "test_macros.h"
+
+int main(int, char**) {
+  std::exception_ptr p21 = std::make_exception_ptr(42);
+  std::exception_ptr p42 = std::make_exception_ptr(21);
+  std::swap(p42, p21);
+
+  try {
+    std::rethrow_exception(p21);
+  } catch (int e) {
+    assert(e == 21);
+  }
+  try {
+    std::rethrow_exception(p42);
+  } catch (int e) {
+    assert(e == 42);
+  }
+
+  return 0;
+}