diff --git a/libcxx/docs/ABIGuarantees.rst b/libcxx/docs/ABIGuarantees.rst index 4d4674c7756a4..7588cf7e55453 100644 --- a/libcxx/docs/ABIGuarantees.rst +++ b/libcxx/docs/ABIGuarantees.rst @@ -114,6 +114,10 @@ hand, backwards compatibility is generally guaranteed. There are multiple ABI flags that change the symbols exported from the built library: +``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN`` +------------------------------------------------- +This removes ``align()``. In the past ``align()`` is not an inline function, but now it changed to an inline function for performance. + ``_LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON`` ------------------------------------------------- This removes ``__basic_string_common::__throw_length_error()`` and diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst index 6ea323ced3003..ece3881334b11 100644 --- a/libcxx/docs/ReleaseNotes/22.rst +++ b/libcxx/docs/ReleaseNotes/22.rst @@ -83,6 +83,8 @@ Improvements and New Features iterators, resulting in a performance improvement for ``std::deque`` and ``std::join_view>>`` iterators. +- The performance of ``align`` has been improved about 2x by making it an inline function. + - The ``num_get::do_get`` integral overloads have been optimized, resulting in a performance improvement of up to 2.8x. Deprecations and Removals @@ -116,6 +118,9 @@ ABI Affecting Changes potentially inheriting from the types they wrap. At this point in time we are not aware of any ABI changes caused by this. +- ``std::align`` is now implemented as an inline function and its definition is removed from the + libc++ built library in ABI v2, or when the ``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN`` ABI configuration is enabled. + - ``ranges::iota_view`` is now aware of ``__int128``. This causes ``iota_view::difference_type`` to change from ``long long`` to ``__int128`` in some cases. diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index 38b85c6ac70de..f88de6442f90d 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -63,6 +63,7 @@ // These flags are documented in ABIGuarantees.rst # define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT +# define _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN # define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON # define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON # define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10 diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h index 402eac3380925..47a7a2d274627 100644 --- a/libcxx/include/__memory/align.h +++ b/libcxx/include/__memory/align.h @@ -11,6 +11,7 @@ #include <__config> #include <__cstddef/size_t.h> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -18,7 +19,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD -_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); +inline namespace __align_inline { +_LIBCPP_HIDE_FROM_ABI inline void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) { + void* __r = nullptr; + if (__sz <= __space) { + char* __p1 = static_cast(__ptr); + char* __p2 = reinterpret_cast(reinterpret_cast(__p1 + (__align - 1)) & -__align); + size_t __d = static_cast(__p2 - __p1); + if (__d <= __space - __sz) { + __r = __p2; + __ptr = __r; + __space -= __d; + } + } + return __r; +} + +} // namespace __align_inline _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp index 9be40cb9c1285..9b218a3dd2213 100644 --- a/libcxx/src/memory.cpp +++ b/libcxx/src/memory.cpp @@ -132,19 +132,12 @@ __sp_mut& __get_sp_mut(const void* p) { #endif // _LIBCPP_HAS_THREADS -void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { - void* r = nullptr; - if (size <= space) { - char* p1 = static_cast(ptr); - char* p2 = reinterpret_cast(reinterpret_cast(p1 + (alignment - 1)) & -alignment); - size_t d = static_cast(p2 - p1); - if (d <= space - size) { - r = p2; - ptr = r; - space -= d; - } - } - return r; +#if !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN) + +_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) { + return __align_inline::align(alignment, size, ptr, space); } +#endif // _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN + _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/test/benchmarks/memory/align.bench.cpp b/libcxx/test/benchmarks/memory/align.bench.cpp new file mode 100644 index 0000000000000..9fb637c21f9ac --- /dev/null +++ b/libcxx/test/benchmarks/memory/align.bench.cpp @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03 + +#include +#include + +#include "benchmark/benchmark.h" + +struct Input { + std::size_t align; + std::size_t size; + void* ptr; + std::size_t buffer_size; +}; + +static void BM_align(benchmark::State& state) { + char buffer[1024]; + Input input{}; + void* ptr = buffer + 123; + std::size_t buffer_size = sizeof(buffer) - 123; + input.align = state.range(); + input.size = state.range(); + for (auto _ : state) { + input.ptr = ptr; + input.buffer_size = buffer_size; + benchmark::DoNotOptimize(input); + benchmark::DoNotOptimize(std::align(input.align, input.size, input.ptr, input.buffer_size)); + } +} +BENCHMARK(BM_align)->Range(1, 256); + +BENCHMARK_MAIN();