diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index 1b89569..f01021b 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake @@ -10,6 +10,7 @@ set(ABSL_INTERNAL_DLL_FILES "base/config.h" "base/const_init.h" "base/dynamic_annotations.h" + "base/fast_type_id.h" "base/internal/atomic_hook.h" "base/internal/cycleclock.cc" "base/internal/cycleclock.h" @@ -18,15 +19,13 @@ set(ABSL_INTERNAL_DLL_FILES "base/internal/dynamic_annotations.h" "base/internal/endian.h" "base/internal/errno_saver.h" - "base/internal/fast_type_id.h" "base/internal/hide_ptr.h" "base/internal/identity.h" - "base/internal/invoke.h" - "base/internal/inline_variable.h" + "base/internal/iterator_traits.h" "base/internal/low_level_alloc.cc" "base/internal/low_level_alloc.h" "base/internal/low_level_scheduling.h" - "base/internal/nullability_impl.h" + "base/internal/nullability_deprecated.h" "base/internal/per_thread_tls.h" "base/internal/poison.cc" "base/internal/poison.h" @@ -83,6 +82,7 @@ set(ABSL_INTERNAL_DLL_FILES "container/internal/container_memory.h" "container/internal/hash_function_defaults.h" "container/internal/hash_policy_traits.h" + "container/internal/hashtable_control_bytes.h" "container/internal/hashtable_debug.h" "container/internal/hashtable_debug_hooks.h" "container/internal/hashtablez_sampler.cc" @@ -94,6 +94,7 @@ set(ABSL_INTERNAL_DLL_FILES "container/internal/raw_hash_map.h" "container/internal/raw_hash_set.cc" "container/internal/raw_hash_set.h" + "container/internal/raw_hash_set_resize_impl.h" "container/internal/tracked.h" "container/node_hash_map.h" "container/node_hash_set.h" @@ -126,6 +127,7 @@ set(ABSL_INTERNAL_DLL_FILES "debugging/symbolize.h" "debugging/internal/address_is_readable.cc" "debugging/internal/address_is_readable.h" + "debugging/internal/addresses.h" "debugging/internal/bounded_utf8_length_sequence.h" "debugging/internal/decode_rust_punycode.cc" "debugging/internal/decode_rust_punycode.h" @@ -160,6 +162,7 @@ set(ABSL_INTERNAL_DLL_FILES "hash/internal/spy_hash_state.h" "hash/internal/low_level_hash.h" "hash/internal/low_level_hash.cc" + "hash/internal/weakly_mixed_integer.h" "log/absl_check.h" "log/absl_log.h" "log/absl_vlog_is_on.h" @@ -201,7 +204,6 @@ set(ABSL_INTERNAL_DLL_FILES "log/initialize.cc" "log/initialize.h" "log/log.h" - "log/log_entry.cc" "log/log_entry.h" "log/log_sink.cc" "log/log_sink.h" @@ -238,8 +240,8 @@ set(ABSL_INTERNAL_DLL_FILES "random/internal/nonsecure_base.h" "random/internal/pcg_engine.h" "random/internal/platform.h" - "random/internal/pool_urbg.cc" - "random/internal/pool_urbg.h" + "random/internal/entropy_pool.cc" + "random/internal/entropy_pool.h" "random/internal/randen.cc" "random/internal/randen.h" "random/internal/randen_detect.cc" @@ -286,7 +288,6 @@ set(ABSL_INTERNAL_DLL_FILES "strings/cord.h" "strings/cord_analysis.cc" "strings/cord_analysis.h" - "strings/cord_buffer.cc" "strings/cord_buffer.h" "strings/escaping.cc" "strings/escaping.h" @@ -432,20 +433,11 @@ set(ABSL_INTERNAL_DLL_FILES "time/internal/cctz/src/tzfile.h" "time/internal/cctz/src/zone_info_source.cc" "types/any.h" - "types/bad_any_cast.cc" - "types/bad_any_cast.h" - "types/bad_optional_access.cc" - "types/bad_optional_access.h" - "types/bad_variant_access.cc" - "types/bad_variant_access.h" "types/compare.h" - "types/internal/variant.h" "types/optional.h" - "types/internal/optional.h" "types/span.h" "types/internal/span.h" "types/variant.h" - "utility/internal/if_constexpr.h" "utility/utility.h" "debugging/leak_check.cc" ) @@ -496,10 +488,6 @@ set(ABSL_INTERNAL_DLL_TARGETS "any" "any_invocable" "atomic_hook" - "bad_any_cast" - "bad_any_cast_impl" - "bad_optional_access" - "bad_variant_access" "base" "base_internal" "bind_front" @@ -731,10 +719,8 @@ int main() { return 0; } if(ABSL_INTERNAL_AT_LEAST_CXX20) set(ABSL_INTERNAL_CXX_STD_FEATURE cxx_std_20) -elseif(ABSL_INTERNAL_AT_LEAST_CXX17) - set(ABSL_INTERNAL_CXX_STD_FEATURE cxx_std_17) else() - set(ABSL_INTERNAL_CXX_STD_FEATURE cxx_std_14) + set(ABSL_INTERNAL_CXX_STD_FEATURE cxx_std_17) endif() function(absl_internal_dll_contains) @@ -899,7 +885,7 @@ Cflags: -I\${includedir}${PC_CFLAGS}\n") ) if(ABSL_PROPAGATE_CXX_STD) - # Abseil libraries require C++14 as the current minimum standard. When + # Abseil libraries require C++17 as the current minimum standard. When # compiled with a higher minimum (either because it is the compiler's # default or explicitly requested), then Abseil requires that standard. target_compile_features(${_dll} PUBLIC ${ABSL_INTERNAL_CXX_STD_FEATURE}) diff --git a/CMake/AbseilHelpers.cmake b/CMake/AbseilHelpers.cmake index d8fb9fe..624a3c7 100644 --- a/CMake/AbseilHelpers.cmake +++ b/CMake/AbseilHelpers.cmake @@ -301,7 +301,7 @@ Cflags: -I\${includedir}${PC_CFLAGS}\n") endif() if(ABSL_PROPAGATE_CXX_STD) - # Abseil libraries require C++14 as the current minimum standard. When + # Abseil libraries require C++17 as the current minimum standard. When # compiled with a higher standard (either because it is the compiler's # default or explicitly requested), then Abseil requires that standard. target_compile_features(${_NAME} PUBLIC ${ABSL_INTERNAL_CXX_STD_FEATURE}) @@ -338,7 +338,7 @@ Cflags: -I\${includedir}${PC_CFLAGS}\n") target_compile_definitions(${_NAME} INTERFACE ${ABSL_CC_LIB_DEFINES}) if(ABSL_PROPAGATE_CXX_STD) - # Abseil libraries require C++14 as the current minimum standard. + # Abseil libraries require C++17 as the current minimum standard. # Top-level application CMake projects should ensure a consistent C++ # standard for all compiled sources by setting CMAKE_CXX_STANDARD. target_compile_features(${_NAME} INTERFACE ${ABSL_INTERNAL_CXX_STD_FEATURE}) @@ -450,7 +450,7 @@ function(absl_cc_test) set_property(TARGET ${_NAME} PROPERTY FOLDER ${ABSL_IDE_FOLDER}/test) if(ABSL_PROPAGATE_CXX_STD) - # Abseil libraries require C++14 as the current minimum standard. + # Abseil libraries require C++17 as the current minimum standard. # Top-level application CMake projects should ensure a consistent C++ # standard for all compiled sources by setting CMAKE_CXX_STANDARD. target_compile_features(${_NAME} PUBLIC ${ABSL_INTERNAL_CXX_STD_FEATURE}) diff --git a/CMake/README.md b/CMake/README.md index 808edfe..ecaeaf0 100644 --- a/CMake/README.md +++ b/CMake/README.md @@ -43,8 +43,8 @@ cmake_minimum_required(VERSION 3.16) project(my_app_project) # Pick the C++ standard to compile with. -# Abseil currently supports C++14, C++17, and C++20. -set(CMAKE_CXX_STANDARD 14) +# Abseil currently supports C++17 and C++20. +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) add_subdirectory(abseil-cpp) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b21ee7..8d3059d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,8 +23,8 @@ if (POLICY CMP0141) cmake_policy(SET CMP0141 NEW) endif (POLICY CMP0141) -project(absl LANGUAGES CXX VERSION 20250127) -set(ABSL_SOVERSION "2501.0.0") +project(absl LANGUAGES CXX VERSION 20250512) +set(ABSL_SOVERSION "2505.0.0") include(CTest) # Output directory is correct by default for most build setups. However, when @@ -46,7 +46,7 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) set(CMAKE_BUILD_RPATH_USE_ORIGIN ON) option(ABSL_PROPAGATE_CXX_STD - "Use CMake C++ standard meta features (e.g. cxx_std_14) that propagate to targets that link to Abseil" + "Use CMake C++ standard meta features (e.g. cxx_std_17) that propagate to targets that link to Abseil" ON) option(ABSL_USE_SYSTEM_INCLUDES diff --git a/MODULE.bazel b/MODULE.bazel index 5c8b337..48a65c7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,7 +16,7 @@ module( name = "abseil-cpp", - version = "20250127.1", + version = "20250512.1", compatibility_level = 1, ) @@ -25,16 +25,13 @@ cc_configure = use_extension("@rules_cc//cc:extensions.bzl", dev_dependency = True) use_repo(cc_configure, "local_config_cc") -# Only direct dependencies need to be listed below. -# Please keep the versions in sync with the versions in the WORKSPACE file. - -bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_cc", version = "0.1.1") bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "platforms", version = "0.0.11") bazel_dep( name = "google_benchmark", - version = "1.8.5", + version = "1.9.2", dev_dependency = True, ) @@ -42,5 +39,5 @@ bazel_dep( # intended to be used by Abseil users depend on GoogleTest. bazel_dep( name = "googletest", - version = "1.15.2", + version = "1.17.0", ) diff --git a/README.md b/README.md index f834fcd..c2c851e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Abseil - C++ Common Libraries The repository contains the Abseil C++ library code. Abseil is an open-source -collection of C++ code (compliant to C++14) designed to augment the C++ +collection of C++ code (compliant to C++17) designed to augment the C++ standard library. ## Table of Contents @@ -99,8 +99,8 @@ Abseil contains the following C++ library components:
The `memory` library contains memory management facilities that augment C++'s `` library. * [`meta`](absl/meta/) -
The `meta` library contains compatible versions of type checks - available within C++14 and C++17 versions of the C++ `` library. +
The `meta` library contains type checks + similar to those available in the C++ `` library. * [`numeric`](absl/numeric/)
The `numeric` library contains 128-bit integer types as well as implementations of C++20's bitwise math functions. @@ -108,15 +108,14 @@ Abseil contains the following C++ library components:
The `profiling` library contains utility code for profiling C++ entities. It is currently a private dependency of other Abseil libraries. * [`random`](absl/random/) -
The `random` library contains functions for generating psuedorandom +
The `random` library contains functions for generating pseudorandom values. * [`status`](absl/status/)
The `status` library contains abstractions for error handling, specifically `absl::Status` and `absl::StatusOr`. * [`strings`](absl/strings/)
The `strings` library contains a variety of strings routines and - utilities, including a C++14-compatible version of the C++17 - `std::string_view` type. + utilities. * [`synchronization`](absl/synchronization/)
The `synchronization` library contains concurrency primitives (Abseil's `absl::Mutex` class, an alternative to `std::mutex`) and a variety of @@ -126,8 +125,7 @@ Abseil contains the following C++ library components: points in time, durations of time, and formatting and parsing time within time zones. * [`types`](absl/types/) -
The `types` library contains non-container utility types, like a - C++14-compatible version of the C++17 `std::optional` type. +
The `types` library contains non-container utility types. * [`utility`](absl/utility/)
The `utility` library contains utility and helper code. diff --git a/absl/abseil.podspec.gen.py b/absl/abseil.podspec.gen.py index cbf7cb4..e1afa21 100755 --- a/absl/abseil.podspec.gen.py +++ b/absl/abseil.podspec.gen.py @@ -43,10 +43,11 @@ 'USE_HEADERMAP' => 'NO', 'ALWAYS_SEARCH_USER_PATHS' => 'NO', } - s.ios.deployment_target = '9.0' - s.osx.deployment_target = '10.11' - s.tvos.deployment_target = '9.0' - s.watchos.deployment_target = '2.0' + s.ios.deployment_target = '12.0' + s.osx.deployment_target = '10.13' + s.tvos.deployment_target = '12.0' + s.watchos.deployment_target = '4.0' + s.visionos.deployment_target = '1.0' s.subspec 'xcprivacy' do |ss| ss.resource_bundles = { ss.module_name => 'PrivacyInfo.xcprivacy', diff --git a/absl/algorithm/container.h b/absl/algorithm/container.h index 3193656..6f9c193 100644 --- a/absl/algorithm/container.h +++ b/absl/algorithm/container.h @@ -44,7 +44,6 @@ #include #include #include -#include #include #include #include @@ -76,8 +75,8 @@ using ContainerIter = decltype(begin(std::declval())); // An MSVC bug involving template parameter substitution requires us to use // decltype() here instead of just std::pair. template -using ContainerIterPairType = - decltype(std::make_pair(ContainerIter(), ContainerIter())); +using ContainerIterPairType = decltype(std::make_pair( + std::declval>(), std::declval>())); template using ContainerDifferenceType = decltype(std::distance( @@ -847,25 +846,9 @@ template OutputIterator c_sample(const C& c, OutputIterator result, Distance n, UniformRandomBitGenerator&& gen) { -#if defined(__cpp_lib_sample) && __cpp_lib_sample >= 201603L return std::sample(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), result, n, std::forward(gen)); -#else - // Fall back to a stable selection-sampling implementation. - auto first = container_algorithm_internal::c_begin(c); - Distance unsampled_elements = c_distance(c); - n = (std::min)(n, unsampled_elements); - for (; n != 0; ++first) { - Distance r = - std::uniform_int_distribution(0, --unsampled_elements)(gen); - if (r < n) { - *result++ = *first; - --n; - } - } - return result; -#endif } //------------------------------------------------------------------------------ diff --git a/absl/base/attributes.h b/absl/base/attributes.h index 95b102e..d009f6d 100644 --- a/absl/base/attributes.h +++ b/absl/base/attributes.h @@ -339,9 +339,9 @@ #ifndef ABSL_ATTRIBUTE_SECTION_VARIABLE #ifdef _AIX // __attribute__((section(#name))) on AIX is achieved by using the `.csect` -// psudo op which includes an additional integer as part of its syntax indcating -// alignment. If data fall under different alignments then you might get a -// compilation error indicating a `Section type conflict`. +// pseudo op which includes an additional integer as part of its syntax +// indicating alignment. If data fall under different alignments then you might +// get a compilation error indicating a `Section type conflict`. #define ABSL_ATTRIBUTE_SECTION_VARIABLE(name) #else #define ABSL_ATTRIBUTE_SECTION_VARIABLE(name) __attribute__((section(#name))) @@ -553,12 +553,11 @@ // // Prevents the compiler from complaining about variables that appear unused. // -// For code or headers that are assured to only build with C++17 and up, prefer -// just using the standard '[[maybe_unused]]' directly over this macro. +// Deprecated: Use the standard C++17 `[[maybe_unused]` instead. // // Due to differences in positioning requirements between the old, compiler -// specific __attribute__ syntax and the now standard [[maybe_unused]], this -// macro does not attempt to take advantage of '[[maybe_unused]]'. +// specific __attribute__ syntax and the now standard `[[maybe_unused]]`, this +// macro does not attempt to take advantage of `[[maybe_unused]]`. #if ABSL_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) #undef ABSL_ATTRIBUTE_UNUSED #define ABSL_ATTRIBUTE_UNUSED __attribute__((__unused__)) @@ -759,6 +758,76 @@ #define ABSL_CONST_INIT #endif +// ABSL_REQUIRE_EXPLICIT_INIT +// +// ABSL_REQUIRE_EXPLICIT_INIT is placed *after* the data members of an aggregate +// type to indicate that the annotated member must be explicitly initialized by +// the user whenever the aggregate is constructed. For example: +// +// struct Coord { +// int x ABSL_REQUIRE_EXPLICIT_INIT; +// int y ABSL_REQUIRE_EXPLICIT_INIT; +// }; +// Coord coord = {1}; // warning: field 'y' is not explicitly initialized +// +// Note that usage on C arrays is not supported in C++. +// Use a struct (such as std::array) to wrap the array member instead. +// +// Avoid applying this attribute to the members of non-aggregate types. +// The behavior within non-aggregates is unspecified and subject to change. +// +// Do NOT attempt to suppress or demote the error generated by this attribute. +// Just like with a missing function argument, it is a hard error by design. +// +// See the upstream documentation for more details: +// https://clang.llvm.org/docs/AttributeReference.html#require-explicit-initialization +#ifdef __cplusplus +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::require_explicit_initialization) +// clang-format off +#define ABSL_REQUIRE_EXPLICIT_INIT \ + [[clang::require_explicit_initialization]] = \ + AbslInternal_YouForgotToExplicitlyInitializeAField::v +#else +#define ABSL_REQUIRE_EXPLICIT_INIT \ + = AbslInternal_YouForgotToExplicitlyInitializeAField::v +#endif +// clang-format on +#else +// clang-format off +#if ABSL_HAVE_ATTRIBUTE(require_explicit_initialization) +#define ABSL_REQUIRE_EXPLICIT_INIT \ + __attribute__((require_explicit_initialization)) +#else +#define ABSL_REQUIRE_EXPLICIT_INIT \ + /* No portable fallback for C is available */ +#endif +// clang-format on +#endif + +#ifdef __cplusplus +struct AbslInternal_YouForgotToExplicitlyInitializeAField { + // A portable version of [[clang::require_explicit_initialization]] that + // never builds, as a last resort for all toolchains. + // The error messages are poor, so we don't rely on this unless we have to. + template +#if !defined(SWIG) + constexpr +#endif + operator T() const /* NOLINT */ { + const void *volatile deliberately_volatile_ptr = nullptr; + // Infinite loop to prevent constexpr compilation + for (;;) { + // This assignment ensures the 'this' pointer is not optimized away, so + // that linking always fails. + deliberately_volatile_ptr = this; // Deliberately not constexpr + (void)deliberately_volatile_ptr; + } + } + // This is deliberately left undefined to prevent linking + static AbslInternal_YouForgotToExplicitlyInitializeAField v; +}; +#endif + // ABSL_ATTRIBUTE_PURE_FUNCTION // // ABSL_ATTRIBUTE_PURE_FUNCTION is used to annotate declarations of "pure" diff --git a/absl/base/call_once.h b/absl/base/call_once.h index b666d36..7bfd916 100644 --- a/absl/base/call_once.h +++ b/absl/base/call_once.h @@ -28,12 +28,12 @@ #include #include #include +#include #include #include #include "absl/base/attributes.h" #include "absl/base/config.h" -#include "absl/base/internal/invoke.h" #include "absl/base/internal/low_level_scheduling.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/scheduling_mode.h" @@ -49,8 +49,8 @@ ABSL_NAMESPACE_BEGIN class once_flag; namespace base_internal { -absl::Nonnull*> ControlWord( - absl::Nonnull flag); +std::atomic* absl_nonnull ControlWord( + absl::once_flag* absl_nonnull flag); } // namespace base_internal // call_once() @@ -93,8 +93,8 @@ class once_flag { once_flag& operator=(const once_flag&) = delete; private: - friend absl::Nonnull*> base_internal::ControlWord( - absl::Nonnull flag); + friend std::atomic* absl_nonnull base_internal::ControlWord( + once_flag* absl_nonnull flag); std::atomic control_; }; @@ -108,7 +108,7 @@ namespace base_internal { // Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to // initialize entities used by the scheduler implementation. template -void LowLevelCallOnce(absl::Nonnull flag, Callable&& fn, +void LowLevelCallOnce(absl::once_flag* absl_nonnull flag, Callable&& fn, Args&&... args); // Disables scheduling while on stack when scheduling mode is non-cooperative. @@ -150,7 +150,7 @@ enum { template void - CallOnceImpl(absl::Nonnull*> control, + CallOnceImpl(std::atomic* absl_nonnull control, base_internal::SchedulingMode scheduling_mode, Callable&& fn, Args&&... args) { #ifndef NDEBUG @@ -181,8 +181,7 @@ template std::memory_order_relaxed) || base_internal::SpinLockWait(control, ABSL_ARRAYSIZE(trans), trans, scheduling_mode) == kOnceInit) { - base_internal::invoke(std::forward(fn), - std::forward(args)...); + std::invoke(std::forward(fn), std::forward(args)...); old_control = control->exchange(base_internal::kOnceDone, std::memory_order_release); if (old_control == base_internal::kOnceWaiter) { @@ -191,13 +190,13 @@ template } // else *control is already kOnceDone } -inline absl::Nonnull*> ControlWord( - absl::Nonnull flag) { +inline std::atomic* absl_nonnull ControlWord( + once_flag* absl_nonnull flag) { return &flag->control_; } template -void LowLevelCallOnce(absl::Nonnull flag, Callable&& fn, +void LowLevelCallOnce(absl::once_flag* absl_nonnull flag, Callable&& fn, Args&&... args) { std::atomic* once = base_internal::ControlWord(flag); uint32_t s = once->load(std::memory_order_acquire); diff --git a/absl/base/config.h b/absl/base/config.h index 63b9642..1a9bc59 100644 --- a/absl/base/config.h +++ b/absl/base/config.h @@ -117,7 +117,7 @@ // // LTS releases can be obtained from // https://github.com/abseil/abseil-cpp/releases. -#define ABSL_LTS_RELEASE_VERSION 20250127 +#define ABSL_LTS_RELEASE_VERSION 20250512 #define ABSL_LTS_RELEASE_PATCH_LEVEL 1 // Helper macro to convert a CPP variable to a string literal. @@ -274,15 +274,12 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #define ABSL_HAVE_STD_IS_TRIVIALLY_COPYABLE 1 #endif - // ABSL_HAVE_THREAD_LOCAL // -// DEPRECATED - `thread_local` is available on all supported platforms. -// Checks whether C++11's `thread_local` storage duration specifier is -// supported. +// Checks whether the `thread_local` storage duration specifier is supported. #ifdef ABSL_HAVE_THREAD_LOCAL #error ABSL_HAVE_THREAD_LOCAL cannot be directly set -#else +#elif !defined(__XTENSA__) #define ABSL_HAVE_THREAD_LOCAL 1 #endif @@ -469,6 +466,9 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // // Checks the endianness of the platform. // +// Prefer using `std::endian` in C++20, or `absl::endian` from +// absl/numeric/bits.h prior to C++20. +// // Notes: uses the built in endian macros provided by GCC (since 4.6) and // Clang (since 3.2); see // https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html. @@ -520,54 +520,22 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #define ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE 0 #endif -// ABSL_HAVE_STD_ANY -// -// Checks whether C++17 std::any is available. -#ifdef ABSL_HAVE_STD_ANY -#error "ABSL_HAVE_STD_ANY cannot be directly set." -#elif defined(__cpp_lib_any) && __cpp_lib_any >= 201606L -#define ABSL_HAVE_STD_ANY 1 -#elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ - ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L && \ - !ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE +// Deprecated macros for polyfill detection. #define ABSL_HAVE_STD_ANY 1 -#endif - -// ABSL_HAVE_STD_OPTIONAL -// -// Checks whether C++17 std::optional is available. -#ifdef ABSL_HAVE_STD_OPTIONAL -#error "ABSL_HAVE_STD_OPTIONAL cannot be directly set." -#elif defined(__cpp_lib_optional) && __cpp_lib_optional >= 202106L -#define ABSL_HAVE_STD_OPTIONAL 1 -#elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ - ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L && \ - !ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE +#define ABSL_USES_STD_ANY 1 #define ABSL_HAVE_STD_OPTIONAL 1 -#endif - -// ABSL_HAVE_STD_VARIANT -// -// Checks whether C++17 std::variant is available. -#ifdef ABSL_HAVE_STD_VARIANT -#error "ABSL_HAVE_STD_VARIANT cannot be directly set." -#elif defined(__cpp_lib_variant) && __cpp_lib_variant >= 201606L -#define ABSL_HAVE_STD_VARIANT 1 -#elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ - ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L && \ - !ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE +#define ABSL_USES_STD_OPTIONAL 1 #define ABSL_HAVE_STD_VARIANT 1 -#endif +#define ABSL_USES_STD_VARIANT 1 // ABSL_HAVE_STD_STRING_VIEW // -// Checks whether C++17 std::string_view is available. +// Deprecated: always defined to 1. +// std::string_view was added in C++17, which means all versions of C++ +// supported by Abseil have it. #ifdef ABSL_HAVE_STD_STRING_VIEW #error "ABSL_HAVE_STD_STRING_VIEW cannot be directly set." -#elif defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L -#define ABSL_HAVE_STD_STRING_VIEW 1 -#elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ - ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L +#else #define ABSL_HAVE_STD_STRING_VIEW 1 #endif @@ -587,63 +555,15 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #define ABSL_HAVE_STD_ORDERING 1 #endif -// ABSL_USES_STD_ANY -// -// Indicates whether absl::any is an alias for std::any. -#if !defined(ABSL_OPTION_USE_STD_ANY) -#error options.h is misconfigured. -#elif ABSL_OPTION_USE_STD_ANY == 0 || \ - (ABSL_OPTION_USE_STD_ANY == 2 && !defined(ABSL_HAVE_STD_ANY)) -#undef ABSL_USES_STD_ANY -#elif ABSL_OPTION_USE_STD_ANY == 1 || \ - (ABSL_OPTION_USE_STD_ANY == 2 && defined(ABSL_HAVE_STD_ANY)) -#define ABSL_USES_STD_ANY 1 -#else -#error options.h is misconfigured. -#endif - -// ABSL_USES_STD_OPTIONAL -// -// Indicates whether absl::optional is an alias for std::optional. -#if !defined(ABSL_OPTION_USE_STD_OPTIONAL) -#error options.h is misconfigured. -#elif ABSL_OPTION_USE_STD_OPTIONAL == 0 || \ - (ABSL_OPTION_USE_STD_OPTIONAL == 2 && !defined(ABSL_HAVE_STD_OPTIONAL)) -#undef ABSL_USES_STD_OPTIONAL -#elif ABSL_OPTION_USE_STD_OPTIONAL == 1 || \ - (ABSL_OPTION_USE_STD_OPTIONAL == 2 && defined(ABSL_HAVE_STD_OPTIONAL)) -#define ABSL_USES_STD_OPTIONAL 1 -#else -#error options.h is misconfigured. -#endif - -// ABSL_USES_STD_VARIANT -// -// Indicates whether absl::variant is an alias for std::variant. -#if !defined(ABSL_OPTION_USE_STD_VARIANT) -#error options.h is misconfigured. -#elif ABSL_OPTION_USE_STD_VARIANT == 0 || \ - (ABSL_OPTION_USE_STD_VARIANT == 2 && !defined(ABSL_HAVE_STD_VARIANT)) -#undef ABSL_USES_STD_VARIANT -#elif ABSL_OPTION_USE_STD_VARIANT == 1 || \ - (ABSL_OPTION_USE_STD_VARIANT == 2 && defined(ABSL_HAVE_STD_VARIANT)) -#define ABSL_USES_STD_VARIANT 1 -#else -#error options.h is misconfigured. -#endif - // ABSL_USES_STD_STRING_VIEW // // Indicates whether absl::string_view is an alias for std::string_view. #if !defined(ABSL_OPTION_USE_STD_STRING_VIEW) #error options.h is misconfigured. -#elif ABSL_OPTION_USE_STD_STRING_VIEW == 0 || \ - (ABSL_OPTION_USE_STD_STRING_VIEW == 2 && \ - !defined(ABSL_HAVE_STD_STRING_VIEW)) +#elif ABSL_OPTION_USE_STD_STRING_VIEW == 0 #undef ABSL_USES_STD_STRING_VIEW #elif ABSL_OPTION_USE_STD_STRING_VIEW == 1 || \ - (ABSL_OPTION_USE_STD_STRING_VIEW == 2 && \ - defined(ABSL_HAVE_STD_STRING_VIEW)) + ABSL_OPTION_USE_STD_STRING_VIEW == 2 #define ABSL_USES_STD_STRING_VIEW 1 #else #error options.h is misconfigured. @@ -665,14 +585,6 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #error options.h is misconfigured. #endif -// In debug mode, MSVC 2017's std::variant throws a EXCEPTION_ACCESS_VIOLATION -// SEH exception from emplace for variant when constructing the -// struct can throw. This defeats some of variant_test and -// variant_exception_safety_test. -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_DEBUG) -#define ABSL_INTERNAL_MSVC_2017_DBG_MODE -#endif - // ABSL_INTERNAL_MANGLED_NS // ABSL_INTERNAL_MANGLED_BACKREFERENCE // @@ -813,36 +725,15 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION // -// Class template argument deduction is a language feature added in C++17. +// Deprecated: always defined to 1. +// Class template argument deduction is a language feature added in C++17, +// which means all versions of C++ supported by Abseil have it. #ifdef ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION #error "ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION cannot be directly set." -#elif defined(__cpp_deduction_guides) +#else #define ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION 1 #endif -// ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -// -// Prior to C++17, static constexpr variables defined in classes required a -// separate definition outside of the class body, for example: -// -// class Foo { -// static constexpr int kBar = 0; -// }; -// constexpr int Foo::kBar; -// -// In C++17, these variables defined in classes are considered inline variables, -// and the extra declaration is redundant. Since some compilers warn on the -// extra declarations, ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL can be used -// conditionally ignore them: -// -// #ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -// constexpr int Foo::kBar; -// #endif -#if defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ - ABSL_INTERNAL_CPLUSPLUS_LANG < 201703L -#define ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL 1 -#endif - // `ABSL_INTERNAL_HAS_RTTI` determines whether abseil is being compiled with // RTTI support. #ifdef ABSL_INTERNAL_HAS_RTTI diff --git a/absl/base/fast_type_id.h b/absl/base/fast_type_id.h new file mode 100644 index 0000000..ff25027 --- /dev/null +++ b/absl/base/fast_type_id.h @@ -0,0 +1,45 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_FAST_TYPE_ID_H_ +#define ABSL_BASE_FAST_TYPE_ID_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace base_internal { +template +struct FastTypeTag { + static constexpr char kDummyVar = 0; +}; +} // namespace base_internal + +// The type returned by `absl::FastTypeId()`. +using FastTypeIdType = const void*; + +// `absl::FastTypeId()` evaluates at compile-time to a unique id for the +// passed-in type. These are meant to be good match for keys into maps or +// straight up comparisons. +template +constexpr FastTypeIdType FastTypeId() { + return &base_internal::FastTypeTag::kDummyVar; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_FAST_TYPE_ID_H_ diff --git a/absl/base/internal/cycleclock.cc b/absl/base/internal/cycleclock.cc index 902e3f5..9946601 100644 --- a/absl/base/internal/cycleclock.cc +++ b/absl/base/internal/cycleclock.cc @@ -35,11 +35,6 @@ namespace base_internal { #if ABSL_USE_UNSCALED_CYCLECLOCK -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr int32_t CycleClock::kShift; -constexpr double CycleClock::kFrequencyScale; -#endif - ABSL_CONST_INIT std::atomic CycleClock::cycle_clock_source_{nullptr}; diff --git a/absl/base/internal/cycleclock_config.h b/absl/base/internal/cycleclock_config.h index 191112b..50a4697 100644 --- a/absl/base/internal/cycleclock_config.h +++ b/absl/base/internal/cycleclock_config.h @@ -18,7 +18,6 @@ #include #include "absl/base/config.h" -#include "absl/base/internal/inline_variable.h" #include "absl/base/internal/unscaledcycleclock_config.h" namespace absl { @@ -31,22 +30,23 @@ namespace base_internal { // Not debug mode and the UnscaledCycleClock frequency is the CPU // frequency. Scale the CycleClock to prevent overflow if someone // tries to represent the time as cycles since the Unix epoch. -ABSL_INTERNAL_INLINE_CONSTEXPR(int32_t, kCycleClockShift, 1); +inline constexpr int32_t kCycleClockShift = 1; #else // Not debug mode and the UnscaledCycleClock isn't operating at the // raw CPU frequency. There is no need to do any scaling, so don't // needlessly sacrifice precision. -ABSL_INTERNAL_INLINE_CONSTEXPR(int32_t, kCycleClockShift, 0); +inline constexpr int32_t kCycleClockShift = 0; #endif #else // NDEBUG // In debug mode use a different shift to discourage depending on a // particular shift value. -ABSL_INTERNAL_INLINE_CONSTEXPR(int32_t, kCycleClockShift, 2); +inline constexpr int32_t kCycleClockShift = 2; #endif // NDEBUG -ABSL_INTERNAL_INLINE_CONSTEXPR(double, kCycleClockFrequencyScale, - 1.0 / (1 << kCycleClockShift)); -#endif // ABSL_USE_UNSCALED_CYCLECLOC +inline constexpr double kCycleClockFrequencyScale = + 1.0 / (1 << kCycleClockShift); + +#endif // ABSL_USE_UNSCALED_CYCLECLOCK } // namespace base_internal ABSL_NAMESPACE_END diff --git a/absl/base/internal/endian.h b/absl/base/internal/endian.h index 943f3d9..fb38f60 100644 --- a/absl/base/internal/endian.h +++ b/absl/base/internal/endian.h @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // +// This file is for Abseil internal use only. +// See //absl/numeric/bits.h for supported functions related to endian-ness. #ifndef ABSL_BASE_INTERNAL_ENDIAN_H_ #define ABSL_BASE_INTERNAL_ENDIAN_H_ @@ -28,44 +30,38 @@ namespace absl { ABSL_NAMESPACE_BEGIN -inline uint64_t gbswap_64(uint64_t host_int) { +constexpr uint64_t gbswap_64(uint64_t x) { #if ABSL_HAVE_BUILTIN(__builtin_bswap64) || defined(__GNUC__) - return __builtin_bswap64(host_int); -#elif defined(_MSC_VER) - return _byteswap_uint64(host_int); + return __builtin_bswap64(x); #else - return (((host_int & uint64_t{0xFF}) << 56) | - ((host_int & uint64_t{0xFF00}) << 40) | - ((host_int & uint64_t{0xFF0000}) << 24) | - ((host_int & uint64_t{0xFF000000}) << 8) | - ((host_int & uint64_t{0xFF00000000}) >> 8) | - ((host_int & uint64_t{0xFF0000000000}) >> 24) | - ((host_int & uint64_t{0xFF000000000000}) >> 40) | - ((host_int & uint64_t{0xFF00000000000000}) >> 56)); + return (((x & uint64_t{0xFF}) << 56) | + ((x & uint64_t{0xFF00}) << 40) | + ((x & uint64_t{0xFF0000}) << 24) | + ((x & uint64_t{0xFF000000}) << 8) | + ((x & uint64_t{0xFF00000000}) >> 8) | + ((x & uint64_t{0xFF0000000000}) >> 24) | + ((x & uint64_t{0xFF000000000000}) >> 40) | + ((x & uint64_t{0xFF00000000000000}) >> 56)); #endif } -inline uint32_t gbswap_32(uint32_t host_int) { +constexpr uint32_t gbswap_32(uint32_t x) { #if ABSL_HAVE_BUILTIN(__builtin_bswap32) || defined(__GNUC__) - return __builtin_bswap32(host_int); -#elif defined(_MSC_VER) - return _byteswap_ulong(host_int); + return __builtin_bswap32(x); #else - return (((host_int & uint32_t{0xFF}) << 24) | - ((host_int & uint32_t{0xFF00}) << 8) | - ((host_int & uint32_t{0xFF0000}) >> 8) | - ((host_int & uint32_t{0xFF000000}) >> 24)); + return (((x & uint32_t{0xFF}) << 24) | + ((x & uint32_t{0xFF00}) << 8) | + ((x & uint32_t{0xFF0000}) >> 8) | + ((x & uint32_t{0xFF000000}) >> 24)); #endif } -inline uint16_t gbswap_16(uint16_t host_int) { +constexpr uint16_t gbswap_16(uint16_t x) { #if ABSL_HAVE_BUILTIN(__builtin_bswap16) || defined(__GNUC__) - return __builtin_bswap16(host_int); -#elif defined(_MSC_VER) - return _byteswap_ushort(host_int); + return __builtin_bswap16(x); #else - return (((host_int & uint16_t{0xFF}) << 8) | - ((host_int & uint16_t{0xFF00}) >> 8)); + return (((x & uint16_t{0xFF}) << 8) | + ((x & uint16_t{0xFF00}) >> 8)); #endif } @@ -161,27 +157,27 @@ inline int64_t ToHost(int64_t x) { } // Functions to do unaligned loads and stores in little-endian order. -inline uint16_t Load16(absl::Nonnull p) { +inline uint16_t Load16(const void* absl_nonnull p) { return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); } -inline void Store16(absl::Nonnull p, uint16_t v) { +inline void Store16(void* absl_nonnull p, uint16_t v) { ABSL_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); } -inline uint32_t Load32(absl::Nonnull p) { +inline uint32_t Load32(const void* absl_nonnull p) { return ToHost32(ABSL_INTERNAL_UNALIGNED_LOAD32(p)); } -inline void Store32(absl::Nonnull p, uint32_t v) { +inline void Store32(void* absl_nonnull p, uint32_t v) { ABSL_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); } -inline uint64_t Load64(absl::Nonnull p) { +inline uint64_t Load64(const void* absl_nonnull p) { return ToHost64(ABSL_INTERNAL_UNALIGNED_LOAD64(p)); } -inline void Store64(absl::Nonnull p, uint64_t v) { +inline void Store64(void* absl_nonnull p, uint64_t v) { ABSL_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); } @@ -251,27 +247,27 @@ inline int64_t ToHost(int64_t x) { } // Functions to do unaligned loads and stores in big-endian order. -inline uint16_t Load16(absl::Nonnull p) { +inline uint16_t Load16(const void* absl_nonnull p) { return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); } -inline void Store16(absl::Nonnull p, uint16_t v) { +inline void Store16(void* absl_nonnull p, uint16_t v) { ABSL_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); } -inline uint32_t Load32(absl::Nonnull p) { +inline uint32_t Load32(const void* absl_nonnull p) { return ToHost32(ABSL_INTERNAL_UNALIGNED_LOAD32(p)); } -inline void Store32(absl::Nonnullp, uint32_t v) { +inline void Store32(void* absl_nonnull p, uint32_t v) { ABSL_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); } -inline uint64_t Load64(absl::Nonnull p) { +inline uint64_t Load64(const void* absl_nonnull p) { return ToHost64(ABSL_INTERNAL_UNALIGNED_LOAD64(p)); } -inline void Store64(absl::Nonnull p, uint64_t v) { +inline void Store64(void* absl_nonnull p, uint64_t v) { ABSL_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); } diff --git a/absl/base/internal/iterator_traits.h b/absl/base/internal/iterator_traits.h new file mode 100644 index 0000000..472c436 --- /dev/null +++ b/absl/base/internal/iterator_traits.h @@ -0,0 +1,71 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: internal/iterator_traits.h +// ----------------------------------------------------------------------------- +// +// Helpers for querying traits of iterators, for implementing containers, etc. + +#ifndef ABSL_BASE_INTERNAL_ITERATOR_TRAITS_H_ +#define ABSL_BASE_INTERNAL_ITERATOR_TRAITS_H_ + +#include +#include + +#include "absl/base/config.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +template +struct IteratorCategory {}; + +template +struct IteratorCategory< + Iterator, + absl::void_t::iterator_category>> { + using type = typename std::iterator_traits::iterator_category; +}; + +template +struct IteratorConceptImpl : IteratorCategory {}; + +template +struct IteratorConceptImpl< + Iterator, + absl::void_t::iterator_concept>> { + using type = typename std::iterator_traits::iterator_concept; +}; + +// The newer `std::iterator_traits::iterator_concept` if available, +// else `std::iterator_traits::iterator_category`. +template +using IteratorConcept = typename IteratorConceptImpl::type; + +template +using IsAtLeastIterator = + std::is_convertible, IteratorTag>; + +template +using IsAtLeastForwardIterator = + IsAtLeastIterator; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_ITERATOR_TRAITS_H_ diff --git a/absl/base/internal/iterator_traits_test_helper.h b/absl/base/internal/iterator_traits_test_helper.h new file mode 100644 index 0000000..707612d --- /dev/null +++ b/absl/base/internal/iterator_traits_test_helper.h @@ -0,0 +1,97 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_ITERATOR_TRAITS_TEST_HELPER_H_ +#define ABSL_BASE_INTERNAL_ITERATOR_TRAITS_TEST_HELPER_H_ + +#include +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// This would be a forward_iterator in C++20, but it's only an input iterator +// before that, since it has a non-reference `reference`. +template +class Cpp20ForwardZipIterator { + using IteratorReference = typename std::iterator_traits::reference; + + public: + Cpp20ForwardZipIterator() = default; + explicit Cpp20ForwardZipIterator(Iterator left, Iterator right) + : left_(left), right_(right) {} + + Cpp20ForwardZipIterator& operator++() { + ++left_; + ++right_; + return *this; + } + + Cpp20ForwardZipIterator operator++(int) { + Cpp20ForwardZipIterator tmp(*this); + ++*this; + return *this; + } + + std::pair operator*() const { + return {*left_, *right_}; + } + + // C++17 input iterators require `operator->`, but this isn't possible to + // implement. C++20 dropped the requirement. + + friend bool operator==(const Cpp20ForwardZipIterator& lhs, + const Cpp20ForwardZipIterator& rhs) { + return lhs.left_ == rhs.left_ && lhs.right_ == rhs.right_; + } + + friend bool operator!=(const Cpp20ForwardZipIterator& lhs, + const Cpp20ForwardZipIterator& rhs) { + return !(lhs == rhs); + } + + private: + Iterator left_{}; + Iterator right_{}; +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +template +struct std::iterator_traits< + absl::base_internal::Cpp20ForwardZipIterator> { + private: + using IteratorReference = typename std::iterator_traits::reference; + + public: + using iterator_category = std::input_iterator_tag; + using iterator_concept = std::forward_iterator_tag; + using value_type = std::pair; + using difference_type = + typename std::iterator_traits::difference_type; + using reference = value_type; + using pointer = void; +}; + +#if defined(__cpp_lib_concepts) +static_assert( + std::forward_iterator>); +#endif // defined(__cpp_lib_concepts) + +#endif // ABSL_BASE_INTERNAL_ITERATOR_TRAITS_TEST_HELPER_H_ diff --git a/absl/base/internal/low_level_alloc.cc b/absl/base/internal/low_level_alloc.cc index a563f7b..158b609 100644 --- a/absl/base/internal/low_level_alloc.cc +++ b/absl/base/internal/low_level_alloc.cc @@ -330,7 +330,7 @@ size_t GetPageSize() { GetSystemInfo(&system_info); return std::max(system_info.dwPageSize, system_info.dwAllocationGranularity); #elif defined(__wasm__) || defined(__asmjs__) || defined(__hexagon__) - return getpagesize(); + return static_cast(getpagesize()); #else return static_cast(sysconf(_SC_PAGESIZE)); #endif @@ -448,8 +448,8 @@ static inline uintptr_t RoundUp(uintptr_t addr, uintptr_t align) { // that the freelist is in the correct order, that it // consists of regions marked "unallocated", and that no two regions // are adjacent in memory (they should have been coalesced). -// L >= arena->mu -static AllocList *Next(int i, AllocList *prev, LowLevelAlloc::Arena *arena) { +static AllocList *Next(int i, AllocList *prev, LowLevelAlloc::Arena *arena) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena->mu) { ABSL_RAW_CHECK(i < prev->levels, "too few levels in Next()"); AllocList *next = prev->next[i]; if (next != nullptr) { @@ -473,6 +473,7 @@ static void Coalesce(AllocList *a) { if (n != nullptr && reinterpret_cast(a) + a->header.size == reinterpret_cast(n)) { LowLevelAlloc::Arena *arena = a->header.arena; + arena->mu.AssertHeld(); a->header.size += n->header.size; n->header.magic = 0; n->header.arena = nullptr; @@ -486,8 +487,8 @@ static void Coalesce(AllocList *a) { } // Adds block at location "v" to the free list -// L >= arena->mu -static void AddToFreelist(void *v, LowLevelAlloc::Arena *arena) { +static void AddToFreelist(void *v, LowLevelAlloc::Arena *arena) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena->mu) { AllocList *f = reinterpret_cast(reinterpret_cast(v) - sizeof(f->header)); ABSL_RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header), diff --git a/absl/base/internal/nullability_deprecated.h b/absl/base/internal/nullability_deprecated.h new file mode 100644 index 0000000..1174a96 --- /dev/null +++ b/absl/base/internal/nullability_deprecated.h @@ -0,0 +1,106 @@ +// Copyright 2023 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef ABSL_BASE_INTERNAL_NULLABILITY_DEPRECATED_H_ +#define ABSL_BASE_INTERNAL_NULLABILITY_DEPRECATED_H_ + +#include "absl/base/attributes.h" +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace nullability_internal { + +template +using NullableImpl +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::annotate) + [[clang::annotate("Nullable")]] +#endif +// Don't add the _Nullable attribute in Objective-C compiles. Many Objective-C +// projects enable the `-Wnullable-to-nonnull-conversion warning`, which is +// liable to produce false positives. +#if ABSL_HAVE_FEATURE(nullability_on_classes) && !defined(__OBJC__) + = T _Nullable; +#else + = T; +#endif + +template +using NonnullImpl +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::annotate) + [[clang::annotate("Nonnull")]] +#endif +#if ABSL_HAVE_FEATURE(nullability_on_classes) && !defined(__OBJC__) + = T _Nonnull; +#else + = T; +#endif + +template +using NullabilityUnknownImpl +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::annotate) + [[clang::annotate("Nullability_Unspecified")]] +#endif +#if ABSL_HAVE_FEATURE(nullability_on_classes) && !defined(__OBJC__) + = T _Null_unspecified; +#else + = T; +#endif + +} // namespace nullability_internal + +// The following template aliases are deprecated forms of nullability +// annotations. They have some limitations, for example, an incompatibility with +// `auto*` pointers, as `auto` cannot be used in a template argument. +// +// It is important to note that these annotations are not distinct strong +// *types*. They are alias templates defined to be equal to the underlying +// pointer type. A pointer annotated `Nonnull`, for example, is simply a +// pointer of type `T*`. +// +// Prefer the macro style annotations in `absl/base/nullability.h` instead. + +// absl::Nonnull, analogous to absl_nonnull +// +// Example: +// absl::Nonnull foo; +// Is equivalent to: +// int* absl_nonnull foo; +template +using Nonnull [[deprecated("Use `absl_nonnull`.")]] = + nullability_internal::NonnullImpl; + +// absl::Nullable, analogous to absl_nullable +// +// Example: +// absl::Nullable foo; +// Is equivalent to: +// int* absl_nullable foo; +template +using Nullable [[deprecated("Use `absl_nullable`.")]] = + nullability_internal::NullableImpl; + +// absl::NullabilityUnknown, analogous to absl_nullability_unknown +// +// Example: +// absl::NullabilityUnknown foo; +// Is equivalent to: +// int* absl_nullability_unknown foo; +template +using NullabilityUnknown [[deprecated("Use `absl_nullability_unknown`.")]] = + nullability_internal::NullabilityUnknownImpl; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_NULLABILITY_DEPRECATED_H_ diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index 381b913..430f775 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc @@ -67,15 +67,6 @@ void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock, submit_profile_data.Store(fn); } -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -// Static member variable definitions. -constexpr uint32_t SpinLock::kSpinLockHeld; -constexpr uint32_t SpinLock::kSpinLockCooperative; -constexpr uint32_t SpinLock::kSpinLockDisabledScheduling; -constexpr uint32_t SpinLock::kSpinLockSleeper; -constexpr uint32_t SpinLock::kWaitTimeMask; -#endif - // Uncommon constructors. SpinLock::SpinLock(base_internal::SchedulingMode mode) : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) { diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index 1bb260f..2a10896 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h @@ -89,8 +89,7 @@ class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED SpinLock { // acquisition was successful. If the lock was not acquired, false is // returned. If this SpinLock is free at the time of the call, TryLock // will return true with high probability. - ABSL_MUST_USE_RESULT inline bool TryLock() - ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { + [[nodiscard]] inline bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock); bool res = TryLockImpl(); ABSL_TSAN_MUTEX_POST_LOCK( @@ -121,7 +120,7 @@ class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED SpinLock { // Determine if the lock is held. When the lock is held by the invoking // thread, true will always be returned. Intended to be used as // CHECK(lock.IsHeld()). - ABSL_MUST_USE_RESULT inline bool IsHeld() const { + [[nodiscard]] inline bool IsHeld() const { return (lockword_.load(std::memory_order_relaxed) & kSpinLockHeld) != 0; } @@ -203,16 +202,7 @@ class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED SpinLock { // Corresponding locker object that arranges to acquire a spinlock for // the duration of a C++ scope. -// -// TODO(b/176172494): Use only [[nodiscard]] when baseline is raised. -// TODO(b/6695610): Remove forward declaration when #ifdef is no longer needed. -#if ABSL_HAVE_CPP_ATTRIBUTE(nodiscard) -class [[nodiscard]] SpinLockHolder; -#else -class ABSL_MUST_USE_RESULT ABSL_ATTRIBUTE_TRIVIAL_ABI SpinLockHolder; -#endif - -class ABSL_SCOPED_LOCKABLE SpinLockHolder { +class ABSL_SCOPED_LOCKABLE [[nodiscard]] SpinLockHolder { public: inline explicit SpinLockHolder(SpinLock* l) ABSL_EXCLUSIVE_LOCK_FUNCTION(l) : lock_(l) { diff --git a/absl/base/internal/unaligned_access.h b/absl/base/internal/unaligned_access.h index 4fea457..3f5dd6f 100644 --- a/absl/base/internal/unaligned_access.h +++ b/absl/base/internal/unaligned_access.h @@ -36,33 +36,33 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { -inline uint16_t UnalignedLoad16(absl::Nonnull p) { +inline uint16_t UnalignedLoad16(const void* absl_nonnull p) { uint16_t t; memcpy(&t, p, sizeof t); return t; } -inline uint32_t UnalignedLoad32(absl::Nonnull p) { +inline uint32_t UnalignedLoad32(const void* absl_nonnull p) { uint32_t t; memcpy(&t, p, sizeof t); return t; } -inline uint64_t UnalignedLoad64(absl::Nonnull p) { +inline uint64_t UnalignedLoad64(const void* absl_nonnull p) { uint64_t t; memcpy(&t, p, sizeof t); return t; } -inline void UnalignedStore16(absl::Nonnull p, uint16_t v) { +inline void UnalignedStore16(void* absl_nonnull p, uint16_t v) { memcpy(p, &v, sizeof v); } -inline void UnalignedStore32(absl::Nonnull p, uint32_t v) { +inline void UnalignedStore32(void* absl_nonnull p, uint32_t v) { memcpy(p, &v, sizeof v); } -inline void UnalignedStore64(absl::Nonnull p, uint64_t v) { +inline void UnalignedStore64(void* absl_nonnull p, uint64_t v) { memcpy(p, &v, sizeof v); } diff --git a/absl/base/internal/unscaledcycleclock.h b/absl/base/internal/unscaledcycleclock.h index 965c42d..bfd9887 100644 --- a/absl/base/internal/unscaledcycleclock.h +++ b/absl/base/internal/unscaledcycleclock.h @@ -88,9 +88,14 @@ inline int64_t UnscaledCycleClock::Now() { #elif defined(__aarch64__) // System timer of ARMv8 runs at a different frequency than the CPU's. -// The frequency is fixed, typically in the range 1-50MHz. It can be -// read at CNTFRQ special register. We assume the OS has set up -// the virtual timer properly. +// +// Frequency is fixed. From Armv8.6-A and Armv9.1-A on, the frequency is 1GHz. +// Pre-Armv8.6-A, the frequency was a system design choice, typically in the +// range of 1MHz to 50MHz. See also: +// https://developer.arm.com/documentation/102379/0101/What-is-the-Generic-Timer- +// +// It can be read at CNTFRQ special register. We assume the OS has set up the +// virtual timer properly. inline int64_t UnscaledCycleClock::Now() { int64_t virtual_timer_value; asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); diff --git a/absl/base/no_destructor.h b/absl/base/no_destructor.h index 43b3540..9d960ee 100644 --- a/absl/base/no_destructor.h +++ b/absl/base/no_destructor.h @@ -135,11 +135,11 @@ class NoDestructor { // Pretend to be a smart pointer to T with deep constness. // Never returns a null pointer. T& operator*() { return *get(); } - absl::Nonnull operator->() { return get(); } - absl::Nonnull get() { return impl_.get(); } + T* absl_nonnull operator->() { return get(); } + T* absl_nonnull get() { return impl_.get(); } const T& operator*() const { return *get(); } - absl::Nonnull operator->() const { return get(); } - absl::Nonnull get() const { return impl_.get(); } + const T* absl_nonnull operator->() const { return get(); } + const T* absl_nonnull get() const { return impl_.get(); } private: class DirectImpl { @@ -147,8 +147,8 @@ class NoDestructor { template explicit constexpr DirectImpl(Args&&... args) : value_(std::forward(args)...) {} - absl::Nonnull get() const { return &value_; } - absl::Nonnull get() { return &value_; } + const T* absl_nonnull get() const { return &value_; } + T* absl_nonnull get() { return &value_; } private: T value_; @@ -160,33 +160,14 @@ class NoDestructor { explicit PlacementImpl(Args&&... args) { new (&space_) T(std::forward(args)...); } - absl::Nonnull get() const { - return Launder(reinterpret_cast(&space_)); + const T* absl_nonnull get() const { + return std::launder(reinterpret_cast(&space_)); } - absl::Nonnull get() { return Launder(reinterpret_cast(&space_)); } - - private: - template - static absl::Nonnull Launder(absl::Nonnull p) { -#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606L - return std::launder(p); -#elif ABSL_HAVE_BUILTIN(__builtin_launder) - return __builtin_launder(p); -#else - // When `std::launder` or equivalent are not available, we rely on - // undefined behavior, which works as intended on Abseil's officially - // supported platforms as of Q3 2023. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif - return p; -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif -#endif + T* absl_nonnull get() { + return std::launder(reinterpret_cast(&space_)); } + private: alignas(T) unsigned char space_[sizeof(T)]; }; @@ -199,12 +180,10 @@ class NoDestructor { impl_; }; -#ifdef ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION // Provide 'Class Template Argument Deduction': the type of NoDestructor's T // will be the same type as the argument passed to NoDestructor's constructor. template NoDestructor(T) -> NoDestructor; -#endif // ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/nullability.h b/absl/base/nullability.h index 241c65a..3a5d6e8 100644 --- a/absl/base/nullability.h +++ b/absl/base/nullability.h @@ -16,21 +16,21 @@ // File: nullability.h // ----------------------------------------------------------------------------- // -// This header file defines a set of "templated annotations" for designating the -// expected nullability of pointers. These annotations allow you to designate -// pointers in one of three classification states: +// This header file defines a set of annotations for designating the expected +// nullability of pointers. These annotations allow you to designate pointers in +// one of three classification states: // -// * "Non-null" (for pointers annotated `Nonnull`), indicating that it is +// * "Non-null" (for pointers annotated `absl_nonnull`), indicating that it is // invalid for the given pointer to ever be null. -// * "Nullable" (for pointers annotated `Nullable`), indicating that it is +// * "Nullable" (for pointers annotated `absl_nullable`), indicating that it is // valid for the given pointer to be null. -// * "Unknown" (for pointers annotated `NullabilityUnknown`), indicating -// that the given pointer has not been yet classified as either nullable or +// * "Unknown" (for pointers annotated `absl_nullability_unknown`), indicating +// that the given pointer has not yet been classified as either nullable or // non-null. This is the default state of unannotated pointers. // -// NOTE: unannotated pointers implicitly bear the annotation -// `NullabilityUnknown`; you should rarely, if ever, see this annotation used -// in the codebase explicitly. +// NOTE: Unannotated pointers implicitly bear the annotation +// `absl_nullability_unknown`; you should rarely, if ever, see this annotation +// used in the codebase explicitly. // // ----------------------------------------------------------------------------- // Nullability and Contracts @@ -64,16 +64,49 @@ // formalize those contracts within the codebase. // // ----------------------------------------------------------------------------- +// Annotation Syntax +// ----------------------------------------------------------------------------- +// +// The annotations should be positioned as a qualifier for the pointer type. For +// example, the position of `const` when declaring a const pointer (not a +// pointer to a const type) is the position you should also use for these +// annotations. +// +// Example: +// +// // A const non-null pointer to an `Employee`. +// Employee* absl_nonnull const e; +// +// // A non-null pointer to a const `Employee`. +// const Employee* absl_nonnull e; +// +// // A non-null pointer to a const nullable pointer to an `Employee`. +// Employee* absl_nullable const* absl_nonnull e = nullptr; +// +// // A non-null function pointer. +// void (*absl_nonnull func)(int, double); +// +// // A non-null array of `Employee`s as a parameter. +// void func(Employee employees[absl_nonnull]); +// +// // A non-null std::unique_ptr to an `Employee`. +// // As with `const`, it is possible to place the annotation on either side of +// // a named type not ending in `*`, but placing it before the type it +// // describes is preferred, unless inconsistent with surrounding code. +// absl_nonnull std::unique_ptr employee; +// +// // Invalid annotation usage – this attempts to declare a pointer to a +// // nullable `Employee`, which is meaningless. +// absl_nullable Employee* e; +// +// ----------------------------------------------------------------------------- // Using Nullability Annotations // ----------------------------------------------------------------------------- // -// It is important to note that these annotations are not distinct strong -// *types*. They are alias templates defined to be equal to the underlying -// pointer type. A pointer annotated `Nonnull`, for example, is simply a -// pointer of type `T*`. Each annotation acts as a form of documentation about -// the contract for the given pointer. Each annotation requires providers or -// consumers of these pointers across API boundaries to take appropriate steps -// when setting or using these pointers: +// Each annotation acts as a form of documentation about the contract for the +// given pointer. Each annotation requires providers or consumers of these +// pointers across API boundaries to take appropriate steps when setting or +// using these pointers: // // * "Non-null" pointers should never be null. It is the responsibility of the // provider of this pointer to ensure that the pointer may never be set to @@ -91,20 +124,20 @@ // Example: // // // PaySalary() requires the passed pointer to an `Employee` to be non-null. -// void PaySalary(absl::Nonnull e) { +// void PaySalary(Employee* absl_nonnull e) { // pay(e->salary); // OK to dereference // } // // // CompleteTransaction() guarantees the returned pointer to an `Account` to // // be non-null. -// absl::Nonnull balance CompleteTransaction(double fee) { +// Account* absl_nonnull balance CompleteTransaction(double fee) { // ... // } // // // Note that specifying a nullability annotation does not prevent someone // // from violating the contract: // -// Nullable find(Map& employees, std::string_view name); +// Employee* absl_nullable find(Map& employees, std::string_view name); // // void g(Map& employees) { // Employee *e = find(employees, "Pat"); @@ -144,14 +177,14 @@ // These nullability annotations are primarily a human readable signal about the // intended contract of the pointer. They are not *types* and do not currently // provide any correctness guarantees. For example, a pointer annotated as -// `Nonnull` is *not guaranteed* to be non-null, and the compiler won't -// alert or prevent assignment of a `Nullable` to a `Nonnull`. +// `absl_nonnull` is *not guaranteed* to be non-null, and the compiler won't +// alert or prevent assignment of a `T* absl_nullable` to a `T* absl_nonnull`. // =========================================================================== #ifndef ABSL_BASE_NULLABILITY_H_ #define ABSL_BASE_NULLABILITY_H_ #include "absl/base/config.h" -#include "absl/base/internal/nullability_impl.h" +#include "absl/base/internal/nullability_deprecated.h" // ABSL_POINTERS_DEFAULT_NONNULL // @@ -168,14 +201,14 @@ // ABSL_POINTERS_DEFAULT_NONNULL // // void FillMessage(Message *m); // implicitly non-null -// absl::Nullable GetNullablePtr(); // explicitly nullable -// absl::NullabilityUnknown GetUnknownPtr(); // explicitly unknown +// T* absl_nullable GetNullablePtr(); // explicitly nullable +// T* absl_nullability_unknown GetUnknownPtr(); // explicitly unknown // -// The macro can be safely used in header files -- it will not affect any files +// The macro can be safely used in header files – it will not affect any files // that include it. // -// In files with the macro, plain `T*` syntax means `absl::Nonnull`, and the -// exceptions (`Nullable` and `NullabilityUnknown`) must be marked +// In files with the macro, plain `T*` syntax means `T* absl_nonnull`, and the +// exceptions (`absl_nullable` and `absl_nullability_unknown`) must be marked // explicitly. The same holds, correspondingly, for smart pointer types. // // For comparison, without the macro, all unannotated pointers would default to @@ -183,17 +216,16 @@ // // #include "absl/base/nullability.h" // -// void FillMessage(absl::Nonnull m); // explicitly non-null -// absl::Nullable GetNullablePtr(); // explicitly nullable +// void FillMessage(Message* absl_nonnull m); // explicitly non-null +// T* absl_nullable GetNullablePtr(); // explicitly nullable // T* GetUnknownPtr(); // implicitly unknown // // No-op except for being a human readable signal. #define ABSL_POINTERS_DEFAULT_NONNULL -namespace absl { -ABSL_NAMESPACE_BEGIN - -// absl::Nonnull (default with `ABSL_POINTERS_DEFAULT_NONNULL`) +#if defined(__clang__) && !defined(__OBJC__) && \ + ABSL_HAVE_FEATURE(nullability_on_classes) +// absl_nonnull (default with `ABSL_POINTERS_DEFAULT_NONNULL`) // // The indicated pointer is never null. It is the responsibility of the provider // of this pointer across an API boundary to ensure that the pointer is never @@ -203,13 +235,12 @@ ABSL_NAMESPACE_BEGIN // Example: // // // `employee` is designated as not null. -// void PaySalary(absl::Nonnull employee) { +// void PaySalary(Employee* absl_nonnull employee) { // pay(*employee); // OK to dereference // } -template -using Nonnull = nullability_internal::NonnullImpl; +#define absl_nonnull _Nonnull -// absl::Nullable +// absl_nullable // // The indicated pointer may, by design, be either null or non-null. Consumers // of this pointer across an API boundary should perform a `nullptr` check @@ -218,15 +249,14 @@ using Nonnull = nullability_internal::NonnullImpl; // Example: // // // `employee` may be null. -// void PaySalary(absl::Nullable employee) { +// void PaySalary(Employee* absl_nullable employee) { // if (employee != nullptr) { // Pay(*employee); // OK to dereference // } // } -template -using Nullable = nullability_internal::NullableImpl; +#define absl_nullable _Nullable -// absl::NullabilityUnknown (default without `ABSL_POINTERS_DEFAULT_NONNULL`) +// absl_nullability_unknown (default without `ABSL_POINTERS_DEFAULT_NONNULL`) // // The indicated pointer has not yet been determined to be definitively // "non-null" or "nullable." Providers of such pointers across API boundaries @@ -234,8 +264,8 @@ using Nullable = nullability_internal::NullableImpl; // Consumers of these pointers across an API boundary should treat such pointers // with the same caution they treat currently unannotated pointers. Most // existing code will have "unknown" pointers, which should eventually be -// migrated into one of the above two nullability states: `Nonnull` or -// `Nullable`. +// migrated into one of the above two nullability states: `absl_nonnull` or +// `absl_nullable`. // // NOTE: For files that do not specify `ABSL_POINTERS_DEFAULT_NONNULL`, // because this annotation is the global default state, unannotated pointers are @@ -245,7 +275,7 @@ using Nullable = nullability_internal::NullableImpl; // Example: // // // `employee`s nullability state is unknown. -// void PaySalary(absl::NullabilityUnknown employee) { +// void PaySalary(Employee* absl_nullability_unknown employee) { // Pay(*employee); // Potentially dangerous. API provider should investigate. // } // @@ -256,11 +286,15 @@ using Nullable = nullability_internal::NullableImpl; // void PaySalary(Employee* employee) { // Pay(*employee); // Potentially dangerous. API provider should investigate. // } -template -using NullabilityUnknown = nullability_internal::NullabilityUnknownImpl; - -ABSL_NAMESPACE_END -} // namespace absl +#define absl_nullability_unknown _Null_unspecified +#else +// No-op for non-Clang compilers or Objective-C. +#define absl_nonnull +// No-op for non-Clang compilers or Objective-C. +#define absl_nullable +// No-op for non-Clang compilers or Objective-C. +#define absl_nullability_unknown +#endif // ABSL_NULLABILITY_COMPATIBLE // @@ -281,26 +315,4 @@ ABSL_NAMESPACE_END #define ABSL_NULLABILITY_COMPATIBLE #endif -// ABSL_NONNULL -// ABSL_NULLABLE -// ABSL_NULLABILITY_UNKNOWN -// -// These macros are analogues of the alias template nullability annotations -// above. -// -// Example: -// int* ABSL_NULLABLE foo; -// Is equivalent to: -// absl::Nullable foo; -#if defined(__clang__) && !defined(__OBJC__) && \ - ABSL_HAVE_FEATURE(nullability_on_classes) -#define ABSL_NONNULL _Nonnull -#define ABSL_NULLABLE _Nullable -#define ABSL_NULLABILITY_UNKNOWN _Null_unspecified -#else -#define ABSL_NONNULL -#define ABSL_NULLABLE -#define ABSL_NULLABILITY_UNKNOWN -#endif - #endif // ABSL_BASE_NULLABILITY_H_ diff --git a/absl/base/options.h b/absl/base/options.h index 5caa58f..f904f64 100644 --- a/absl/base/options.h +++ b/absl/base/options.h @@ -64,65 +64,14 @@ // proper Abseil implementation at compile-time, which will not be sufficient // to guarantee ABI stability to package managers. +// SKIP_ABSL_INLINE_NAMESPACE_CHECK + #ifndef ABSL_BASE_OPTIONS_H_ #define ABSL_BASE_OPTIONS_H_ // ----------------------------------------------------------------------------- // Type Compatibility Options // ----------------------------------------------------------------------------- -// -// ABSL_OPTION_USE_STD_ANY -// -// This option controls whether absl::any is implemented as an alias to -// std::any, or as an independent implementation. -// -// A value of 0 means to use Abseil's implementation. This requires only C++11 -// support, and is expected to work on every toolchain we support. -// -// A value of 1 means to use an alias to std::any. This requires that all code -// using Abseil is built in C++17 mode or later. -// -// A value of 2 means to detect the C++ version being used to compile Abseil, -// and use an alias only if a working std::any is available. This option is -// useful when you are building your entire program, including all of its -// dependencies, from source. It should not be used otherwise -- for example, -// if you are distributing Abseil in a binary package manager -- since in -// mode 2, absl::any will name a different type, with a different mangled name -// and binary layout, depending on the compiler flags passed by the end user. -// For more info, see https://abseil.io/about/design/dropin-types. -// -// User code should not inspect this macro. To check in the preprocessor if -// absl::any is a typedef of std::any, use the feature macro ABSL_USES_STD_ANY. - -#define ABSL_OPTION_USE_STD_ANY 2 - - -// ABSL_OPTION_USE_STD_OPTIONAL -// -// This option controls whether absl::optional is implemented as an alias to -// std::optional, or as an independent implementation. -// -// A value of 0 means to use Abseil's implementation. This requires only C++11 -// support, and is expected to work on every toolchain we support. -// -// A value of 1 means to use an alias to std::optional. This requires that all -// code using Abseil is built in C++17 mode or later. -// -// A value of 2 means to detect the C++ version being used to compile Abseil, -// and use an alias only if a working std::optional is available. This option -// is useful when you are building your program from source. It should not be -// used otherwise -- for example, if you are distributing Abseil in a binary -// package manager -- since in mode 2, absl::optional will name a different -// type, with a different mangled name and binary layout, depending on the -// compiler flags passed by the end user. For more info, see -// https://abseil.io/about/design/dropin-types. - -// User code should not inspect this macro. To check in the preprocessor if -// absl::optional is a typedef of std::optional, use the feature macro -// ABSL_USES_STD_OPTIONAL. - -#define ABSL_OPTION_USE_STD_OPTIONAL 2 - // ABSL_OPTION_USE_STD_STRING_VIEW // @@ -150,32 +99,6 @@ #define ABSL_OPTION_USE_STD_STRING_VIEW 2 -// ABSL_OPTION_USE_STD_VARIANT -// -// This option controls whether absl::variant is implemented as an alias to -// std::variant, or as an independent implementation. -// -// A value of 0 means to use Abseil's implementation. This requires only C++11 -// support, and is expected to work on every toolchain we support. -// -// A value of 1 means to use an alias to std::variant. This requires that all -// code using Abseil is built in C++17 mode or later. -// -// A value of 2 means to detect the C++ version being used to compile Abseil, -// and use an alias only if a working std::variant is available. This option -// is useful when you are building your program from source. It should not be -// used otherwise -- for example, if you are distributing Abseil in a binary -// package manager -- since in mode 2, absl::variant will name a different -// type, with a different mangled name and binary layout, depending on the -// compiler flags passed by the end user. For more info, see -// https://abseil.io/about/design/dropin-types. -// -// User code should not inspect this macro. To check in the preprocessor if -// absl::variant is a typedef of std::variant, use the feature macro -// ABSL_USES_STD_VARIANT. - -#define ABSL_OPTION_USE_STD_VARIANT 2 - // ABSL_OPTION_USE_STD_ORDERING // // This option controls whether absl::{partial,weak,strong}_ordering are @@ -226,7 +149,7 @@ // allowed. #define ABSL_OPTION_USE_INLINE_NAMESPACE 1 -#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20250127 +#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20250512 // ABSL_OPTION_HARDENED // diff --git a/absl/base/policy_checks.h b/absl/base/policy_checks.h index 7538166..f84944c 100644 --- a/absl/base/policy_checks.h +++ b/absl/base/policy_checks.h @@ -71,15 +71,15 @@ // C++ Version Check // ----------------------------------------------------------------------------- -// Enforce C++14 as the minimum. +// Enforce C++17 as the minimum. #if defined(_MSVC_LANG) -#if _MSVC_LANG < 201402L -#error "C++ versions less than C++14 are not supported." -#endif // _MSVC_LANG < 201402L +#if _MSVC_LANG < 201703L +#error "C++ versions less than C++17 are not supported." +#endif // _MSVC_LANG < 201703L #elif defined(__cplusplus) -#if __cplusplus < 201402L -#error "C++ versions less than C++14 are not supported." -#endif // __cplusplus < 201402L +#if __cplusplus < 201703L +#error "C++ versions less than C++17 are not supported." +#endif // __cplusplus < 201703L #endif // ----------------------------------------------------------------------------- diff --git a/absl/cleanup/cleanup.h b/absl/cleanup/cleanup.h index 960ccd0..311e482 100644 --- a/absl/cleanup/cleanup.h +++ b/absl/cleanup/cleanup.h @@ -78,7 +78,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN template -class ABSL_MUST_USE_RESULT Cleanup final { +class [[nodiscard]] Cleanup final { static_assert(cleanup_internal::WasDeduced(), "Explicit template parameters are not supported."); @@ -115,10 +115,8 @@ class ABSL_MUST_USE_RESULT Cleanup final { // `absl::Cleanup c = /* callback */;` // // C++17 type deduction API for creating an instance of `absl::Cleanup` -#if defined(ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION) template Cleanup(Callback callback) -> Cleanup; -#endif // defined(ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION) // `auto c = absl::MakeCleanup(/* callback */);` // diff --git a/absl/cleanup/internal/cleanup.h b/absl/cleanup/internal/cleanup.h index 2783fcb..4dd6f91 100644 --- a/absl/cleanup/internal/cleanup.h +++ b/absl/cleanup/internal/cleanup.h @@ -19,7 +19,6 @@ #include #include -#include "absl/base/internal/invoke.h" #include "absl/base/macros.h" #include "absl/base/thread_annotations.h" #include "absl/utility/utility.h" @@ -39,7 +38,7 @@ constexpr bool WasDeduced() { template constexpr bool ReturnsVoid() { - return (std::is_same, void>::value); + return (std::is_same, void>::value); } template @@ -70,7 +69,7 @@ class Storage { Storage& operator=(const Storage& other) = delete; - void* GetCallbackBuffer() { return static_cast(+callback_buffer_); } + void* GetCallbackBuffer() { return static_cast(callback_buffer_); } Callback& GetCallback() { return *reinterpret_cast(GetCallbackBuffer()); @@ -89,7 +88,7 @@ class Storage { private: bool is_callback_engaged_; - alignas(Callback) char callback_buffer_[sizeof(Callback)]; + alignas(Callback) unsigned char callback_buffer_[sizeof(Callback)]; }; } // namespace cleanup_internal diff --git a/absl/container/btree_map.h b/absl/container/btree_map.h index 470de2a..32a82ef 100644 --- a/absl/container/btree_map.h +++ b/absl/container/btree_map.h @@ -47,8 +47,10 @@ // iterator at the current position. Another important difference is that // key-types must be copy-constructible. // -// Another API difference is that btree iterators can be subtracted, and this -// is faster than using std::distance. +// There are other API differences: first, btree iterators can be subtracted, +// and this is faster than using `std::distance`. Additionally, btree +// iterators can be advanced via `operator+=` and `operator-=`, which is faster +// than using `std::advance`. // // B-tree maps are not exception-safe. diff --git a/absl/container/btree_set.h b/absl/container/btree_set.h index e57d6d9..16181de 100644 --- a/absl/container/btree_set.h +++ b/absl/container/btree_set.h @@ -46,8 +46,10 @@ // reason, `insert()`, `erase()`, and `extract_and_get_next()` return a valid // iterator at the current position. // -// Another API difference is that btree iterators can be subtracted, and this -// is faster than using std::distance. +// There are other API differences: first, btree iterators can be subtracted, +// and this is faster than using `std::distance`. Additionally, btree +// iterators can be advanced via `operator+=` and `operator-=`, which is faster +// than using `std::advance`. // // B-tree sets are not exception-safe. diff --git a/absl/container/fixed_array.h b/absl/container/fixed_array.h index 95abb0a..6c238fc 100644 --- a/absl/container/fixed_array.h +++ b/absl/container/fixed_array.h @@ -44,11 +44,13 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/iterator_traits.h" #include "absl/base/internal/throw_delegate.h" #include "absl/base/macros.h" #include "absl/base/optimization.h" #include "absl/base/port.h" #include "absl/container/internal/compressed_tuple.h" +#include "absl/hash/internal/weakly_mixed_integer.h" #include "absl/memory/memory.h" namespace absl { @@ -85,9 +87,8 @@ class ABSL_ATTRIBUTE_WARN_UNUSED FixedArray { // std::iterator_traits isn't guaranteed to be SFINAE-friendly until C++17, // but this seems to be mostly pedantic. template - using EnableIfForwardIterator = absl::enable_if_t::iterator_category, - std::forward_iterator_tag>::value>; + using EnableIfForwardIterator = std::enable_if_t< + base_internal::IsAtLeastForwardIterator::value>; static constexpr bool NoexceptCopyable() { return std::is_nothrow_copy_constructible::value && absl::allocator_is_nothrow::value; @@ -392,7 +393,7 @@ class ABSL_ATTRIBUTE_WARN_UNUSED FixedArray { template friend H AbslHashValue(H h, const FixedArray& v) { return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()), - v.size()); + hash_internal::WeaklyMixedInteger{v.size()}); } private: @@ -447,7 +448,8 @@ class ABSL_ATTRIBUTE_WARN_UNUSED FixedArray { private: ABSL_ADDRESS_SANITIZER_REDZONE(redzone_begin_); - alignas(StorageElement) char buff_[sizeof(StorageElement[inline_elements])]; + alignas(StorageElement) unsigned char buff_[sizeof( + StorageElement[inline_elements])]; ABSL_ADDRESS_SANITIZER_REDZONE(redzone_end_); }; @@ -517,15 +519,6 @@ class ABSL_ATTRIBUTE_WARN_UNUSED FixedArray { Storage storage_; }; -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -template -constexpr size_t FixedArray::kInlineBytesDefault; - -template -constexpr typename FixedArray::size_type - FixedArray::inline_elements; -#endif - template void FixedArray::NonEmptyInlinedStorage::AnnotateConstruct( typename FixedArray::size_type n) { diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index 735ee34..bc86ced 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -104,6 +104,11 @@ struct FlatHashMapPolicy; // If your types are not moveable or you require pointer stability for keys, // consider `absl::node_hash_map`. // +// PERFORMANCE WARNING: Erasure & sparsity can negatively affect performance: +// * Iteration takes O(capacity) time, not O(size). +// * erase() slows down begin() and ++iterator. +// * Capacity only shrinks on rehash() or clear() -- not on erase(). +// // Example: // // // Create a flat hash map of three strings (that map to strings) diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index b5d0f7f..bf63eb5 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -62,7 +62,7 @@ struct FlatHashSetPolicy; // Its interface is similar to that of `std::unordered_set` with the // following notable differences: // -// * Requires keys that are CopyConstructible +// * Requires keys that are MoveConstructible // * Supports heterogeneous lookup, through `find()` and `insert()`, provided // that the set is provided a compatible heterogeneous hashing function and // equality operator. See below for details. @@ -103,6 +103,11 @@ struct FlatHashSetPolicy; // `absl::flat_hash_set>`. If your type is not moveable and // you require pointer stability, consider `absl::node_hash_set` instead. // +// PERFORMANCE WARNING: Erasure & sparsity can negatively affect performance: +// * Iteration takes O(capacity) time, not O(size). +// * erase() slows down begin() and ++iterator. +// * Capacity only shrinks on rehash() or clear() -- not on erase(). +// // Example: // // // Create a flat hash set of three strings diff --git a/absl/container/inlined_vector.h b/absl/container/inlined_vector.h index cbf8bc2..f871b34 100644 --- a/absl/container/inlined_vector.h +++ b/absl/container/inlined_vector.h @@ -47,11 +47,13 @@ #include "absl/algorithm/algorithm.h" #include "absl/base/attributes.h" +#include "absl/base/internal/iterator_traits.h" #include "absl/base/internal/throw_delegate.h" #include "absl/base/macros.h" #include "absl/base/optimization.h" #include "absl/base/port.h" #include "absl/container/internal/inlined_vector.h" +#include "absl/hash/internal/weakly_mixed_integer.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" @@ -90,11 +92,11 @@ class ABSL_ATTRIBUTE_WARN_UNUSED InlinedVector { inlined_vector_internal::DefaultValueAdapter; template - using EnableIfAtLeastForwardIterator = absl::enable_if_t< - inlined_vector_internal::IsAtLeastForwardIterator::value, int>; + using EnableIfAtLeastForwardIterator = std::enable_if_t< + base_internal::IsAtLeastForwardIterator::value, int>; template - using DisableIfAtLeastForwardIterator = absl::enable_if_t< - !inlined_vector_internal::IsAtLeastForwardIterator::value, int>; + using DisableIfAtLeastForwardIterator = std::enable_if_t< + !base_internal::IsAtLeastForwardIterator::value, int>; using MemcpyPolicy = typename Storage::MemcpyPolicy; using ElementwiseAssignPolicy = typename Storage::ElementwiseAssignPolicy; @@ -1007,7 +1009,8 @@ bool operator>=(const absl::InlinedVector& a, template H AbslHashValue(H h, const absl::InlinedVector& a) { auto size = a.size(); - return H::combine(H::combine_contiguous(std::move(h), a.data(), size), size); + return H::combine(H::combine_contiguous(std::move(h), a.data(), size), + hash_internal::WeaklyMixedInteger{size}); } ABSL_NAMESPACE_END diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index 689e71a..ed541e7 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -60,6 +60,7 @@ #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" +#include "absl/base/optimization.h" #include "absl/container/internal/common.h" #include "absl/container/internal/common_policy_traits.h" #include "absl/container/internal/compressed_tuple.h" @@ -224,7 +225,7 @@ struct key_compare_adapter { public: using Base::Base; - checked_compare(Compare comp) : Base(std::move(comp)) {} // NOLINT + checked_compare(Compare cmp) : Base(std::move(cmp)) {} // NOLINT // Allow converting to Compare for use in key_comp()/value_comp(). explicit operator Compare() const { return comp(); } @@ -708,6 +709,8 @@ class btree_node { } // Getter for the parent of this node. + // TODO(ezb): assert that the child of the returned node at position + // `node_->position()` maps to the current node. btree_node *parent() const { return *GetField<0>(); } // Getter for whether the node is the root of the tree. The parent of the // root of the tree is the leftmost node in the tree which is guaranteed to @@ -1175,6 +1178,26 @@ class btree_iterator : private btree_iterator_generation_info { return distance_slow(other); } + // Advances the iterator by `n`. Values of `n` must not result in going past + // the `end` iterator (for a positive `n`) or before the `begin` iterator (for + // a negative `n`). + btree_iterator &operator+=(difference_type n) { + assert_valid_generation(node_); + if (n == 0) return *this; + if (n < 0) return decrement_n_slow(-n); + return increment_n_slow(n); + } + + // Moves the iterator by `n` positions backwards. Values of `n` must not + // result in going before the `begin` iterator (for a positive `n`) or past + // the `end` iterator (for a negative `n`). + btree_iterator &operator-=(difference_type n) { + assert_valid_generation(node_); + if (n == 0) return *this; + if (n < 0) return increment_n_slow(-n); + return decrement_n_slow(n); + } + // Accessors for the key/value the iterator is pointing at. reference operator*() const { ABSL_HARDENING_ASSERT(node_ != nullptr); @@ -1277,6 +1300,7 @@ class btree_iterator : private btree_iterator_generation_info { increment_slow(); } void increment_slow(); + btree_iterator &increment_n_slow(difference_type n); void decrement() { assert_valid_generation(node_); @@ -1286,6 +1310,7 @@ class btree_iterator : private btree_iterator_generation_info { decrement_slow(); } void decrement_slow(); + btree_iterator &decrement_n_slow(difference_type n); const key_type &key() const { return node_->key(static_cast(position_)); @@ -2126,50 +2151,128 @@ auto btree_iterator::distance_slow(const_iterator other) const template void btree_iterator::increment_slow() { - if (node_->is_leaf()) { - assert(position_ >= node_->finish()); - btree_iterator save(*this); - while (position_ == node_->finish() && !node_->is_root()) { - assert(node_->parent()->child(node_->position()) == node_); - position_ = node_->position(); - node_ = node_->parent(); + N* node = node_; + int position = position_; + if (node->is_leaf()) { + assert(position >= node->finish()); + while (position == node->finish() && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position(); + node = node->parent(); } // TODO(ezb): assert we aren't incrementing end() instead of handling. - if (position_ == node_->finish()) { - *this = save; + if (position == node->finish()) { + return; } } else { - assert(position_ < node_->finish()); - node_ = node_->child(static_cast(position_ + 1)); - while (node_->is_internal()) { - node_ = node_->start_child(); + assert(position < node->finish()); + node = node->child(static_cast(position + 1)); + while (node->is_internal()) { + node = node->start_child(); } - position_ = node_->start(); + position = node->start(); } + *this = {node, position}; } template void btree_iterator::decrement_slow() { - if (node_->is_leaf()) { - assert(position_ <= -1); - btree_iterator save(*this); - while (position_ < node_->start() && !node_->is_root()) { - assert(node_->parent()->child(node_->position()) == node_); - position_ = node_->position() - 1; - node_ = node_->parent(); + N* node = node_; + int position = position_; + if (node->is_leaf()) { + assert(position <= -1); + while (position < node->start() && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position() - 1; + node = node->parent(); } // TODO(ezb): assert we aren't decrementing begin() instead of handling. - if (position_ < node_->start()) { - *this = save; + if (position < node->start()) { + return; } } else { - assert(position_ >= node_->start()); - node_ = node_->child(static_cast(position_)); - while (node_->is_internal()) { - node_ = node_->child(node_->finish()); + assert(position >= node->start()); + node = node->child(static_cast(position)); + while (node->is_internal()) { + node = node->child(node->finish()); } - position_ = node_->finish() - 1; + position = node->finish() - 1; } + *this = {node, position}; +} + +template +btree_iterator &btree_iterator::increment_n_slow( + difference_type n) { + N *node = node_; + int position = position_; + ABSL_ASSUME(n > 0); + while (n > 0) { + if (node->is_leaf()) { + if (position + n < node->finish()) { + position += n; + break; + } else { + n -= node->finish() - position; + position = node->finish(); + btree_iterator save = {node, position}; + while (position == node->finish() && !node->is_root()) { + position = node->position(); + node = node->parent(); + } + if (position == node->finish()) { + ABSL_HARDENING_ASSERT(n == 0); + return *this = save; + } + } + } else { + --n; + assert(position < node->finish()); + node = node->child(static_cast(position + 1)); + while (node->is_internal()) { + node = node->start_child(); + } + position = node->start(); + } + } + node_ = node; + position_ = position; + return *this; +} + +template +btree_iterator &btree_iterator::decrement_n_slow( + difference_type n) { + N *node = node_; + int position = position_; + ABSL_ASSUME(n > 0); + while (n > 0) { + if (node->is_leaf()) { + if (position - n >= node->start()) { + position -= n; + break; + } else { + n -= 1 + position - node->start(); + position = node->start() - 1; + while (position < node->start() && !node->is_root()) { + position = node->position() - 1; + node = node->parent(); + } + ABSL_HARDENING_ASSERT(position >= node->start()); + } + } else { + --n; + assert(position >= node->start()); + node = node->child(static_cast(position)); + while (node->is_internal()) { + node = node->child(node->finish()); + } + position = node->finish() - 1; + } + } + node_ = node; + position_ = position; + return *this; } //// diff --git a/absl/container/internal/btree_container.h b/absl/container/internal/btree_container.h index a68ce44..21f00ae 100644 --- a/absl/container/internal/btree_container.h +++ b/absl/container/internal/btree_container.h @@ -18,12 +18,14 @@ #include #include #include +#include #include #include "absl/base/attributes.h" #include "absl/base/internal/throw_delegate.h" #include "absl/container/internal/btree.h" // IWYU pragma: export #include "absl/container/internal/common.h" +#include "absl/hash/internal/weakly_mixed_integer.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" @@ -266,7 +268,8 @@ class btree_container { for (const auto &v : b) { h = State::combine(std::move(h), v); } - return State::combine(std::move(h), b.size()); + return State::combine(std::move(h), + hash_internal::WeaklyMixedInteger{b.size()}); } protected: @@ -451,6 +454,29 @@ class btree_map_container : public btree_set_container { template using key_arg = typename super_type::template key_arg; + // NOTE: The mess here is to shorten the code for the (very repetitive) + // function overloads, and to allow the lifetime-bound overloads to dispatch + // to the non-lifetime-bound overloads, to ensure there is a single source of + // truth for each overload set. + // + // Enabled if an assignment from the given type would require the + // source object to remain alive for the life of the element. + // + // TODO(b/402804213): Remove these traits and simplify the overloads whenever + // we have a better mechanism available to handle lifetime analysis. + template + using LifetimeBoundK = + HasValue>; + template + using LifetimeBoundV = + HasValue>; + template + using LifetimeBoundKV = + absl::conjunction>, + LifetimeBoundV>; + public: using key_type = typename Tree::key_type; using mapped_type = typename params_type::mapped_type; @@ -464,85 +490,163 @@ class btree_map_container : public btree_set_container { using super_type::super_type; btree_map_container() {} + // TODO(b/402804213): Remove these macros whenever we have a better mechanism + // available to handle lifetime analysis. +#define ABSL_INTERNAL_X(Func, Callee, KQual, MQual, KValue, MValue, ...) \ + template < \ + typename K = key_type, class M, \ + ABSL_INTERNAL_IF_##KValue##_NOR_##MValue( \ + int = (EnableIf::AddPtr, \ + IfRRef::AddPtr>>()), \ + ABSL_INTERNAL_SINGLE_ARG( \ + int &..., \ + decltype(EnableIf>()) = \ + 0))> \ + decltype(auto) Func( \ + __VA_ARGS__ key_arg KQual k ABSL_INTERNAL_IF_##KValue( \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)), \ + M MQual obj ABSL_INTERNAL_IF_##MValue( \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this))) \ + ABSL_ATTRIBUTE_LIFETIME_BOUND { \ + return ABSL_INTERNAL_IF_##KValue##_OR_##MValue( \ + (this->template Func), Callee)( \ + __VA_ARGS__ std::forward(k), \ + std::forward(obj)); \ + } \ + friend struct std::enable_if /* just to force a semicolon */ // Insertion routines. // Note: the nullptr template arguments and extra `const M&` overloads allow // for supporting bitfield arguments. - template - std::pair insert_or_assign(const key_arg &k, const M &obj) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(k, obj); - } - template - std::pair insert_or_assign(key_arg &&k, const M &obj) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(std::forward(k), obj); - } - template - std::pair insert_or_assign(const key_arg &k, M &&obj) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(k, std::forward(obj)); - } - template - std::pair insert_or_assign(key_arg &&k, M &&obj) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(std::forward(k), std::forward(obj)); - } - template - iterator insert_or_assign(const_iterator hint, const key_arg &k, - const M &obj) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_hint_impl(hint, k, obj); - } - template - iterator insert_or_assign(const_iterator hint, key_arg &&k, - const M &obj) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_hint_impl(hint, std::forward(k), obj); - } - template - iterator insert_or_assign(const_iterator hint, const key_arg &k, - M &&obj) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_hint_impl(hint, k, std::forward(obj)); - } - template - iterator insert_or_assign(const_iterator hint, key_arg &&k, - M &&obj) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_hint_impl(hint, std::forward(k), - std::forward(obj)); - } - - template ::value, int> = 0> - std::pair try_emplace(const key_arg &k, Args &&...args) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return try_emplace_impl(k, std::forward(args)...); - } - template ::value, int> = 0> - std::pair try_emplace(key_arg &&k, Args &&...args) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return try_emplace_impl(std::forward(k), std::forward(args)...); - } - template - iterator try_emplace(const_iterator hint, const key_arg &k, - Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return try_emplace_hint_impl(hint, k, std::forward(args)...); - } - template - iterator try_emplace(const_iterator hint, key_arg &&k, - Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return try_emplace_hint_impl(hint, std::forward(k), - std::forward(args)...); - } - - template + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + false, false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + false, true); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + true, false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + true, true); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, false, + false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, false, + true); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, true, + false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, true, + true); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, false, + false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, false, + true); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, true, + false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, true, + true); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, false, + false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, false, true); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, true, false); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, true, true); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, + const &, false, false, + const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, + const &, false, true, + const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, + const &, true, false, + const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, + const &, true, true, + const_iterator(hint) ABSL_INTERNAL_COMMA); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, &&, + false, false, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, &&, + false, true, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, &&, + true, false, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, const &, &&, + true, true, const_iterator(hint) ABSL_INTERNAL_COMMA); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, const &, + false, false, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, const &, + false, true, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, const &, + true, false, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, const &, + true, true, const_iterator(hint) ABSL_INTERNAL_COMMA); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, &&, false, + false, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, &&, false, + true, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, &&, true, + false, const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_hint_impl, &&, &&, true, + true, const_iterator(hint) ABSL_INTERNAL_COMMA); +#undef ABSL_INTERNAL_X + +#define ABSL_INTERNAL_X(Func, Callee, KQual, KValue, ...) \ + template < \ + class K = key_type, \ + ABSL_INTERNAL_IF_##KValue( \ + class... Args, \ + int = (EnableIf< \ + LifetimeBoundK::AddPtr>>())), \ + ABSL_INTERNAL_IF_##KValue( \ + decltype(EnableIf::AddPtr>>()) = 0, \ + class... Args), \ + std::enable_if_t::value, int> = \ + 0> \ + decltype(auto) Func( \ + __VA_ARGS__ key_arg KQual k ABSL_INTERNAL_IF_##KValue( \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)), \ + Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { \ + return ABSL_INTERNAL_IF_##KValue((this->template Func), Callee)( \ + __VA_ARGS__ std::forward(k), \ + std::forward(args)...); \ + } \ + friend struct std::enable_if /* just to force a semicolon */ + ABSL_INTERNAL_X(try_emplace, try_emplace_impl, const &, false); + ABSL_INTERNAL_X(try_emplace, try_emplace_impl, const &, true); + ABSL_INTERNAL_X(try_emplace, try_emplace_impl, &&, false); + ABSL_INTERNAL_X(try_emplace, try_emplace_impl, &&, true); + ABSL_INTERNAL_X(try_emplace, try_emplace_hint_impl, const &, false, + const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(try_emplace, try_emplace_hint_impl, const &, true, + const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(try_emplace, try_emplace_hint_impl, &&, false, + const_iterator(hint) ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(try_emplace, try_emplace_hint_impl, &&, true, + const_iterator(hint) ABSL_INTERNAL_COMMA); +#undef ABSL_INTERNAL_X + + template >()> mapped_type &operator[](const key_arg &k) ABSL_ATTRIBUTE_LIFETIME_BOUND { return try_emplace(k).first->second; } - template + template > = 0> + mapped_type &operator[]( + const key_arg &k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template operator[](k); + } + template >()> mapped_type &operator[](key_arg &&k) ABSL_ATTRIBUTE_LIFETIME_BOUND { return try_emplace(std::forward(k)).first->second; } + template > = 0> + mapped_type &operator[](key_arg &&k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY( + this)) ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template operator[](std::forward(k)); + } template mapped_type &at(const key_arg &key) ABSL_ATTRIBUTE_LIFETIME_BOUND { diff --git a/absl/container/internal/common.h b/absl/container/internal/common.h index 9239bb4..5ef6c56 100644 --- a/absl/container/internal/common.h +++ b/absl/container/internal/common.h @@ -21,10 +21,53 @@ #include "absl/meta/type_traits.h" #include "absl/types/optional.h" +// TODO(b/402804213): Clean up these macros when no longer needed. +#define ABSL_INTERNAL_SINGLE_ARG(...) __VA_ARGS__ + +#define ABSL_INTERNAL_IF_true(if_satisfied, ...) if_satisfied +#define ABSL_INTERNAL_IF_false(if_satisfied, ...) __VA_ARGS__ + +#define ABSL_INTERNAL_IF_true_AND_true ABSL_INTERNAL_IF_true +#define ABSL_INTERNAL_IF_false_AND_false ABSL_INTERNAL_IF_false +#define ABSL_INTERNAL_IF_true_AND_false ABSL_INTERNAL_IF_false_AND_false +#define ABSL_INTERNAL_IF_false_AND_true ABSL_INTERNAL_IF_false_AND_false + +#define ABSL_INTERNAL_IF_true_OR_true ABSL_INTERNAL_IF_true +#define ABSL_INTERNAL_IF_false_OR_false ABSL_INTERNAL_IF_false +#define ABSL_INTERNAL_IF_true_OR_false ABSL_INTERNAL_IF_true_OR_true +#define ABSL_INTERNAL_IF_false_OR_true ABSL_INTERNAL_IF_true_OR_true + +#define ABSL_INTERNAL_IF_true_NOR_true ABSL_INTERNAL_IF_false_AND_false +#define ABSL_INTERNAL_IF_false_NOR_false ABSL_INTERNAL_IF_true_AND_true +#define ABSL_INTERNAL_IF_true_NOR_false ABSL_INTERNAL_IF_false_AND_true +#define ABSL_INTERNAL_IF_false_NOR_true ABSL_INTERNAL_IF_true_AND_false + +#define ABSL_INTERNAL_COMMA , + namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { +// TODO(b/402804213): Clean up these traits when no longer needed or +// deduplicate them with absl::functional_internal::EnableIf. +template +using EnableIf = std::enable_if_t; + +template +using HasValue = std::conditional_t>; + +template +struct IfRRef { + template + using AddPtr = Other; +}; + +template +struct IfRRef { + template + using AddPtr = Other*; +}; + template struct IsTransparent : std::false_type {}; template diff --git a/absl/container/internal/common_policy_traits.h b/absl/container/internal/common_policy_traits.h index bbf5475..86e038e 100644 --- a/absl/container/internal/common_policy_traits.h +++ b/absl/container/internal/common_policy_traits.h @@ -119,7 +119,7 @@ struct common_policy_traits { old_slot)) { return P::transfer(alloc, new_slot, old_slot); } -#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 + // This overload returns true_type for the trait below. // The conditional_t is to make the enabler type dependent. template (&element(old_slot)), sizeof(value_type)); return {}; } -#endif template static void transfer_impl(Alloc* alloc, slot_type* new_slot, diff --git a/absl/container/internal/container_memory.h b/absl/container/internal/container_memory.h index e703179..e7ac1db 100644 --- a/absl/container/internal/container_memory.h +++ b/absl/container/internal/container_memory.h @@ -374,9 +374,6 @@ struct map_slot_policy { return slot->value; } - // When C++17 is available, we can use std::launder to provide mutable - // access to the key for use in node handle. -#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 static K& mutable_key(slot_type* slot) { // Still check for kMutableKeys so that we can avoid calling std::launder // unless necessary because it can interfere with optimizations. @@ -384,9 +381,6 @@ struct map_slot_policy { : *std::launder(const_cast( std::addressof(slot->value.first))); } -#else // !(defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606) - static const K& mutable_key(slot_type* slot) { return key(slot); } -#endif static const K& key(const slot_type* slot) { return kMutableKeys::value ? slot->key : slot->value.first; @@ -439,11 +433,17 @@ struct map_slot_policy { template static auto transfer(Allocator* alloc, slot_type* new_slot, slot_type* old_slot) { - auto is_relocatable = - typename absl::is_trivially_relocatable::type(); + // This should really just be + // typename absl::is_trivially_relocatable::type() + // but std::pair is not trivially copyable in C++23 in some standard + // library versions. + // See https://github.com/llvm/llvm-project/pull/95444 for instance. + auto is_relocatable = typename std::conjunction< + absl::is_trivially_relocatable, + absl::is_trivially_relocatable>:: + type(); emplace(new_slot); -#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 if (is_relocatable) { // TODO(b/247130232,b/251814870): remove casts after fixing warnings. std::memcpy(static_cast(std::launder(&new_slot->value)), @@ -451,7 +451,6 @@ struct map_slot_policy { sizeof(value_type)); return is_relocatable; } -#endif if (kMutableKeys::value) { absl::allocator_traits::construct( diff --git a/absl/container/internal/hash_function_defaults.h b/absl/container/internal/hash_function_defaults.h index 0f07bcf..c2a757b 100644 --- a/absl/container/internal/hash_function_defaults.h +++ b/absl/container/internal/hash_function_defaults.h @@ -49,6 +49,7 @@ #include #include #include +#include #include #include "absl/base/config.h" @@ -58,10 +59,6 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#ifdef ABSL_HAVE_STD_STRING_VIEW -#include -#endif - namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { @@ -113,8 +110,6 @@ struct HashEq : StringHashEq {}; template <> struct HashEq : StringHashEq {}; -#ifdef ABSL_HAVE_STD_STRING_VIEW - template struct BasicStringHash { using is_transparent = void; @@ -153,8 +148,6 @@ struct HashEq : BasicStringHashEq {}; template <> struct HashEq : BasicStringHashEq {}; -#endif // ABSL_HAVE_STD_STRING_VIEW - // Supports heterogeneous lookup for pointers and smart pointers. template struct HashEq { diff --git a/absl/container/internal/hash_generator_testing.cc b/absl/container/internal/hash_generator_testing.cc index e89dfdb..be20e21 100644 --- a/absl/container/internal/hash_generator_testing.cc +++ b/absl/container/internal/hash_generator_testing.cc @@ -14,61 +14,39 @@ #include "absl/container/internal/hash_generator_testing.h" +#include #include +#include +#include #include "absl/base/no_destructor.h" +#include "absl/random/random.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { namespace hash_internal { -namespace { - -class RandomDeviceSeedSeq { - public: - using result_type = typename std::random_device::result_type; - - template - void generate(Iterator start, Iterator end) { - while (start != end) { - *start = gen_(); - ++start; - } - } - - private: - std::random_device gen_; -}; - -} // namespace - -std::mt19937_64* GetSharedRng() { - static absl::NoDestructor rng([] { - RandomDeviceSeedSeq seed_seq; - return std::mt19937_64(seed_seq); - }()); - return rng.get(); -} std::string Generator::operator()() const { + absl::InsecureBitGen gen; // NOLINTNEXTLINE(runtime/int) std::uniform_int_distribution chars(0x20, 0x7E); std::string res; res.resize(32); - std::generate(res.begin(), res.end(), - [&]() { return chars(*GetSharedRng()); }); + std::generate(res.begin(), res.end(), [&]() { return chars(gen); }); return res; } absl::string_view Generator::operator()() const { static absl::NoDestructor> arena; + absl::InsecureBitGen gen; // NOLINTNEXTLINE(runtime/int) std::uniform_int_distribution chars(0x20, 0x7E); arena->emplace_back(); auto& res = arena->back(); res.resize(32); - std::generate(res.begin(), res.end(), - [&]() { return chars(*GetSharedRng()); }); + std::generate(res.begin(), res.end(), [&]() { return chars(gen); }); return res; } diff --git a/absl/container/internal/hash_generator_testing.h b/absl/container/internal/hash_generator_testing.h index f1f555a..14c878e 100644 --- a/absl/container/internal/hash_generator_testing.h +++ b/absl/container/internal/hash_generator_testing.h @@ -23,7 +23,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -32,6 +34,7 @@ #include "absl/container/internal/hash_policy_testing.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" +#include "absl/random/random.h" #include "absl/strings/string_view.h" namespace absl { @@ -48,9 +51,7 @@ struct IsMap> : std::true_type {}; } // namespace generator_internal -std::mt19937_64* GetSharedRng(); - -enum Enum { +enum Enum : uint64_t { kEnumEmpty, kEnumDeleted, }; @@ -69,37 +70,27 @@ struct Generator; template struct Generator::value>::type> { - T operator()() const { - std::uniform_int_distribution dist; - return dist(*GetSharedRng()); - } + T operator()() const { return dist(gen); } + mutable absl::InsecureBitGen gen; + mutable std::uniform_int_distribution dist; }; template <> struct Generator { - Enum operator()() const { - std::uniform_int_distribution::type> - dist; - while (true) { - auto variate = dist(*GetSharedRng()); - if (variate != kEnumEmpty && variate != kEnumDeleted) - return static_cast(variate); - } - } + Enum operator()() const { return static_cast(dist(gen)); } + mutable absl::InsecureBitGen gen; + mutable std::uniform_int_distribution< + typename std::underlying_type::type> + dist; }; template <> struct Generator { - EnumClass operator()() const { - std::uniform_int_distribution< - typename std::underlying_type::type> - dist; - while (true) { - EnumClass variate = static_cast(dist(*GetSharedRng())); - if (variate != EnumClass::kEmpty && variate != EnumClass::kDeleted) - return static_cast(variate); - } - } + EnumClass operator()() const { return static_cast(dist(gen)); } + mutable absl::InsecureBitGen gen; + mutable std::uniform_int_distribution< + typename std::underlying_type::type> + dist; }; template <> @@ -143,17 +134,17 @@ struct Generator> { template struct Generator().key()), - decltype(std::declval().value())>> + decltype(std::declval().value())>> : Generator().key())>::type, typename std::decay().value())>::type>> {}; template -using GeneratedType = decltype( - std::declval::value, - typename Container::value_type, - typename Container::key_type>::type>&>()()); +using GeneratedType = + decltype(std::declval::value, + typename Container::value_type, + typename Container::key_type>::type>&>()()); // Naive wrapper that performs a linear search of previous values. // Beware this is O(SQR), which is reasonable for smaller kMaxValues. diff --git a/absl/container/internal/hash_policy_testing.h b/absl/container/internal/hash_policy_testing.h index 66bb12e..e9f5757 100644 --- a/absl/container/internal/hash_policy_testing.h +++ b/absl/container/internal/hash_policy_testing.h @@ -119,7 +119,11 @@ struct Alloc : std::allocator { using propagate_on_container_swap = std::true_type; // Using old paradigm for this to ensure compatibility. - explicit Alloc(size_t id = 0) : id_(id) {} + // + // NOTE: As of 2025-05, this constructor cannot be explicit in order to work + // with the libstdc++ that ships with GCC15. + // NOLINTNEXTLINE(google-explicit-constructor) + Alloc(size_t id = 0) : id_(id) {} Alloc(const Alloc&) = default; Alloc& operator=(const Alloc&) = default; diff --git a/absl/container/internal/hash_policy_traits.h b/absl/container/internal/hash_policy_traits.h index ad835d6..cd6b42f 100644 --- a/absl/container/internal/hash_policy_traits.h +++ b/absl/container/internal/hash_policy_traits.h @@ -36,16 +36,12 @@ struct hash_policy_traits : common_policy_traits { private: struct ReturnKey { - // When C++17 is available, we can use std::launder to provide mutable - // access to the key for use in node handle. -#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 template ::value, int> = 0> static key_type& Impl(Key&& k, int) { return *std::launder( const_cast(std::addressof(std::forward(k)))); } -#endif template static Key Impl(Key&& k, char) { diff --git a/absl/container/internal/hashtable_control_bytes.h b/absl/container/internal/hashtable_control_bytes.h new file mode 100644 index 0000000..abaadc3 --- /dev/null +++ b/absl/container/internal/hashtable_control_bytes.h @@ -0,0 +1,527 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file contains the implementation of the hashtable control bytes +// manipulation. + +#ifndef ABSL_CONTAINER_INTERNAL_HASHTABLE_CONTROL_BYTES_H_ +#define ABSL_CONTAINER_INTERNAL_HASHTABLE_CONTROL_BYTES_H_ + +#include +#include +#include +#include + +#include "absl/base/config.h" + +#ifdef ABSL_INTERNAL_HAVE_SSE2 +#include +#endif + +#ifdef ABSL_INTERNAL_HAVE_SSSE3 +#include +#endif + +#ifdef _MSC_VER +#include +#endif + +#ifdef ABSL_INTERNAL_HAVE_ARM_NEON +#include +#endif + +#include "absl/base/optimization.h" +#include "absl/numeric/bits.h" +#include "absl/base/internal/endian.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +#ifdef ABSL_SWISSTABLE_ASSERT +#error ABSL_SWISSTABLE_ASSERT cannot be directly set +#else +// We use this macro for assertions that users may see when the table is in an +// invalid state that sanitizers may help diagnose. +#define ABSL_SWISSTABLE_ASSERT(CONDITION) \ + assert((CONDITION) && "Try enabling sanitizers.") +#endif + + +template +uint32_t TrailingZeros(T x) { + ABSL_ASSUME(x != 0); + return static_cast(countr_zero(x)); +} + +// 8 bytes bitmask with most significant bit set for every byte. +constexpr uint64_t kMsbs8Bytes = 0x8080808080808080ULL; +// 8 kEmpty bytes that is useful for small table initialization. +constexpr uint64_t k8EmptyBytes = kMsbs8Bytes; + +// An abstract bitmask, such as that emitted by a SIMD instruction. +// +// Specifically, this type implements a simple bitset whose representation is +// controlled by `SignificantBits` and `Shift`. `SignificantBits` is the number +// of abstract bits in the bitset, while `Shift` is the log-base-two of the +// width of an abstract bit in the representation. +// This mask provides operations for any number of real bits set in an abstract +// bit. To add iteration on top of that, implementation must guarantee no more +// than the most significant real bit is set in a set abstract bit. +template +class NonIterableBitMask { + public: + explicit NonIterableBitMask(T mask) : mask_(mask) {} + + explicit operator bool() const { return this->mask_ != 0; } + + // Returns the index of the lowest *abstract* bit set in `self`. + uint32_t LowestBitSet() const { + return container_internal::TrailingZeros(mask_) >> Shift; + } + + // Returns the index of the highest *abstract* bit set in `self`. + uint32_t HighestBitSet() const { + return static_cast((bit_width(mask_) - 1) >> Shift); + } + + // Returns the number of trailing zero *abstract* bits. + uint32_t TrailingZeros() const { + return container_internal::TrailingZeros(mask_) >> Shift; + } + + // Returns the number of leading zero *abstract* bits. + uint32_t LeadingZeros() const { + constexpr int total_significant_bits = SignificantBits << Shift; + constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; + return static_cast( + countl_zero(static_cast(mask_ << extra_bits))) >> + Shift; + } + + T mask_; +}; + +// Mask that can be iterable +// +// For example, when `SignificantBits` is 16 and `Shift` is zero, this is just +// an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When +// `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as +// the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask. +// If NullifyBitsOnIteration is true (only allowed for Shift == 3), +// non zero abstract bit is allowed to have additional bits +// (e.g., `0xff`, `0x83` and `0x9c` are ok, but `0x6f` is not). +// +// For example: +// for (int i : BitMask(0b101)) -> yields 0, 2 +// for (int i : BitMask(0x0000000080800000)) -> yields 2, 3 +template +class BitMask : public NonIterableBitMask { + using Base = NonIterableBitMask; + static_assert(std::is_unsigned::value, ""); + static_assert(Shift == 0 || Shift == 3, ""); + static_assert(!NullifyBitsOnIteration || Shift == 3, ""); + + public: + explicit BitMask(T mask) : Base(mask) { + if (Shift == 3 && !NullifyBitsOnIteration) { + ABSL_SWISSTABLE_ASSERT(this->mask_ == (this->mask_ & kMsbs8Bytes)); + } + } + // BitMask is an iterator over the indices of its abstract bits. + using value_type = int; + using iterator = BitMask; + using const_iterator = BitMask; + + BitMask& operator++() { + if (Shift == 3 && NullifyBitsOnIteration) { + this->mask_ &= kMsbs8Bytes; + } + this->mask_ &= (this->mask_ - 1); + return *this; + } + + uint32_t operator*() const { return Base::LowestBitSet(); } + + BitMask begin() const { return *this; } + BitMask end() const { return BitMask(0); } + + private: + friend bool operator==(const BitMask& a, const BitMask& b) { + return a.mask_ == b.mask_; + } + friend bool operator!=(const BitMask& a, const BitMask& b) { + return a.mask_ != b.mask_; + } +}; + +using h2_t = uint8_t; + +// The values here are selected for maximum performance. See the static asserts +// below for details. + +// A `ctrl_t` is a single control byte, which can have one of four +// states: empty, deleted, full (which has an associated seven-bit h2_t value) +// and the sentinel. They have the following bit patterns: +// +// empty: 1 0 0 0 0 0 0 0 +// deleted: 1 1 1 1 1 1 1 0 +// full: 0 h h h h h h h // h represents the hash bits. +// sentinel: 1 1 1 1 1 1 1 1 +// +// These values are specifically tuned for SSE-flavored SIMD. +// The static_asserts below detail the source of these choices. +// +// We use an enum class so that when strict aliasing is enabled, the compiler +// knows ctrl_t doesn't alias other types. +enum class ctrl_t : int8_t { + kEmpty = -128, // 0b10000000 + kDeleted = -2, // 0b11111110 + kSentinel = -1, // 0b11111111 +}; +static_assert( + (static_cast(ctrl_t::kEmpty) & + static_cast(ctrl_t::kDeleted) & + static_cast(ctrl_t::kSentinel) & 0x80) != 0, + "Special markers need to have the MSB to make checking for them efficient"); +static_assert( + ctrl_t::kEmpty < ctrl_t::kSentinel && ctrl_t::kDeleted < ctrl_t::kSentinel, + "ctrl_t::kEmpty and ctrl_t::kDeleted must be smaller than " + "ctrl_t::kSentinel to make the SIMD test of IsEmptyOrDeleted() efficient"); +static_assert( + ctrl_t::kSentinel == static_cast(-1), + "ctrl_t::kSentinel must be -1 to elide loading it from memory into SIMD " + "registers (pcmpeqd xmm, xmm)"); +static_assert(ctrl_t::kEmpty == static_cast(-128), + "ctrl_t::kEmpty must be -128 to make the SIMD check for its " + "existence efficient (psignb xmm, xmm)"); +static_assert( + (~static_cast(ctrl_t::kEmpty) & + ~static_cast(ctrl_t::kDeleted) & + static_cast(ctrl_t::kSentinel) & 0x7F) != 0, + "ctrl_t::kEmpty and ctrl_t::kDeleted must share an unset bit that is not " + "shared by ctrl_t::kSentinel to make the scalar test for " + "MaskEmptyOrDeleted() efficient"); +static_assert(ctrl_t::kDeleted == static_cast(-2), + "ctrl_t::kDeleted must be -2 to make the implementation of " + "ConvertSpecialToEmptyAndFullToDeleted efficient"); + +// Helpers for checking the state of a control byte. +inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } +inline bool IsFull(ctrl_t c) { + // Cast `c` to the underlying type instead of casting `0` to `ctrl_t` as `0` + // is not a value in the enum. Both ways are equivalent, but this way makes + // linters happier. + return static_cast>(c) >= 0; +} +inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } +inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } + +#ifdef ABSL_INTERNAL_HAVE_SSE2 +// Quick reference guide for intrinsics used below: +// +// * __m128i: An XMM (128-bit) word. +// +// * _mm_setzero_si128: Returns a zero vector. +// * _mm_set1_epi8: Returns a vector with the same i8 in each lane. +// +// * _mm_subs_epi8: Saturating-subtracts two i8 vectors. +// * _mm_and_si128: Ands two i128s together. +// * _mm_or_si128: Ors two i128s together. +// * _mm_andnot_si128: And-nots two i128s together. +// +// * _mm_cmpeq_epi8: Component-wise compares two i8 vectors for equality, +// filling each lane with 0x00 or 0xff. +// * _mm_cmpgt_epi8: Same as above, but using > rather than ==. +// +// * _mm_loadu_si128: Performs an unaligned load of an i128. +// * _mm_storeu_si128: Performs an unaligned store of an i128. +// +// * _mm_sign_epi8: Retains, negates, or zeroes each i8 lane of the first +// argument if the corresponding lane of the second +// argument is positive, negative, or zero, respectively. +// * _mm_movemask_epi8: Selects the sign bit out of each i8 lane and produces a +// bitmask consisting of those bits. +// * _mm_shuffle_epi8: Selects i8s from the first argument, using the low +// four bits of each i8 lane in the second argument as +// indices. + +// https://github.com/abseil/abseil-cpp/issues/209 +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 +// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char +// Work around this by using the portable implementation of Group +// when using -funsigned-char under GCC. +inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { +#if defined(__GNUC__) && !defined(__clang__) + if (std::is_unsigned::value) { + const __m128i mask = _mm_set1_epi8(0x80); + const __m128i diff = _mm_subs_epi8(b, a); + return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); + } +#endif + return _mm_cmpgt_epi8(a, b); +} + +struct GroupSse2Impl { + static constexpr size_t kWidth = 16; // the number of slots per group + using BitMaskType = BitMask; + using NonIterableBitMaskType = NonIterableBitMask; + + explicit GroupSse2Impl(const ctrl_t* pos) { + ctrl = _mm_loadu_si128(reinterpret_cast(pos)); + } + + // Returns a bitmask representing the positions of slots that match hash. + BitMaskType Match(h2_t hash) const { + auto match = _mm_set1_epi8(static_cast(hash)); + return BitMaskType( + static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); + } + + // Returns a bitmask representing the positions of empty slots. + NonIterableBitMaskType MaskEmpty() const { +#ifdef ABSL_INTERNAL_HAVE_SSSE3 + // This only works because ctrl_t::kEmpty is -128. + return NonIterableBitMaskType( + static_cast(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)))); +#else + auto match = _mm_set1_epi8(static_cast(ctrl_t::kEmpty)); + return NonIterableBitMaskType( + static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); +#endif + } + + // Returns a bitmask representing the positions of full slots. + // Note: for `is_small()` tables group may contain the "same" slot twice: + // original and mirrored. + BitMaskType MaskFull() const { + return BitMaskType(static_cast(_mm_movemask_epi8(ctrl) ^ 0xffff)); + } + + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + return BitMaskType(static_cast(_mm_movemask_epi8(ctrl))); + } + + // Returns a bitmask representing the positions of empty or deleted slots. + NonIterableBitMaskType MaskEmptyOrDeleted() const { + auto special = _mm_set1_epi8(static_cast(ctrl_t::kSentinel)); + return NonIterableBitMaskType(static_cast( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)))); + } + + // Returns the number of trailing empty or deleted elements in the group. + uint32_t CountLeadingEmptyOrDeleted() const { + auto special = _mm_set1_epi8(static_cast(ctrl_t::kSentinel)); + return TrailingZeros(static_cast( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1)); + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + auto msbs = _mm_set1_epi8(static_cast(-128)); + auto x126 = _mm_set1_epi8(126); +#ifdef ABSL_INTERNAL_HAVE_SSSE3 + auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); +#else + auto zero = _mm_setzero_si128(); + auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); + auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); +#endif + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); + } + + __m128i ctrl; +}; +#endif // ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 + +#if defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) +struct GroupAArch64Impl { + static constexpr size_t kWidth = 8; + using BitMaskType = BitMask; + using NonIterableBitMaskType = + NonIterableBitMask; + + explicit GroupAArch64Impl(const ctrl_t* pos) { + ctrl = vld1_u8(reinterpret_cast(pos)); + } + + auto Match(h2_t hash) const { + uint8x8_t dup = vdup_n_u8(hash); + auto mask = vceq_u8(ctrl, dup); + return BitMaskType(vget_lane_u64(vreinterpret_u64_u8(mask), 0)); + } + + auto MaskEmpty() const { + uint64_t mask = + vget_lane_u64(vreinterpret_u64_u8(vceq_s8( + vdup_n_s8(static_cast(ctrl_t::kEmpty)), + vreinterpret_s8_u8(ctrl))), + 0); + return NonIterableBitMaskType(mask); + } + + // Returns a bitmask representing the positions of full slots. + // Note: for `is_small()` tables group may contain the "same" slot twice: + // original and mirrored. + auto MaskFull() const { + uint64_t mask = vget_lane_u64( + vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl), + vdup_n_s8(static_cast(0)))), + 0); + return BitMaskType(mask); + } + + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + uint64_t mask = vget_lane_u64( + vreinterpret_u64_u8(vclt_s8(vreinterpret_s8_u8(ctrl), + vdup_n_s8(static_cast(0)))), + 0); + return BitMaskType(mask); + } + + auto MaskEmptyOrDeleted() const { + uint64_t mask = + vget_lane_u64(vreinterpret_u64_u8(vcgt_s8( + vdup_n_s8(static_cast(ctrl_t::kSentinel)), + vreinterpret_s8_u8(ctrl))), + 0); + return NonIterableBitMaskType(mask); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + uint64_t mask = + vget_lane_u64(vreinterpret_u64_u8(vcle_s8( + vdup_n_s8(static_cast(ctrl_t::kSentinel)), + vreinterpret_s8_u8(ctrl))), + 0); + // Similar to MaskEmptyorDeleted() but we invert the logic to invert the + // produced bitfield. We then count number of trailing zeros. + // Clang and GCC optimize countr_zero to rbit+clz without any check for 0, + // so we should be fine. + return static_cast(countr_zero(mask)) >> 3; + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0); + constexpr uint64_t slsbs = 0x0202020202020202ULL; + constexpr uint64_t midbs = 0x7e7e7e7e7e7e7e7eULL; + auto x = slsbs & (mask >> 6); + auto res = (x + midbs) | kMsbs8Bytes; + little_endian::Store64(dst, res); + } + + uint8x8_t ctrl; +}; +#endif // ABSL_INTERNAL_HAVE_ARM_NEON && ABSL_IS_LITTLE_ENDIAN + +struct GroupPortableImpl { + static constexpr size_t kWidth = 8; + using BitMaskType = BitMask; + using NonIterableBitMaskType = + NonIterableBitMask; + + explicit GroupPortableImpl(const ctrl_t* pos) + : ctrl(little_endian::Load64(pos)) {} + + BitMaskType Match(h2_t hash) const { + // For the technique, see: + // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + // (Determine if a word has a byte equal to n). + // + // Caveat: there are false positives but: + // - they only occur if there is a real match + // - they never occur on ctrl_t::kEmpty, ctrl_t::kDeleted, ctrl_t::kSentinel + // - they will be handled gracefully by subsequent checks in code + // + // Example: + // v = 0x1716151413121110 + // hash = 0x12 + // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl ^ (lsbs * hash); + return BitMaskType((x - lsbs) & ~x & kMsbs8Bytes); + } + + auto MaskEmpty() const { + return NonIterableBitMaskType((ctrl & ~(ctrl << 6)) & kMsbs8Bytes); + } + + // Returns a bitmask representing the positions of full slots. + // Note: for `is_small()` tables group may contain the "same" slot twice: + // original and mirrored. + auto MaskFull() const { + return BitMaskType((ctrl ^ kMsbs8Bytes) & kMsbs8Bytes); + } + + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { return BitMaskType(ctrl & kMsbs8Bytes); } + + auto MaskEmptyOrDeleted() const { + return NonIterableBitMaskType((ctrl & ~(ctrl << 7)) & kMsbs8Bytes); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + // ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and + // kDeleted. We lower all other bits and count number of trailing zeros. + constexpr uint64_t bits = 0x0101010101010101ULL; + return static_cast(countr_zero((ctrl | ~(ctrl >> 7)) & bits) >> + 3); + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl & kMsbs8Bytes; + auto res = (~x + (x >> 7)) & ~lsbs; + little_endian::Store64(dst, res); + } + + uint64_t ctrl; +}; + +#ifdef ABSL_INTERNAL_HAVE_SSE2 +using Group = GroupSse2Impl; +using GroupFullEmptyOrDeleted = GroupSse2Impl; +#elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) +using Group = GroupAArch64Impl; +// For Aarch64, we use the portable implementation for counting and masking +// full, empty or deleted group elements. This is to avoid the latency of moving +// between data GPRs and Neon registers when it does not provide a benefit. +// Using Neon is profitable when we call Match(), but is not when we don't, +// which is the case when we do *EmptyOrDeleted and MaskFull operations. +// It is difficult to make a similar approach beneficial on other architectures +// such as x86 since they have much lower GPR <-> vector register transfer +// latency and 16-wide Groups. +using GroupFullEmptyOrDeleted = GroupPortableImpl; +#else +using Group = GroupPortableImpl; +using GroupFullEmptyOrDeleted = GroupPortableImpl; +#endif + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#undef ABSL_SWISSTABLE_ASSERT + +#endif // ABSL_CONTAINER_INTERNAL_HASHTABLE_CONTROL_BYTES_H_ diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index fd21d96..c0fce87 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc @@ -42,10 +42,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr int HashtablezInfo::kMaxStackDepth; -#endif - namespace { ABSL_CONST_INIT std::atomic g_hashtablez_enabled{ false @@ -126,6 +122,26 @@ static bool ShouldForceSampling() { return state == kForce; } +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +HashtablezInfoHandle ForcedTrySample(size_t inline_element_size, + size_t key_size, size_t value_size, + uint16_t soo_capacity) { + return HashtablezInfoHandle(SampleSlow(global_next_sample, + inline_element_size, key_size, + value_size, soo_capacity)); +} +void TestOnlyRefreshSamplingStateForCurrentThread() { + global_next_sample.next_sample = + g_hashtablez_sample_parameter.load(std::memory_order_relaxed); + global_next_sample.sample_stride = global_next_sample.next_sample; +} +#else +HashtablezInfoHandle ForcedTrySample(size_t, size_t, size_t, uint16_t) { + return HashtablezInfoHandle{nullptr}; +} +void TestOnlyRefreshSamplingStateForCurrentThread() {} +#endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE + HashtablezInfo* SampleSlow(SamplingState& next_sample, size_t inline_element_size, size_t key_size, size_t value_size, uint16_t soo_capacity) { diff --git a/absl/container/internal/hashtablez_sampler.h b/absl/container/internal/hashtablez_sampler.h index d74acf8..305dc85 100644 --- a/absl/container/internal/hashtablez_sampler.h +++ b/absl/container/internal/hashtablez_sampler.h @@ -219,22 +219,41 @@ class HashtablezInfoHandle { extern ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample; #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -// Returns a sampling handle. -inline HashtablezInfoHandle Sample( - ABSL_ATTRIBUTE_UNUSED size_t inline_element_size, - ABSL_ATTRIBUTE_UNUSED size_t key_size, - ABSL_ATTRIBUTE_UNUSED size_t value_size, - ABSL_ATTRIBUTE_UNUSED uint16_t soo_capacity) { +// Returns true if the next table should be sampled. +// This function updates the global state. +// If the function returns true, actual sampling should be done by calling +// ForcedTrySample(). +inline bool ShouldSampleNextTable() { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) if (ABSL_PREDICT_TRUE(--global_next_sample.next_sample > 0)) { - return HashtablezInfoHandle(nullptr); + return false; } - return HashtablezInfoHandle(SampleSlow(global_next_sample, - inline_element_size, key_size, - value_size, soo_capacity)); + return true; #else - return HashtablezInfoHandle(nullptr); -#endif // !ABSL_PER_THREAD_TLS + return false; +#endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE +} + +// Returns a sampling handle. +// Must be called only if HashSetShouldBeSampled() returned true. +// Returned handle still can be unsampled if sampling is not possible. +HashtablezInfoHandle ForcedTrySample(size_t inline_element_size, + size_t key_size, size_t value_size, + uint16_t soo_capacity); + +// In case sampling needs to be disabled and re-enabled in tests, this function +// can be used to reset the sampling state for the current thread. +// It is useful to avoid sampling attempts and sampling delays in tests. +void TestOnlyRefreshSamplingStateForCurrentThread(); + +// Returns a sampling handle. +inline HashtablezInfoHandle Sample(size_t inline_element_size, size_t key_size, + size_t value_size, uint16_t soo_capacity) { + if (ABSL_PREDICT_TRUE(!ShouldSampleNextTable())) { + return HashtablezInfoHandle(nullptr); + } + return ForcedTrySample(inline_element_size, key_size, value_size, + soo_capacity); } using HashtablezSampler = diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index 0bd0a1c..b0d3f07 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -73,11 +73,6 @@ using ConstReverseIterator = typename std::reverse_iterator>; template using MoveIterator = typename std::move_iterator>; -template -using IsAtLeastForwardIterator = std::is_convertible< - typename std::iterator_traits::iterator_category, - std::forward_iterator_tag>; - template using IsMoveAssignOk = std::is_move_assignable>; template @@ -234,7 +229,7 @@ class AllocationTransaction { return result.data; } - ABSL_MUST_USE_RESULT Allocation Release() && { + [[nodiscard]] Allocation Release() && { Allocation result = {GetData(), GetCapacity()}; Reset(); return result; @@ -548,7 +543,7 @@ class Storage { (std::max)(N, sizeof(Allocated) / sizeof(ValueType)); struct Inlined { - alignas(ValueType) char inlined_data[sizeof( + alignas(ValueType) unsigned char inlined_data[sizeof( ValueType[kOptimalInlinedSize])]; }; diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index f8b425c..58c8d4f 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -192,7 +192,6 @@ #include #include -#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/debugging/internal/demangle.h" #include "absl/meta/type_traits.h" @@ -316,9 +315,6 @@ std::string TypeName() { } // namespace adl_barrier -template -using EnableIf = typename std::enable_if::type; - // Can `T` be a template argument of `Layout`? template using IsLegalElementType = std::integral_constant< @@ -418,17 +414,16 @@ class LayoutImpl< // assert(x.Offset<1>() == 16); // The doubles starts from 16. // // Requires: `N <= NumSizes && N < sizeof...(Ts)`. - template = 0> - constexpr size_t Offset() const { - return 0; - } - - template = 0> + template constexpr size_t Offset() const { - static_assert(N < NumOffsets, "Index out of bounds"); - return adl_barrier::Align( - Offset() + SizeOf>::value * Size(), - ElementAlignment::value); + if constexpr (N == 0) { + return 0; + } else { + static_assert(N < NumOffsets, "Index out of bounds"); + return adl_barrier::Align( + Offset() + SizeOf>::value * Size(), + ElementAlignment::value); + } } // Offset in bytes of the array with the specified element type. There must @@ -457,15 +452,14 @@ class LayoutImpl< // assert(x.Size<1>() == 4); // // Requires: `N < NumSizes`. - template = 0> - constexpr size_t Size() const { - return kStaticSizes[N]; - } - - template = NumStaticSizes)> = 0> + template constexpr size_t Size() const { - static_assert(N < NumSizes, "Index out of bounds"); - return size_[N - NumStaticSizes]; + if constexpr (N < NumStaticSizes) { + return kStaticSizes[N]; + } else { + static_assert(N < NumSizes, "Index out of bounds"); + return size_[N - NumStaticSizes]; + } } // The number of elements in the array with the specified element type. @@ -596,10 +590,10 @@ class LayoutImpl< // // Requires: `p` is aligned to `Alignment()`. // - // Note: We mark the parameter as unused because GCC detects it is not used - // when `SizeSeq` is empty [-Werror=unused-but-set-parameter]. + // Note: We mark the parameter as maybe_unused because GCC detects it is not + // used when `SizeSeq` is empty [-Werror=unused-but-set-parameter]. template - auto Slices(ABSL_ATTRIBUTE_UNUSED Char* p) const { + auto Slices([[maybe_unused]] Char* p) const { return std::tuple>>...>( Slice(p)...); } @@ -624,15 +618,13 @@ class LayoutImpl< // `Char` must be `[const] [signed|unsigned] char`. // // Requires: `p` is aligned to `Alignment()`. - template = 0> - void PoisonPadding(const Char* p) const { - Pointer<0>(p); // verify the requirements on `Char` and `p` - } - - template = 0> + template void PoisonPadding(const Char* p) const { - static_assert(N < NumOffsets, "Index out of bounds"); - (void)p; + if constexpr (N == 0) { + Pointer<0>(p); // verify the requirements on `Char` and `p` + } else { + static_assert(N < NumOffsets, "Index out of bounds"); + (void)p; #ifdef ABSL_HAVE_ADDRESS_SANITIZER PoisonPadding(p); // The `if` is an optimization. It doesn't affect the observable behaviour. @@ -642,6 +634,7 @@ class LayoutImpl< ASAN_POISON_MEMORY_REGION(p + start, Offset() - start); } #endif + } } // Human-readable description of the memory layout. Useful for debugging. @@ -692,15 +685,6 @@ class LayoutImpl< size_t size_[NumRuntimeSizes > 0 ? NumRuntimeSizes : 1]; }; -// Defining a constexpr static class member variable is redundant and deprecated -// in C++17, but required in C++14. -template -constexpr std::array LayoutImpl< - std::tuple, absl::index_sequence, - absl::index_sequence, absl::index_sequence, - absl::index_sequence>::kStaticSizes; - template using LayoutType = LayoutImpl< std::tuple, StaticSizeSeq, diff --git a/absl/container/internal/raw_hash_map.h b/absl/container/internal/raw_hash_map.h index 464bf23..b42a4f2 100644 --- a/absl/container/internal/raw_hash_map.h +++ b/absl/container/internal/raw_hash_map.h @@ -22,8 +22,10 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/throw_delegate.h" +#include "absl/container/internal/common_policy_traits.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -43,14 +45,39 @@ class raw_hash_map : public raw_hash_set { using MappedConstReference = decltype(P::value( std::addressof(std::declval()))); - using KeyArgImpl = - KeyArg::value && IsTransparent::value>; + template + using key_arg = + typename KeyArg::value && IsTransparent::value>:: + template type; + + // NOTE: The mess here is to shorten the code for the (very repetitive) + // function overloads, and to allow the lifetime-bound overloads to dispatch + // to the non-lifetime-bound overloads, to ensure there is a single source of + // truth for each overload set. + // + // Enabled if an assignment from the given type would require the + // source object to remain alive for the life of the element. + // + // TODO(b/402804213): Remove these traits and simplify the overloads whenever + // we have a better mechanism available to handle lifetime analysis. + template + using LifetimeBoundK = HasValue< + Value, std::conditional_t::value, + std::false_type, + type_traits_internal::IsLifetimeBoundAssignment< + typename Policy::key_type, K>>>; + template + using LifetimeBoundV = + HasValue>; + template + using LifetimeBoundKV = + absl::conjunction>, + LifetimeBoundV>; public: using key_type = typename Policy::key_type; using mapped_type = typename Policy::mapped_type; - template - using key_arg = typename KeyArgImpl::template type; static_assert(!std::is_reference::value, ""); @@ -71,87 +98,175 @@ class raw_hash_map : public raw_hash_set { // union { int n : 1; }; // flat_hash_map m; // m.insert_or_assign(n, n); - template - std::pair insert_or_assign(key_arg&& k, V&& v) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(std::forward(k), std::forward(v)); - } - - template - std::pair insert_or_assign(key_arg&& k, const V& v) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(std::forward(k), v); - } - - template - std::pair insert_or_assign(const key_arg& k, V&& v) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(k, std::forward(v)); - } - - template - std::pair insert_or_assign(const key_arg& k, const V& v) - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign_impl(k, v); - } - - template - iterator insert_or_assign(const_iterator, key_arg&& k, - V&& v) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign(std::forward(k), std::forward(v)).first; - } - - template - iterator insert_or_assign(const_iterator, key_arg&& k, - const V& v) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign(std::forward(k), v).first; - } - - template - iterator insert_or_assign(const_iterator, const key_arg& k, - V&& v) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign(k, std::forward(v)).first; - } - - template - iterator insert_or_assign(const_iterator, const key_arg& k, - const V& v) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert_or_assign(k, v).first; - } + // + // TODO(b/402804213): Remove these macros whenever we have a better mechanism + // available to handle lifetime analysis. +#define ABSL_INTERNAL_X(Func, Callee, KQual, VQual, KValue, VValue, Tail, ...) \ + template < \ + typename K = key_type, class V = mapped_type, \ + ABSL_INTERNAL_IF_##KValue##_NOR_##VValue( \ + int = (EnableIf::AddPtr, \ + IfRRef::AddPtr>>()), \ + ABSL_INTERNAL_SINGLE_ARG( \ + int &..., \ + decltype(EnableIf>()) = \ + 0))> \ + decltype(auto) Func( \ + __VA_ARGS__ key_arg KQual k ABSL_INTERNAL_IF_##KValue( \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)), \ + V VQual v ABSL_INTERNAL_IF_##VValue(ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY( \ + this))) ABSL_ATTRIBUTE_LIFETIME_BOUND { \ + return ABSL_INTERNAL_IF_##KValue##_OR_##VValue( \ + (this->template Func), Callee)( \ + std::forward(k), std::forward(v)) Tail; \ + } \ + static_assert(true, "This is to force a semicolon.") + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + false, false, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + false, true, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + true, false, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + true, true, ABSL_INTERNAL_SINGLE_ARG()); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, false, + false, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, false, + true, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, true, + false, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, true, + true, ABSL_INTERNAL_SINGLE_ARG()); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, false, + false, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, false, + true, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, true, + false, ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, true, + true, ABSL_INTERNAL_SINGLE_ARG()); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, false, false, + ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, false, true, + ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, true, false, + ABSL_INTERNAL_SINGLE_ARG()); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, true, true, + ABSL_INTERNAL_SINGLE_ARG()); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + false, false, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + false, true, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + true, false, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, const &, + true, true, .first, const_iterator ABSL_INTERNAL_COMMA); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, false, + false, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, false, + true, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, true, + false, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, const &, &&, true, + true, .first, const_iterator ABSL_INTERNAL_COMMA); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, false, + false, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, false, + true, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, true, + false, .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, const &, true, + true, .first, const_iterator ABSL_INTERNAL_COMMA); + + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, false, false, + .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, false, true, + .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, true, false, + .first, const_iterator ABSL_INTERNAL_COMMA); + ABSL_INTERNAL_X(insert_or_assign, insert_or_assign_impl, &&, &&, true, true, + .first, const_iterator ABSL_INTERNAL_COMMA); +#undef ABSL_INTERNAL_X // All `try_emplace()` overloads make the same guarantees regarding rvalue // arguments as `std::unordered_map::try_emplace()`, namely that these // functions will not move from rvalue arguments if insertions do not happen. - template >(), + class... Args, typename std::enable_if< - !std::is_convertible::value, int>::type = 0, - K* = nullptr> - std::pair try_emplace(key_arg&& k, Args&&... args) + !std::is_convertible::value, int>::type = 0> + std::pair try_emplace(key_arg &&k, Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return try_emplace_impl(std::forward(k), std::forward(args)...); } template > = 0, typename std::enable_if< !std::is_convertible::value, int>::type = 0> - std::pair try_emplace(const key_arg& k, Args&&... args) + std::pair try_emplace( + key_arg &&k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template try_emplace(std::forward(k), + std::forward(args)...); + } + + template >(), + class... Args, + typename std::enable_if< + !std::is_convertible::value, int>::type = 0> + std::pair try_emplace(const key_arg &k, Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return try_emplace_impl(k, std::forward(args)...); } + template > = 0, + typename std::enable_if< + !std::is_convertible::value, int>::type = 0> + std::pair try_emplace( + const key_arg &k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template try_emplace(k, std::forward(args)...); + } - template - iterator try_emplace(const_iterator, key_arg&& k, - Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { + template >(), + class... Args> + iterator try_emplace(const_iterator, key_arg &&k, + Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return try_emplace(std::forward(k), std::forward(args)...).first; } + template > = 0> + iterator try_emplace(const_iterator hint, + key_arg &&k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template try_emplace(hint, std::forward(k), + std::forward(args)...); + } - template - iterator try_emplace(const_iterator, const key_arg& k, - Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { + template >(), + class... Args> + iterator try_emplace(const_iterator, const key_arg &k, + Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return try_emplace(k, std::forward(args)...).first; } + template > = 0> + iterator try_emplace(const_iterator hint, + const key_arg &k + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + Args &&...args) ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template try_emplace(hint, std::forward(k), + std::forward(args)...); + } template MappedReference

at(const key_arg& key) ABSL_ATTRIBUTE_LIFETIME_BOUND { @@ -174,8 +289,9 @@ class raw_hash_map : public raw_hash_set { return Policy::value(&*it); } - template - MappedReference

operator[](key_arg&& key) + template >()> + MappedReference

operator[](key_arg &&key) ABSL_ATTRIBUTE_LIFETIME_BOUND { // It is safe to use unchecked_deref here because try_emplace // will always return an iterator pointing to a valid item in the table, @@ -183,15 +299,30 @@ class raw_hash_map : public raw_hash_set { return Policy::value( &this->unchecked_deref(try_emplace(std::forward(key)).first)); } + template > = 0> + MappedReference

operator[]( + key_arg &&key ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template operator[](std::forward(key)); + } - template - MappedReference

operator[](const key_arg& key) + template >()> + MappedReference

operator[](const key_arg &key) ABSL_ATTRIBUTE_LIFETIME_BOUND { // It is safe to use unchecked_deref here because try_emplace // will always return an iterator pointing to a valid item in the table, // since it inserts if nothing is found for the given key. return Policy::value(&this->unchecked_deref(try_emplace(key).first)); } + template > = 0> + MappedReference

operator[]( + const key_arg &key ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template operator[](key); + } private: template diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 8911aa3..339e662 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -27,9 +27,11 @@ #include "absl/base/internal/raw_logging.h" #include "absl/base/optimization.h" #include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hashtable_control_bytes.h" #include "absl/container/internal/hashtablez_sampler.h" +#include "absl/container/internal/raw_hash_set_resize_impl.h" +#include "absl/functional/function_ref.h" #include "absl/hash/hash.h" -#include "absl/numeric/bits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -65,21 +67,31 @@ ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[17] = { static_assert(NumControlBytes(SooCapacity()) <= 17, "kSooControl capacity too small"); -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr size_t Group::kWidth; +namespace { + +#ifdef ABSL_SWISSTABLE_ASSERT +#error ABSL_SWISSTABLE_ASSERT cannot be directly set +#else +// We use this macro for assertions that users may see when the table is in an +// invalid state that sanitizers may help diagnose. +#define ABSL_SWISSTABLE_ASSERT(CONDITION) \ + assert((CONDITION) && "Try enabling sanitizers.") #endif -namespace { +[[noreturn]] ABSL_ATTRIBUTE_NOINLINE void HashTableSizeOverflow() { + ABSL_RAW_LOG(FATAL, "Hash table size overflow"); +} + +void ValidateMaxSize(size_t size, size_t slot_size) { + if (IsAboveValidSize(size, slot_size)) { + HashTableSizeOverflow(); + } +} // Returns "random" seed. inline size_t RandomSeed() { #ifdef ABSL_HAVE_THREAD_LOCAL static thread_local size_t counter = 0; - // On Linux kernels >= 5.4 the MSAN runtime has a false-positive when - // accessing thread local storage data from loaded libraries - // (https://github.com/google/sanitizers/issues/1265), for this reason counter - // needs to be annotated as initialized. - ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(&counter, sizeof(size_t)); size_t value = ++counter; #else // ABSL_HAVE_THREAD_LOCAL static std::atomic counter(0); @@ -88,24 +100,35 @@ inline size_t RandomSeed() { return value ^ static_cast(reinterpret_cast(&counter)); } -bool ShouldRehashForBugDetection(const ctrl_t* ctrl, size_t capacity) { +bool ShouldRehashForBugDetection(PerTableSeed seed, size_t capacity) { // Note: we can't use the abseil-random library because abseil-random // depends on swisstable. We want to return true with probability // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this, // we probe based on a random hash and see if the offset is less than // RehashProbabilityConstant(). - return probe(ctrl, capacity, absl::HashOf(RandomSeed())).offset() < + return probe(seed, capacity, absl::HashOf(RandomSeed())).offset() < RehashProbabilityConstant(); } // Find a non-deterministic hash for single group table. // Last two bits are used to find a position for a newly inserted element after // resize. -// This function is mixing all bits of hash and control pointer to maximize -// entropy. -size_t SingleGroupTableH1(size_t hash, ctrl_t* control) { - return static_cast(absl::popcount( - hash ^ static_cast(reinterpret_cast(control)))); +// This function basically using H2 last bits to save on shift operation. +size_t SingleGroupTableH1(size_t hash, PerTableSeed seed) { + return hash ^ seed.seed(); +} + +// Returns the address of the slot `i` iterations after `slot` assuming each +// slot has the specified size. +inline void* NextSlot(void* slot, size_t slot_size, size_t i = 1) { + return reinterpret_cast(reinterpret_cast(slot) + + slot_size * i); +} + +// Returns the address of the slot just before `slot` assuming each slot has the +// specified size. +inline void* PrevSlot(void* slot, size_t slot_size) { + return reinterpret_cast(reinterpret_cast(slot) - slot_size); } } // namespace @@ -121,42 +144,104 @@ GenerationType* EmptyGeneration() { } bool CommonFieldsGenerationInfoEnabled:: - should_rehash_for_bug_detection_on_insert(const ctrl_t* ctrl, + should_rehash_for_bug_detection_on_insert(PerTableSeed seed, size_t capacity) const { if (reserved_growth_ == kReservedGrowthJustRanOut) return true; if (reserved_growth_ > 0) return false; - return ShouldRehashForBugDetection(ctrl, capacity); + return ShouldRehashForBugDetection(seed, capacity); } bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move( - const ctrl_t* ctrl, size_t capacity) const { - return ShouldRehashForBugDetection(ctrl, capacity); + PerTableSeed seed, size_t capacity) const { + return ShouldRehashForBugDetection(seed, capacity); +} + +namespace { + +FindInfo find_first_non_full_from_h1(const ctrl_t* ctrl, size_t h1, + size_t capacity) { + auto seq = probe(h1, capacity); + if (IsEmptyOrDeleted(ctrl[seq.offset()])) { + return {seq.offset(), /*probe_length=*/0}; + } + while (true) { + GroupFullEmptyOrDeleted g{ctrl + seq.offset()}; + auto mask = g.MaskEmptyOrDeleted(); + if (mask) { + return {seq.offset(mask.LowestBitSet()), seq.index()}; + } + seq.next(); + ABSL_SWISSTABLE_ASSERT(seq.index() <= capacity && "full table!"); + } } -bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, - const ctrl_t* ctrl) { - // To avoid problems with weak hashes and single bit tests, we use % 13. - // TODO(kfm,sbenza): revisit after we do unconditional mixing - return !is_small(capacity) && (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; +// Whether a table is "small". A small table fits entirely into a probing +// group, i.e., has a capacity < `Group::kWidth`. +// +// In small mode we are able to use the whole capacity. The extra control +// bytes give us at least one "empty" control byte to stop the iteration. +// This is important to make 1 a valid capacity. +// +// In small mode only the first `capacity` control bytes after the sentinel +// are valid. The rest contain dummy ctrl_t::kEmpty values that do not +// represent a real slot. +constexpr bool is_small(size_t capacity) { + return capacity < Group::kWidth - 1; } -size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, - CommonFields& common) { - assert(common.capacity() == NextCapacity(SooCapacity())); - // After resize from capacity 1 to 3, we always have exactly the slot with - // index 1 occupied, so we need to insert either at index 0 or index 2. - assert(HashSetResizeHelper::SooSlotIndex() == 1); - PrepareInsertCommon(common); - const size_t offset = SingleGroupTableH1(hash, common.control()) & 2; - common.growth_info().OverwriteEmptyAsFull(); - SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size); - common.infoz().RecordInsert(hash, /*distance_from_desired=*/0); - return offset; +template +void IterateOverFullSlotsImpl(const CommonFields& c, size_t slot_size, Fn cb) { + const size_t cap = c.capacity(); + const ctrl_t* ctrl = c.control(); + void* slot = c.slot_array(); + if (is_small(cap)) { + // Mirrored/cloned control bytes in small table are also located in the + // first group (starting from position 0). We are taking group from position + // `capacity` in order to avoid duplicates. + + // Small tables capacity fits into portable group, where + // GroupPortableImpl::MaskFull is more efficient for the + // capacity <= GroupPortableImpl::kWidth. + ABSL_SWISSTABLE_ASSERT(cap <= GroupPortableImpl::kWidth && + "unexpectedly large small capacity"); + static_assert(Group::kWidth >= GroupPortableImpl::kWidth, + "unexpected group width"); + // Group starts from kSentinel slot, so indices in the mask will + // be increased by 1. + const auto mask = GroupPortableImpl(ctrl + cap).MaskFull(); + --ctrl; + slot = PrevSlot(slot, slot_size); + for (uint32_t i : mask) { + cb(ctrl + i, SlotAddress(slot, i, slot_size)); + } + return; + } + size_t remaining = c.size(); + ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining; + while (remaining != 0) { + for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { + ABSL_SWISSTABLE_ASSERT(IsFull(ctrl[i]) && + "hash table was modified unexpectedly"); + cb(ctrl + i, SlotAddress(slot, i, slot_size)); + --remaining; + } + ctrl += Group::kWidth; + slot = NextSlot(slot, slot_size, Group::kWidth); + ABSL_SWISSTABLE_ASSERT( + (remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && + "hash table was modified unexpectedly"); + } + // NOTE: erasure of the current element is allowed in callback for + // absl::erase_if specialization. So we use `>=`. + ABSL_SWISSTABLE_ASSERT(original_size_for_assert >= c.size() && + "hash table was modified unexpectedly"); } +} // namespace + void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { - assert(ctrl[capacity] == ctrl_t::kSentinel); - assert(IsValidCapacity(capacity)); + ABSL_SWISSTABLE_ASSERT(ctrl[capacity] == ctrl_t::kSentinel); + ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); for (ctrl_t* pos = ctrl; pos < ctrl + capacity; pos += Group::kWidth) { Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); } @@ -164,26 +249,25 @@ void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { std::memcpy(ctrl + capacity + 1, ctrl, NumClonedBytes()); ctrl[capacity] = ctrl_t::kSentinel; } -// Extern template instantiation for inline function. -template FindInfo find_first_non_full(const CommonFields&, size_t); -FindInfo find_first_non_full_outofline(const CommonFields& common, - size_t hash) { - return find_first_non_full(common, hash); +FindInfo find_first_non_full(const CommonFields& common, size_t hash) { + return find_first_non_full_from_h1(common.control(), H1(hash, common.seed()), + common.capacity()); +} + +void IterateOverFullSlots(const CommonFields& c, size_t slot_size, + absl::FunctionRef cb) { + IterateOverFullSlotsImpl(c, slot_size, cb); } namespace { -// Returns the address of the slot just after slot assuming each slot has the -// specified size. -static inline void* NextSlot(void* slot, size_t slot_size) { - return reinterpret_cast(reinterpret_cast(slot) + slot_size); +void ResetGrowthLeft(GrowthInfo& growth_info, size_t capacity, size_t size) { + growth_info.InitGrowthLeftNoDeleted(CapacityToGrowth(capacity) - size); } -// Returns the address of the slot just before slot assuming each slot has the -// specified size. -static inline void* PrevSlot(void* slot, size_t slot_size) { - return reinterpret_cast(reinterpret_cast(slot) - slot_size); +void ResetGrowthLeft(CommonFields& common) { + ResetGrowthLeft(common.growth_info(), common.capacity(), common.size()); } // Finds guaranteed to exists empty slot from the given position. @@ -196,17 +280,34 @@ size_t FindEmptySlot(size_t start, size_t end, const ctrl_t* ctrl) { return i; } } - assert(false && "no empty slot"); - return ~size_t{}; + ABSL_UNREACHABLE(); } -void DropDeletesWithoutResize(CommonFields& common, - const PolicyFunctions& policy) { +// Finds guaranteed to exist full slot starting from the given position. +// NOTE: this function is only triggered for rehash(0), when we need to +// go back to SOO state, so we keep it simple. +size_t FindFirstFullSlot(size_t start, size_t end, const ctrl_t* ctrl) { + for (size_t i = start; i < end; ++i) { + if (IsFull(ctrl[i])) { + return i; + } + } + ABSL_UNREACHABLE(); +} + +void PrepareInsertCommon(CommonFields& common) { + common.increment_size(); + common.maybe_increment_generation_on_insert(); +} + +size_t DropDeletesWithoutResizeAndPrepareInsert( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_hash) { void* set = &common; void* slot_array = common.slot_array(); const size_t capacity = common.capacity(); - assert(IsValidCapacity(capacity)); - assert(!is_small(capacity)); + ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); + ABSL_SWISSTABLE_ASSERT(!is_single_group(capacity)); // Algorithm: // - mark all DELETED slots as EMPTY // - mark all FULL slots as DELETED @@ -227,7 +328,7 @@ void DropDeletesWithoutResize(CommonFields& common, ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity); const void* hash_fn = policy.hash_fn(common); auto hasher = policy.hash_slot; - auto transfer = policy.transfer; + auto transfer_n = policy.transfer_n; const size_t slot_size = policy.slot_size; size_t total_probe_length = 0; @@ -240,7 +341,7 @@ void DropDeletesWithoutResize(CommonFields& common, for (size_t i = 0; i != capacity; ++i, slot_ptr = NextSlot(slot_ptr, slot_size)) { - assert(slot_ptr == SlotAddress(slot_array, i, slot_size)); + ABSL_SWISSTABLE_ASSERT(slot_ptr == SlotAddress(slot_array, i, slot_size)); if (IsEmpty(ctrl[i])) { tmp_space_id = i; continue; @@ -255,13 +356,14 @@ void DropDeletesWithoutResize(CommonFields& common, // If they do, we don't need to move the object as it falls already in the // best probe we can. const size_t probe_offset = probe(common, hash).offset(); + const h2_t h2 = H2(hash); const auto probe_index = [probe_offset, capacity](size_t pos) { return ((pos - probe_offset) & capacity) / Group::kWidth; }; // Element doesn't move. if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - SetCtrl(common, i, H2(hash), slot_size); + SetCtrlInLargeTable(common, i, h2, slot_size); continue; } @@ -270,14 +372,14 @@ void DropDeletesWithoutResize(CommonFields& common, // Transfer element to the empty spot. // SetCtrl poisons/unpoisons the slots so we have to call it at the // right time. - SetCtrl(common, new_i, H2(hash), slot_size); - (*transfer)(set, new_slot_ptr, slot_ptr); - SetCtrl(common, i, ctrl_t::kEmpty, slot_size); + SetCtrlInLargeTable(common, new_i, h2, slot_size); + (*transfer_n)(set, new_slot_ptr, slot_ptr, 1); + SetCtrlInLargeTable(common, i, ctrl_t::kEmpty, slot_size); // Initialize or change empty space id. tmp_space_id = i; } else { - assert(IsDeleted(ctrl[new_i])); - SetCtrl(common, new_i, H2(hash), slot_size); + ABSL_SWISSTABLE_ASSERT(IsDeleted(ctrl[new_i])); + SetCtrlInLargeTable(common, new_i, h2, slot_size); // Until we are done rehashing, DELETED marks previously FULL slots. if (tmp_space_id == kUnknownId) { @@ -287,9 +389,9 @@ void DropDeletesWithoutResize(CommonFields& common, SanitizerUnpoisonMemoryRegion(tmp_space, slot_size); // Swap i and new_i elements. - (*transfer)(set, tmp_space, new_slot_ptr); - (*transfer)(set, new_slot_ptr, slot_ptr); - (*transfer)(set, slot_ptr, tmp_space); + (*transfer_n)(set, tmp_space, new_slot_ptr, 1); + (*transfer_n)(set, new_slot_ptr, slot_ptr, 1); + (*transfer_n)(set, slot_ptr, tmp_space, 1); SanitizerPoisonMemoryRegion(tmp_space, slot_size); @@ -298,8 +400,14 @@ void DropDeletesWithoutResize(CommonFields& common, slot_ptr = PrevSlot(slot_ptr, slot_size); } } + // Prepare insert for the new element. + PrepareInsertCommon(common); ResetGrowthLeft(common); + FindInfo find_info = find_first_non_full(common, new_hash); + SetCtrlInLargeTable(common, find_info.offset, H2(new_hash), slot_size); + common.infoz().RecordInsert(new_hash, find_info.probe_length); common.infoz().RecordRehash(total_probe_length); + return find_info.offset; } static bool WasNeverFull(CommonFields& c, size_t index) { @@ -319,10 +427,126 @@ static bool WasNeverFull(CommonFields& c, size_t index) { Group::kWidth; } +// Updates the control bytes to indicate a completely empty table such that all +// control bytes are kEmpty except for the kSentinel byte. +void ResetCtrl(CommonFields& common, size_t slot_size) { + const size_t capacity = common.capacity(); + ctrl_t* ctrl = common.control(); + static constexpr size_t kTwoGroupCapacity = 2 * Group::kWidth - 1; + if (ABSL_PREDICT_TRUE(capacity <= kTwoGroupCapacity)) { + std::memset(ctrl, static_cast(ctrl_t::kEmpty), Group::kWidth); + std::memset(ctrl + capacity, static_cast(ctrl_t::kEmpty), + Group::kWidth); + if (capacity == kTwoGroupCapacity) { + std::memset(ctrl + Group::kWidth, static_cast(ctrl_t::kEmpty), + Group::kWidth); + } + } else { + std::memset(ctrl, static_cast(ctrl_t::kEmpty), + capacity + 1 + NumClonedBytes()); + } + ctrl[capacity] = ctrl_t::kSentinel; + SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity); +} + +// Initializes control bytes for single element table. +// Capacity of the table must be 1. +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void InitializeSingleElementControlBytes( + uint64_t h2, ctrl_t* new_ctrl) { + static constexpr uint64_t kEmptyXorSentinel = + static_cast(ctrl_t::kEmpty) ^ + static_cast(ctrl_t::kSentinel); + static constexpr uint64_t kEmpty64 = static_cast(ctrl_t::kEmpty); + // The first 8 bytes, where present slot positions are replaced with 0. + static constexpr uint64_t kFirstCtrlBytesWithZeroes = + k8EmptyBytes ^ kEmpty64 ^ (kEmptyXorSentinel << 8) ^ (kEmpty64 << 16); + + // Fill the original 0th and mirrored 2nd bytes with the hash. + // Result will look like: + // HSHEEEEE + // Where H = h2, E = kEmpty, S = kSentinel. + const uint64_t first_ctrl_bytes = + (h2 | kFirstCtrlBytesWithZeroes) | (h2 << 16); + // Fill last bytes with kEmpty. + std::memset(new_ctrl + 1, static_cast(ctrl_t::kEmpty), Group::kWidth); + // Overwrite the first 3 bytes with HSH. Other bytes will not be changed. + absl::little_endian::Store64(new_ctrl, first_ctrl_bytes); +} + +// Initializes control bytes for growing after SOO to the next capacity. +// `soo_ctrl` is placed in the position `SooSlotIndex()`. +// `new_hash` is placed in the position `new_offset`. +// The table must be non-empty SOO. +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void +InitializeThreeElementsControlBytesAfterSoo(ctrl_t soo_ctrl, size_t new_hash, + size_t new_offset, + ctrl_t* new_ctrl) { + static constexpr size_t kNewCapacity = NextCapacity(SooCapacity()); + static_assert(kNewCapacity == 3); + static_assert(is_single_group(kNewCapacity)); + static_assert(SooSlotIndex() == 1); + ABSL_SWISSTABLE_ASSERT(new_offset == 0 || new_offset == 2); + + static constexpr uint64_t kEmptyXorSentinel = + static_cast(ctrl_t::kEmpty) ^ + static_cast(ctrl_t::kSentinel); + static constexpr uint64_t kEmpty64 = static_cast(ctrl_t::kEmpty); + static constexpr size_t kMirroredSooSlotIndex = + SooSlotIndex() + kNewCapacity + 1; + // The first 8 bytes, where SOO slot original and mirrored positions are + // replaced with 0. + // Result will look like: E0ESE0EE + static constexpr uint64_t kFirstCtrlBytesWithZeroes = + k8EmptyBytes ^ (kEmpty64 << (8 * SooSlotIndex())) ^ + (kEmptyXorSentinel << (8 * kNewCapacity)) ^ + (kEmpty64 << (8 * kMirroredSooSlotIndex)); + + const uint64_t soo_h2 = static_cast(soo_ctrl); + const uint64_t new_h2_xor_empty = static_cast( + H2(new_hash) ^ static_cast(ctrl_t::kEmpty)); + // Fill the original and mirrored bytes for SOO slot. + // Result will look like: + // EHESEHEE + // Where H = soo_h2, E = kEmpty, S = kSentinel. + uint64_t first_ctrl_bytes = + ((soo_h2 << (8 * SooSlotIndex())) | kFirstCtrlBytesWithZeroes) | + (soo_h2 << (8 * kMirroredSooSlotIndex)); + // Replace original and mirrored empty bytes for the new position. + // Result for new_offset 0 will look like: + // NHESNHEE + // Where H = soo_h2, N = H2(new_hash), E = kEmpty, S = kSentinel. + // Result for new_offset 2 will look like: + // EHNSEHNE + first_ctrl_bytes ^= (new_h2_xor_empty << (8 * new_offset)); + size_t new_mirrored_offset = new_offset + kNewCapacity + 1; + first_ctrl_bytes ^= (new_h2_xor_empty << (8 * new_mirrored_offset)); + + // Fill last bytes with kEmpty. + std::memset(new_ctrl + kNewCapacity, static_cast(ctrl_t::kEmpty), + Group::kWidth); + // Overwrite the first 8 bytes with first_ctrl_bytes. + absl::little_endian::Store64(new_ctrl, first_ctrl_bytes); + + // Example for group size 16: + // new_ctrl after 1st memset = ???EEEEEEEEEEEEEEEE + // new_offset 0: + // new_ctrl after 2nd store = NHESNHEEEEEEEEEEEEE + // new_offset 2: + // new_ctrl after 2nd store = EHNSEHNEEEEEEEEEEEE + + // Example for group size 8: + // new_ctrl after 1st memset = ???EEEEEEEE + // new_offset 0: + // new_ctrl after 2nd store = NHESNHEEEEE + // new_offset 2: + // new_ctrl after 2nd store = EHNSEHNEEEE +} + } // namespace void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { - assert(IsFull(c.control()[index]) && "erasing a dangling iterator"); + ABSL_SWISSTABLE_ASSERT(IsFull(c.control()[index]) && + "erasing a dangling iterator"); c.decrement_size(); c.infoz().RecordErase(); @@ -333,14 +557,15 @@ void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { } c.growth_info().OverwriteFullAsDeleted(); - SetCtrl(c, index, ctrl_t::kDeleted, slot_size); + SetCtrlInLargeTable(c, index, ctrl_t::kDeleted, slot_size); } -void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, +void ClearBackingArray(CommonFields& c, + const PolicyFunctions& __restrict policy, void* alloc, bool reuse, bool soo_enabled) { - c.set_size(0); if (reuse) { - assert(!soo_enabled || c.capacity() > SooCapacity()); + c.set_size_to_zero(); + ABSL_SWISSTABLE_ASSERT(!soo_enabled || c.capacity() > SooCapacity()); ResetCtrl(c, policy.slot_size); ResetGrowthLeft(c); c.infoz().RecordStorageChanged(0, c.capacity()); @@ -349,17 +574,224 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, // infoz. c.infoz().RecordClearedReservation(); c.infoz().RecordStorageChanged(0, soo_enabled ? SooCapacity() : 0); - (*policy.dealloc)(c, policy); + c.infoz().Unregister(); + (*policy.dealloc)(alloc, c.capacity(), c.control(), policy.slot_size, + policy.slot_align, c.has_infoz()); c = soo_enabled ? CommonFields{soo_tag_t{}} : CommonFields{non_soo_tag_t{}}; } } -void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( - ctrl_t* __restrict new_ctrl, size_t new_capacity) const { - assert(is_single_group(new_capacity)); +namespace { + +enum class ResizeNonSooMode { + kGuaranteedEmpty, + kGuaranteedAllocated, +}; + +// Iterates over full slots in old table, finds new positions for them and +// transfers the slots. +// This function is used for reserving or rehashing non-empty tables. +// This use case is rare so the function is type erased. +// Returns the total probe length. +size_t FindNewPositionsAndTransferSlots( + CommonFields& common, const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots, size_t old_capacity) { + void* new_slots = common.slot_array(); + const void* hash_fn = policy.hash_fn(common); + const size_t slot_size = policy.slot_size; + + const auto insert_slot = [&](void* slot) { + size_t hash = policy.hash_slot(hash_fn, slot); + auto target = find_first_non_full(common, hash); + SetCtrl(common, target.offset, H2(hash), slot_size); + policy.transfer_n(&common, SlotAddress(new_slots, target.offset, slot_size), + slot, 1); + return target.probe_length; + }; + size_t total_probe_length = 0; + for (size_t i = 0; i < old_capacity; ++i) { + if (IsFull(old_ctrl[i])) { + total_probe_length += insert_slot(old_slots); + } + old_slots = NextSlot(old_slots, slot_size); + } + return total_probe_length; +} + +template +void ResizeNonSooImpl(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t new_capacity, HashtablezInfoHandle infoz) { + ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity)); + ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity()); + + const size_t old_capacity = common.capacity(); + [[maybe_unused]] ctrl_t* old_ctrl = common.control(); + [[maybe_unused]] void* old_slots = common.slot_array(); + + const size_t slot_size = policy.slot_size; + const size_t slot_align = policy.slot_align; + const bool has_infoz = infoz.IsSampled(); + + common.set_capacity(new_capacity); + RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz); + void* alloc = policy.get_char_alloc(common); + char* mem = static_cast(policy.alloc(alloc, layout.alloc_size())); + const GenerationType old_generation = common.generation(); + common.set_generation_ptr( + reinterpret_cast(mem + layout.generation_offset())); + common.set_generation(NextGeneration(old_generation)); + + ctrl_t* new_ctrl = reinterpret_cast(mem + layout.control_offset()); + common.set_control(new_ctrl); + common.set_slots(mem + layout.slot_offset()); + + size_t total_probe_length = 0; + ResetCtrl(common, slot_size); + ABSL_SWISSTABLE_ASSERT(kMode != ResizeNonSooMode::kGuaranteedEmpty || + old_capacity == policy.soo_capacity()); + ABSL_SWISSTABLE_ASSERT(kMode != ResizeNonSooMode::kGuaranteedAllocated || + old_capacity > 0); + if constexpr (kMode == ResizeNonSooMode::kGuaranteedAllocated) { + total_probe_length = FindNewPositionsAndTransferSlots( + common, policy, old_ctrl, old_slots, old_capacity); + (*policy.dealloc)(alloc, old_capacity, old_ctrl, slot_size, slot_align, + has_infoz); + ResetGrowthLeft(GetGrowthInfoFromControl(new_ctrl), new_capacity, + common.size()); + } else { + GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted( + CapacityToGrowth(new_capacity)); + } + + if (has_infoz) { + common.set_has_infoz(); + infoz.RecordStorageChanged(common.size(), new_capacity); + infoz.RecordRehash(total_probe_length); + common.set_infoz(infoz); + } +} + +void ResizeEmptyNonAllocatedTableImpl(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t new_capacity, bool force_infoz) { + ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity)); + ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity()); + ABSL_SWISSTABLE_ASSERT(!force_infoz || policy.soo_enabled); + ABSL_SWISSTABLE_ASSERT(common.capacity() <= policy.soo_capacity()); + ABSL_SWISSTABLE_ASSERT(common.empty()); + const size_t slot_size = policy.slot_size; + HashtablezInfoHandle infoz; + const bool should_sample = + policy.is_hashtablez_eligible && (force_infoz || ShouldSampleNextTable()); + if (ABSL_PREDICT_FALSE(should_sample)) { + infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, + policy.soo_capacity()); + } + ResizeNonSooImpl(common, policy, + new_capacity, infoz); +} + +// If the table was SOO, initializes new control bytes and transfers slot. +// After transferring the slot, sets control and slots in CommonFields. +// It is rare to resize an SOO table with one element to a large size. +// Requires: `c` contains SOO data. +void InsertOldSooSlotAndInitializeControlBytes( + CommonFields& c, const PolicyFunctions& __restrict policy, size_t hash, + ctrl_t* new_ctrl, void* new_slots) { + ABSL_SWISSTABLE_ASSERT(c.size() == policy.soo_capacity()); + ABSL_SWISSTABLE_ASSERT(policy.soo_enabled); + size_t new_capacity = c.capacity(); + + c.generate_new_seed(); + size_t offset = probe(c.seed(), new_capacity, hash).offset(); + offset = offset == new_capacity ? 0 : offset; + SanitizerPoisonMemoryRegion(new_slots, policy.slot_size * new_capacity); + void* target_slot = SlotAddress(new_slots, offset, policy.slot_size); + SanitizerUnpoisonMemoryRegion(target_slot, policy.slot_size); + policy.transfer_n(&c, target_slot, c.soo_data(), 1); + c.set_control(new_ctrl); + c.set_slots(new_slots); + ResetCtrl(c, policy.slot_size); + SetCtrl(c, offset, H2(hash), policy.slot_size); +} + +enum class ResizeFullSooTableSamplingMode { + kNoSampling, + // Force sampling. If the table was still not sampled, do not resize. + kForceSampleNoResizeIfUnsampled, +}; + +void AssertSoo([[maybe_unused]] CommonFields& common, + [[maybe_unused]] const PolicyFunctions& policy) { + ABSL_SWISSTABLE_ASSERT(policy.soo_enabled); + ABSL_SWISSTABLE_ASSERT(common.capacity() == policy.soo_capacity()); +} +void AssertFullSoo([[maybe_unused]] CommonFields& common, + [[maybe_unused]] const PolicyFunctions& policy) { + AssertSoo(common, policy); + ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity()); +} + +void ResizeFullSooTable(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t new_capacity, + ResizeFullSooTableSamplingMode sampling_mode) { + AssertFullSoo(common, policy); + const size_t slot_size = policy.slot_size; + const size_t slot_align = policy.slot_align; + + HashtablezInfoHandle infoz; + if (sampling_mode == + ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled) { + if (ABSL_PREDICT_FALSE(policy.is_hashtablez_eligible)) { + infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, + policy.soo_capacity()); + } + + if (!infoz.IsSampled()) { + return; + } + } + + const bool has_infoz = infoz.IsSampled(); + + common.set_capacity(new_capacity); + + RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz); + void* alloc = policy.get_char_alloc(common); + char* mem = static_cast(policy.alloc(alloc, layout.alloc_size())); + const GenerationType old_generation = common.generation(); + common.set_generation_ptr( + reinterpret_cast(mem + layout.generation_offset())); + common.set_generation(NextGeneration(old_generation)); + + // We do not set control and slots in CommonFields yet to avoid overriding + // SOO data. + ctrl_t* new_ctrl = reinterpret_cast(mem + layout.control_offset()); + void* new_slots = mem + layout.slot_offset(); + + const size_t soo_slot_hash = + policy.hash_slot(policy.hash_fn(common), common.soo_data()); + + InsertOldSooSlotAndInitializeControlBytes(common, policy, soo_slot_hash, + new_ctrl, new_slots); + ResetGrowthLeft(common); + if (has_infoz) { + common.set_has_infoz(); + common.set_infoz(infoz); + infoz.RecordStorageChanged(common.size(), new_capacity); + } +} + +void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* __restrict old_ctrl, + size_t old_capacity, + ctrl_t* __restrict new_ctrl, + size_t new_capacity) { + ABSL_SWISSTABLE_ASSERT(is_single_group(new_capacity)); constexpr size_t kHalfWidth = Group::kWidth / 2; - ABSL_ASSUME(old_capacity_ < kHalfWidth); - ABSL_ASSUME(old_capacity_ > 0); + ABSL_ASSUME(old_capacity < kHalfWidth); + ABSL_ASSUME(old_capacity > 0); static_assert(Group::kWidth == 8 || Group::kWidth == 16, "Group size is not supported."); @@ -373,8 +805,7 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( // Example: // old_ctrl = 012S012EEEEEEEEE... // copied_bytes = S012EEEE - uint64_t copied_bytes = - absl::little_endian::Load64(old_ctrl() + old_capacity_); + uint64_t copied_bytes = absl::little_endian::Load64(old_ctrl + old_capacity); // We change the sentinel byte to kEmpty before storing to both the start of // the new_ctrl, and past the end of the new_ctrl later for the new cloned @@ -395,7 +826,8 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( if (Group::kWidth == 8) { // With group size 8, we can grow with two write operations. - assert(old_capacity_ < 8 && "old_capacity_ is too large for group size 8"); + ABSL_SWISSTABLE_ASSERT(old_capacity < 8 && + "old_capacity is too large for group size 8"); absl::little_endian::Store64(new_ctrl, copied_bytes); static constexpr uint64_t kSentinal64 = @@ -421,7 +853,7 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( return; } - assert(Group::kWidth == 16); + ABSL_SWISSTABLE_ASSERT(Group::kWidth == 16); // Fill the second half of the main control bytes with kEmpty. // For small capacity that may write into mirrored control bytes. @@ -463,7 +895,6 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( // >! // new_ctrl after 2nd store = E012EEESE012EEEEEEEEEEE - // Example for growth capacity 7->15: // old_ctrl = 0123456S0123456EEEEEEEE // new_ctrl at the end = E0123456EEEEEEESE0123456EEEEEEE @@ -478,58 +909,428 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( // new_ctrl after 2nd store = E0123456EEEEEEESE0123456EEEEEEE } -void HashSetResizeHelper::InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2, - size_t new_capacity) { - assert(is_single_group(new_capacity)); - std::memset(new_ctrl, static_cast(ctrl_t::kEmpty), - NumControlBytes(new_capacity)); - assert(HashSetResizeHelper::SooSlotIndex() == 1); - // This allows us to avoid branching on had_soo_slot_. - assert(had_soo_slot_ || h2 == ctrl_t::kEmpty); - new_ctrl[1] = new_ctrl[new_capacity + 2] = h2; +// Size of the buffer we allocate on stack for storing probed elements in +// GrowToNextCapacity algorithm. +constexpr size_t kProbedElementsBufferSize = 512; + +// Decodes information about probed elements from contiguous memory. +// Finds new position for each element and transfers it to the new slots. +// Returns the total probe length. +template +ABSL_ATTRIBUTE_NOINLINE size_t DecodeAndInsertImpl( + CommonFields& c, const PolicyFunctions& __restrict policy, + const ProbedItem* start, const ProbedItem* end, void* old_slots) { + const size_t new_capacity = c.capacity(); + + void* new_slots = c.slot_array(); + ctrl_t* new_ctrl = c.control(); + size_t total_probe_length = 0; + + const size_t slot_size = policy.slot_size; + auto transfer_n = policy.transfer_n; + + for (; start < end; ++start) { + const FindInfo target = find_first_non_full_from_h1( + new_ctrl, static_cast(start->h1), new_capacity); + total_probe_length += target.probe_length; + const size_t old_index = static_cast(start->source_offset); + const size_t new_i = target.offset; + ABSL_SWISSTABLE_ASSERT(old_index < new_capacity / 2); + ABSL_SWISSTABLE_ASSERT(new_i < new_capacity); + ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[new_i])); + void* src_slot = SlotAddress(old_slots, old_index, slot_size); + void* dst_slot = SlotAddress(new_slots, new_i, slot_size); + SanitizerUnpoisonMemoryRegion(dst_slot, slot_size); + transfer_n(&c, dst_slot, src_slot, 1); + SetCtrlInLargeTable(c, new_i, static_cast(start->h2), slot_size); + } + return total_probe_length; +} + +// Sentinel value for the start of marked elements. +// Signals that there are no marked elements. +constexpr size_t kNoMarkedElementsSentinel = ~size_t{}; + +// Process probed elements that did not fit into available buffers. +// We marked them in control bytes as kSentinel. +// Hash recomputation and full probing is done here. +// This use case should be extremely rare. +ABSL_ATTRIBUTE_NOINLINE size_t ProcessProbedMarkedElements( + CommonFields& c, const PolicyFunctions& __restrict policy, ctrl_t* old_ctrl, + void* old_slots, size_t start) { + size_t old_capacity = PreviousCapacity(c.capacity()); + const size_t slot_size = policy.slot_size; + void* new_slots = c.slot_array(); + size_t total_probe_length = 0; + const void* hash_fn = policy.hash_fn(c); + auto hash_slot = policy.hash_slot; + auto transfer_n = policy.transfer_n; + for (size_t old_index = start; old_index < old_capacity; ++old_index) { + if (old_ctrl[old_index] != ctrl_t::kSentinel) { + continue; + } + void* src_slot = SlotAddress(old_slots, old_index, slot_size); + const size_t hash = hash_slot(hash_fn, src_slot); + const FindInfo target = find_first_non_full(c, hash); + total_probe_length += target.probe_length; + const size_t new_i = target.offset; + void* dst_slot = SlotAddress(new_slots, new_i, slot_size); + SetCtrlInLargeTable(c, new_i, H2(hash), slot_size); + transfer_n(&c, dst_slot, src_slot, 1); + } + return total_probe_length; +} + +// The largest old capacity for which it is guaranteed that all probed elements +// fit in ProbedItemEncoder's local buffer. +// For such tables, `encode_probed_element` is trivial. +constexpr size_t kMaxLocalBufferOldCapacity = + kProbedElementsBufferSize / sizeof(ProbedItem4Bytes) - 1; +static_assert(IsValidCapacity(kMaxLocalBufferOldCapacity)); +constexpr size_t kMaxLocalBufferNewCapacity = + NextCapacity(kMaxLocalBufferOldCapacity); +static_assert(kMaxLocalBufferNewCapacity <= ProbedItem4Bytes::kMaxNewCapacity); +static_assert(NextCapacity(kMaxLocalBufferNewCapacity) <= + ProbedItem4Bytes::kMaxNewCapacity); + +// Initializes mirrored control bytes after +// transfer_unprobed_elements_to_next_capacity. +void InitializeMirroredControlBytes(ctrl_t* new_ctrl, size_t new_capacity) { + std::memcpy(new_ctrl + new_capacity, + // We own GrowthInfo just before control bytes. So it is ok + // to read one byte from it. + new_ctrl - 1, Group::kWidth); new_ctrl[new_capacity] = ctrl_t::kSentinel; } -void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots( - void* new_slots, size_t slot_size) const { - ABSL_ASSUME(old_capacity_ > 0); - SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_); - std::memcpy(SlotAddress(new_slots, 1, slot_size), old_slots(), - slot_size * old_capacity_); +// Encodes probed elements into available memory. +// At first, a local (on stack) buffer is used. The size of the buffer is +// kProbedElementsBufferSize bytes. +// When the local buffer is full, we switch to `control_` buffer. We are allowed +// to overwrite `control_` buffer till the `source_offset` byte. In case we have +// no space in `control_` buffer, we fallback to a naive algorithm for all the +// rest of the probed elements. We mark elements as kSentinel in control bytes +// and later process them fully. See ProcessMarkedElements for details. It +// should be extremely rare. +template +class ProbedItemEncoder { + public: + using ProbedItem = ProbedItemType; + explicit ProbedItemEncoder(ctrl_t* control) : control_(control) {} + + // Encode item into the best available location. + void EncodeItem(ProbedItem item) { + if (ABSL_PREDICT_FALSE(!kGuaranteedFitToBuffer && pos_ >= end_)) { + return ProcessEncodeWithOverflow(item); + } + ABSL_SWISSTABLE_ASSERT(pos_ < end_); + *pos_ = item; + ++pos_; + } + + // Decodes information about probed elements from all available sources. + // Finds new position for each element and transfers it to the new slots. + // Returns the total probe length. + size_t DecodeAndInsertToTable(CommonFields& common, + const PolicyFunctions& __restrict policy, + void* old_slots) const { + if (pos_ == buffer_) { + return 0; + } + if constexpr (kGuaranteedFitToBuffer) { + return DecodeAndInsertImpl(common, policy, buffer_, pos_, old_slots); + } + size_t total_probe_length = DecodeAndInsertImpl( + common, policy, buffer_, + local_buffer_full_ ? buffer_ + kBufferSize : pos_, old_slots); + if (!local_buffer_full_) { + return total_probe_length; + } + total_probe_length += + DecodeAndInsertToTableOverflow(common, policy, old_slots); + return total_probe_length; + } + + private: + static ProbedItem* AlignToNextItem(void* ptr) { + return reinterpret_cast(AlignUpTo( + reinterpret_cast(ptr), alignof(ProbedItem))); + } + + ProbedItem* OverflowBufferStart() const { + // We reuse GrowthInfo memory as well. + return AlignToNextItem(control_ - ControlOffset(/*has_infoz=*/false)); + } + + // Encodes item when previously allocated buffer is full. + // At first that happens when local buffer is full. + // We switch from the local buffer to the control buffer. + // Every time this function is called, the available buffer is extended till + // `item.source_offset` byte in the control buffer. + // After the buffer is extended, this function wouldn't be called till the + // buffer is exhausted. + // + // If there's no space in the control buffer, we fallback to naive algorithm + // and mark probed elements as kSentinel in the control buffer. In this case, + // we will call this function for every subsequent probed element. + ABSL_ATTRIBUTE_NOINLINE void ProcessEncodeWithOverflow(ProbedItem item) { + if (!local_buffer_full_) { + local_buffer_full_ = true; + pos_ = OverflowBufferStart(); + } + const size_t source_offset = static_cast(item.source_offset); + // We are in fallback mode so we can't reuse control buffer anymore. + // Probed elements are marked as kSentinel in the control buffer. + if (ABSL_PREDICT_FALSE(marked_elements_starting_position_ != + kNoMarkedElementsSentinel)) { + control_[source_offset] = ctrl_t::kSentinel; + return; + } + // Refresh the end pointer to the new available position. + // Invariant: if pos < end, then we have at least sizeof(ProbedItem) bytes + // to write. + end_ = control_ + source_offset + 1 - sizeof(ProbedItem); + if (ABSL_PREDICT_TRUE(pos_ < end_)) { + *pos_ = item; + ++pos_; + return; + } + control_[source_offset] = ctrl_t::kSentinel; + marked_elements_starting_position_ = source_offset; + // Now we will always fall down to `ProcessEncodeWithOverflow`. + ABSL_SWISSTABLE_ASSERT(pos_ >= end_); + } + + // Decodes information about probed elements from control buffer and processes + // marked elements. + // Finds new position for each element and transfers it to the new slots. + // Returns the total probe length. + ABSL_ATTRIBUTE_NOINLINE size_t DecodeAndInsertToTableOverflow( + CommonFields& common, const PolicyFunctions& __restrict policy, + void* old_slots) const { + ABSL_SWISSTABLE_ASSERT(local_buffer_full_ && + "must not be called when local buffer is not full"); + size_t total_probe_length = DecodeAndInsertImpl( + common, policy, OverflowBufferStart(), pos_, old_slots); + if (ABSL_PREDICT_TRUE(marked_elements_starting_position_ == + kNoMarkedElementsSentinel)) { + return total_probe_length; + } + total_probe_length += + ProcessProbedMarkedElements(common, policy, control_, old_slots, + marked_elements_starting_position_); + return total_probe_length; + } + + static constexpr size_t kBufferSize = + kProbedElementsBufferSize / sizeof(ProbedItem); + ProbedItem buffer_[kBufferSize]; + // If local_buffer_full_ is false, then pos_/end_ are in the local buffer, + // otherwise, they're in the overflow buffer. + ProbedItem* pos_ = buffer_; + const void* end_ = buffer_ + kBufferSize; + ctrl_t* const control_; + size_t marked_elements_starting_position_ = kNoMarkedElementsSentinel; + bool local_buffer_full_ = false; +}; + +// Grows to next capacity with specified encoder type. +// Encoder is used to store probed elements that are processed later. +// Different encoder is used depending on the capacity of the table. +// Returns total probe length. +template +size_t GrowToNextCapacity(CommonFields& common, + const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots) { + using ProbedItem = typename Encoder::ProbedItem; + ABSL_SWISSTABLE_ASSERT(common.capacity() <= ProbedItem::kMaxNewCapacity); + Encoder encoder(old_ctrl); + policy.transfer_unprobed_elements_to_next_capacity( + common, old_ctrl, old_slots, &encoder, + [](void* probed_storage, h2_t h2, size_t source_offset, size_t h1) { + auto encoder_ptr = static_cast(probed_storage); + encoder_ptr->EncodeItem(ProbedItem(h2, source_offset, h1)); + }); + InitializeMirroredControlBytes(common.control(), common.capacity()); + return encoder.DecodeAndInsertToTable(common, policy, old_slots); } -void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable( - CommonFields& c, size_t slot_size) { - assert(old_capacity_ < Group::kWidth / 2); - assert(is_single_group(c.capacity())); - assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); +// Grows to next capacity for relatively small tables so that even if all +// elements are probed, we don't need to overflow the local buffer. +// Returns total probe length. +size_t GrowToNextCapacityThatFitsInLocalBuffer( + CommonFields& common, const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots) { + ABSL_SWISSTABLE_ASSERT(common.capacity() <= kMaxLocalBufferNewCapacity); + return GrowToNextCapacity< + ProbedItemEncoder>( + common, policy, old_ctrl, old_slots); +} - GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity()); - GrowIntoSingleGroupShuffleTransferableSlots(c.slot_array(), slot_size); +// Grows to next capacity with different encodings. Returns total probe length. +// These functions are useful to simplify profile analysis. +size_t GrowToNextCapacity4BytesEncoder(CommonFields& common, + const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots) { + return GrowToNextCapacity>( + common, policy, old_ctrl, old_slots); +} +size_t GrowToNextCapacity8BytesEncoder(CommonFields& common, + const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots) { + return GrowToNextCapacity>( + common, policy, old_ctrl, old_slots); +} +size_t GrowToNextCapacity16BytesEncoder( + CommonFields& common, const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots) { + return GrowToNextCapacity>( + common, policy, old_ctrl, old_slots); +} - // We poison since GrowIntoSingleGroupShuffleTransferableSlots - // may leave empty slots unpoisoned. - PoisonSingleGroupEmptySlots(c, slot_size); +// Grows to next capacity for tables with relatively large capacity so that we +// can't guarantee that all probed elements fit in the local buffer. Returns +// total probe length. +size_t GrowToNextCapacityOverflowLocalBuffer( + CommonFields& common, const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots) { + const size_t new_capacity = common.capacity(); + if (ABSL_PREDICT_TRUE(new_capacity <= ProbedItem4Bytes::kMaxNewCapacity)) { + return GrowToNextCapacity4BytesEncoder(common, policy, old_ctrl, old_slots); + } + if (ABSL_PREDICT_TRUE(new_capacity <= ProbedItem8Bytes::kMaxNewCapacity)) { + return GrowToNextCapacity8BytesEncoder(common, policy, old_ctrl, old_slots); + } + // 16 bytes encoding supports the maximum swisstable capacity. + return GrowToNextCapacity16BytesEncoder(common, policy, old_ctrl, old_slots); } -void HashSetResizeHelper::TransferSlotAfterSoo(CommonFields& c, - size_t slot_size) { - assert(was_soo_); - assert(had_soo_slot_); - assert(is_single_group(c.capacity())); - std::memcpy(SlotAddress(c.slot_array(), SooSlotIndex(), slot_size), - old_soo_data(), slot_size); - PoisonSingleGroupEmptySlots(c, slot_size); +// Dispatches to the appropriate `GrowToNextCapacity*` function based on the +// capacity of the table. Returns total probe length. +ABSL_ATTRIBUTE_NOINLINE +size_t GrowToNextCapacityDispatch(CommonFields& common, + const PolicyFunctions& __restrict policy, + ctrl_t* old_ctrl, void* old_slots) { + const size_t new_capacity = common.capacity(); + if (ABSL_PREDICT_TRUE(new_capacity <= kMaxLocalBufferNewCapacity)) { + return GrowToNextCapacityThatFitsInLocalBuffer(common, policy, old_ctrl, + old_slots); + } else { + return GrowToNextCapacityOverflowLocalBuffer(common, policy, old_ctrl, + old_slots); + } } -namespace { +// Grows to next capacity and prepares insert for the given new_hash. +// Returns the offset of the new element. +size_t GrowToNextCapacityAndPrepareInsert( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_hash) { + ABSL_SWISSTABLE_ASSERT(common.growth_left() == 0); + const size_t old_capacity = common.capacity(); + ABSL_SWISSTABLE_ASSERT(old_capacity == 0 || + old_capacity > policy.soo_capacity()); + + const size_t new_capacity = NextCapacity(old_capacity); + ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity)); + ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity()); + + ctrl_t* old_ctrl = common.control(); + void* old_slots = common.slot_array(); + + common.set_capacity(new_capacity); + const size_t slot_size = policy.slot_size; + const size_t slot_align = policy.slot_align; + HashtablezInfoHandle infoz; + if (old_capacity > 0) { + infoz = common.infoz(); + } else { + const bool should_sample = + policy.is_hashtablez_eligible && ShouldSampleNextTable(); + if (ABSL_PREDICT_FALSE(should_sample)) { + infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, + policy.soo_capacity()); + } + } + const bool has_infoz = infoz.IsSampled(); + + RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz); + void* alloc = policy.get_char_alloc(common); + char* mem = static_cast(policy.alloc(alloc, layout.alloc_size())); + const GenerationType old_generation = common.generation(); + common.set_generation_ptr( + reinterpret_cast(mem + layout.generation_offset())); + common.set_generation(NextGeneration(old_generation)); + + ctrl_t* new_ctrl = reinterpret_cast(mem + layout.control_offset()); + void* new_slots = mem + layout.slot_offset(); + common.set_control(new_ctrl); + common.set_slots(new_slots); + SanitizerPoisonMemoryRegion(new_slots, new_capacity * slot_size); + + h2_t new_h2 = H2(new_hash); + size_t total_probe_length = 0; + FindInfo find_info; + if (old_capacity == 0) { + static_assert(NextCapacity(0) == 1); + InitializeSingleElementControlBytes(new_h2, new_ctrl); + common.generate_new_seed(); + find_info = FindInfo{0, 0}; + SanitizerUnpoisonMemoryRegion(new_slots, slot_size); + } else { + if (ABSL_PREDICT_TRUE(is_single_group(new_capacity))) { + GrowIntoSingleGroupShuffleControlBytes(old_ctrl, old_capacity, new_ctrl, + new_capacity); + // Single group tables have all slots full on resize. So we can transfer + // all slots without checking the control bytes. + ABSL_SWISSTABLE_ASSERT(common.size() == old_capacity); + auto* target = NextSlot(new_slots, slot_size); + SanitizerUnpoisonMemoryRegion(target, old_capacity * slot_size); + policy.transfer_n(&common, target, old_slots, old_capacity); + // We put the new element either at the beginning or at the end of the + // table with approximately equal probability. + size_t offset = SingleGroupTableH1(new_hash, common.seed()) & 1 + ? 0 + : new_capacity - 1; + + ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[offset])); + SetCtrlInSingleGroupTable(common, offset, new_h2, policy.slot_size); + find_info = FindInfo{offset, 0}; + } else { + total_probe_length = + GrowToNextCapacityDispatch(common, policy, old_ctrl, old_slots); + find_info = find_first_non_full(common, new_hash); + SetCtrlInLargeTable(common, find_info.offset, new_h2, policy.slot_size); + } + ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity()); + (*policy.dealloc)(alloc, old_capacity, old_ctrl, slot_size, slot_align, + has_infoz); + } + PrepareInsertCommon(common); + ResetGrowthLeft(GetGrowthInfoFromControl(new_ctrl), new_capacity, + common.size()); + + if (ABSL_PREDICT_FALSE(has_infoz)) { + common.set_has_infoz(); + infoz.RecordStorageChanged(common.size() - 1, new_capacity); + infoz.RecordRehash(total_probe_length); + infoz.RecordInsert(new_hash, find_info.probe_length); + common.set_infoz(infoz); + } + return find_info.offset; +} // Called whenever the table needs to vacate empty slots either by removing -// tombstones via rehash or growth. +// tombstones via rehash or growth to next capacity. ABSL_ATTRIBUTE_NOINLINE -FindInfo FindInsertPositionWithGrowthOrRehash(CommonFields& common, size_t hash, - const PolicyFunctions& policy) { +size_t RehashOrGrowToNextCapacityAndPrepareInsert( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_hash) { const size_t cap = common.capacity(); + ABSL_ASSUME(cap > 0); if (cap > Group::kWidth && // Do these calculations in 64-bit to avoid overflow. common.size() * uint64_t{32} <= cap * uint64_t{25}) { @@ -574,97 +1375,467 @@ FindInfo FindInsertPositionWithGrowthOrRehash(CommonFields& common, size_t hash, // 762 | 149836 0.37 13 | 148559 0.74 190 // 807 | 149736 0.39 14 | 151107 0.39 14 // 852 | 150204 0.42 15 | 151019 0.42 15 - DropDeletesWithoutResize(common, policy); + return DropDeletesWithoutResizeAndPrepareInsert(common, policy, new_hash); } else { // Otherwise grow the container. - policy.resize(common, NextCapacity(cap), HashtablezInfoHandle{}); + return GrowToNextCapacityAndPrepareInsert(common, policy, new_hash); + } +} + +// Slow path for PrepareInsertNonSoo that is called when the table has deleted +// slots or need to be resized or rehashed. +size_t PrepareInsertNonSooSlow(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t hash) { + const GrowthInfo growth_info = common.growth_info(); + ABSL_SWISSTABLE_ASSERT(!growth_info.HasNoDeletedAndGrowthLeft()); + if (ABSL_PREDICT_TRUE(growth_info.HasNoGrowthLeftAndNoDeleted())) { + // Table without deleted slots (>95% cases) that needs to be resized. + ABSL_SWISSTABLE_ASSERT(growth_info.HasNoDeleted() && + growth_info.GetGrowthLeft() == 0); + return GrowToNextCapacityAndPrepareInsert(common, policy, hash); + } + if (ABSL_PREDICT_FALSE(growth_info.HasNoGrowthLeftAssumingMayHaveDeleted())) { + // Table with deleted slots that needs to be rehashed or resized. + return RehashOrGrowToNextCapacityAndPrepareInsert(common, policy, hash); + } + // Table with deleted slots that has space for the inserting element. + FindInfo target = find_first_non_full(common, hash); + PrepareInsertCommon(common); + common.growth_info().OverwriteControlAsFull(common.control()[target.offset]); + SetCtrlInLargeTable(common, target.offset, H2(hash), policy.slot_size); + common.infoz().RecordInsert(hash, target.probe_length); + return target.offset; +} + +// Resizes empty non-allocated SOO table to NextCapacity(SooCapacity()), +// forces the table to be sampled and prepares the insert. +// SOO tables need to switch from SOO to heap in order to store the infoz. +// Requires: +// 1. `c.capacity() == SooCapacity()`. +// 2. `c.empty()`. +ABSL_ATTRIBUTE_NOINLINE size_t +GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_hash) { + ResizeEmptyNonAllocatedTableImpl(common, policy, NextCapacity(SooCapacity()), + /*force_infoz=*/true); + PrepareInsertCommon(common); + common.growth_info().OverwriteEmptyAsFull(); + SetCtrlInSingleGroupTable(common, SooSlotIndex(), H2(new_hash), + policy.slot_size); + common.infoz().RecordInsert(new_hash, /*distance_from_desired=*/0); + return SooSlotIndex(); +} + +// Resizes empty non-allocated table to the capacity to fit new_size elements. +// Requires: +// 1. `c.capacity() == policy.soo_capacity()`. +// 2. `c.empty()`. +// 3. `new_size > policy.soo_capacity()`. +// The table will be attempted to be sampled. +void ReserveEmptyNonAllocatedTableToFitNewSize( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_size) { + ValidateMaxSize(new_size, policy.slot_size); + ABSL_ASSUME(new_size > 0); + ResizeEmptyNonAllocatedTableImpl(common, policy, SizeToCapacity(new_size), + /*force_infoz=*/false); + // This is after resize, to ensure that we have completed the allocation + // and have potentially sampled the hashtable. + common.infoz().RecordReservation(new_size); +} + +// Type erased version of raw_hash_set::reserve for tables that have an +// allocated backing array. +// +// Requires: +// 1. `c.capacity() > policy.soo_capacity()` OR `!c.empty()`. +// Reserving already allocated tables is considered to be a rare case. +ABSL_ATTRIBUTE_NOINLINE void ReserveAllocatedTable( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_size) { + const size_t cap = common.capacity(); + ValidateMaxSize(new_size, policy.slot_size); + ABSL_ASSUME(new_size > 0); + const size_t new_capacity = SizeToCapacity(new_size); + if (cap == policy.soo_capacity()) { + ABSL_SWISSTABLE_ASSERT(!common.empty()); + ResizeFullSooTable(common, policy, new_capacity, + ResizeFullSooTableSamplingMode::kNoSampling); + } else { + ABSL_SWISSTABLE_ASSERT(cap > policy.soo_capacity()); + // TODO(b/382423690): consider using GrowToNextCapacity, when applicable. + ResizeAllocatedTableWithSeedChange(common, policy, new_capacity); } - // This function is typically called with tables containing deleted slots. - // The table will be big and `FindFirstNonFullAfterResize` will always - // fallback to `find_first_non_full`. So using `find_first_non_full` directly. - return find_first_non_full(common, hash); + common.infoz().RecordReservation(new_size); } } // namespace -const void* GetHashRefForEmptyHasher(const CommonFields& common) { +void* GetRefForEmptyClass(CommonFields& common) { // Empty base optimization typically make the empty base class address to be // the same as the first address of the derived class object. - // But we generally assume that for empty hasher we can return any valid + // But we generally assume that for empty classes we can return any valid // pointer. return &common; } -FindInfo HashSetResizeHelper::FindFirstNonFullAfterResize(const CommonFields& c, - size_t old_capacity, - size_t hash) { - size_t new_capacity = c.capacity(); - if (!IsGrowingIntoSingleGroupApplicable(old_capacity, new_capacity)) { - return find_first_non_full(c, hash); +void ResizeAllocatedTableWithSeedChange( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_capacity) { + ResizeNonSooImpl( + common, policy, new_capacity, common.infoz()); +} + +void ReserveEmptyNonAllocatedTableToFitBucketCount( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t bucket_count) { + size_t new_capacity = NormalizeCapacity(bucket_count); + ValidateMaxSize(CapacityToGrowth(new_capacity), policy.slot_size); + ResizeEmptyNonAllocatedTableImpl(common, policy, new_capacity, + /*force_infoz=*/false); +} + +// Resizes a full SOO table to the NextCapacity(SooCapacity()). +template +size_t GrowSooTableToNextCapacityAndPrepareInsert( + CommonFields& common, const PolicyFunctions& __restrict policy, + size_t new_hash, ctrl_t soo_slot_ctrl) { + AssertSoo(common, policy); + if (ABSL_PREDICT_FALSE(soo_slot_ctrl == ctrl_t::kEmpty)) { + // The table is empty, it is only used for forced sampling of SOO tables. + return GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert( + common, policy, new_hash); } + ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity()); + static constexpr size_t kNewCapacity = NextCapacity(SooCapacity()); + const size_t slot_size = policy.slot_size; + const size_t slot_align = policy.slot_align; + common.set_capacity(kNewCapacity); + + // Since the table is not empty, it will not be sampled. + // The decision to sample was already made during the first insertion. + RawHashSetLayout layout(kNewCapacity, slot_size, slot_align, + /*has_infoz=*/false); + void* alloc = policy.get_char_alloc(common); + char* mem = static_cast(policy.alloc(alloc, layout.alloc_size())); + const GenerationType old_generation = common.generation(); + common.set_generation_ptr( + reinterpret_cast(mem + layout.generation_offset())); + common.set_generation(NextGeneration(old_generation)); + + // We do not set control and slots in CommonFields yet to avoid overriding + // SOO data. + ctrl_t* new_ctrl = reinterpret_cast(mem + layout.control_offset()); + void* new_slots = mem + layout.slot_offset(); - // We put the new element either at the beginning or at the end of the table - // with approximately equal probability. - size_t offset = - SingleGroupTableH1(hash, c.control()) & 1 ? 0 : new_capacity - 1; + PrepareInsertCommon(common); + ABSL_SWISSTABLE_ASSERT(common.size() == 2); + GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(kNewCapacity - 2); + common.generate_new_seed(); - assert(IsEmpty(c.control()[offset])); - return FindInfo{offset, 0}; + // After resize from capacity 1 to 3, we always have exactly the slot with + // index 1 occupied, so we need to insert either at index 0 or index 2. + static_assert(SooSlotIndex() == 1); + const size_t offset = SingleGroupTableH1(new_hash, common.seed()) & 2; + InitializeThreeElementsControlBytesAfterSoo(soo_slot_ctrl, new_hash, offset, + new_ctrl); + + SanitizerPoisonMemoryRegion(new_slots, slot_size * kNewCapacity); + void* target_slot = SlotAddress(new_slots, SooSlotIndex(), slot_size); + SanitizerUnpoisonMemoryRegion(target_slot, slot_size); + if constexpr (TransferUsesMemcpy) { + // Target slot is placed at index 1, but capacity is at + // minimum 3. So we are allowed to copy at least twice as much + // memory. + static_assert(SooSlotIndex() == 1); + static_assert(SooSlotMemcpySize > 0); + static_assert(SooSlotMemcpySize <= MaxSooSlotSize()); + ABSL_SWISSTABLE_ASSERT(SooSlotMemcpySize <= 2 * slot_size); + ABSL_SWISSTABLE_ASSERT(SooSlotMemcpySize >= slot_size); + void* next_slot = SlotAddress(target_slot, 1, slot_size); + SanitizerUnpoisonMemoryRegion(next_slot, SooSlotMemcpySize - slot_size); + std::memcpy(target_slot, common.soo_data(), SooSlotMemcpySize); + SanitizerPoisonMemoryRegion(next_slot, SooSlotMemcpySize - slot_size); + } else { + static_assert(SooSlotMemcpySize == 0); + policy.transfer_n(&common, target_slot, common.soo_data(), 1); + } + // Seed was already generated above. + common.set_control(new_ctrl); + common.set_slots(new_slots); + + common.infoz().RecordInsert(new_hash, /*distance_from_desired=*/0); + SanitizerUnpoisonMemoryRegion(SlotAddress(new_slots, offset, slot_size), + slot_size); + return offset; } -size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target, - const PolicyFunctions& policy) { - // When there are no deleted slots in the table - // and growth_left is positive, we can insert at the first - // empty slot in the probe sequence (target). - const bool use_target_hint = - // Optimization is disabled when generations are enabled. - // We have to rehash even sparse tables randomly in such mode. - !SwisstableGenerationsEnabled() && - common.growth_info().HasNoDeletedAndGrowthLeft(); - if (ABSL_PREDICT_FALSE(!use_target_hint)) { - // Notes about optimized mode when generations are disabled: - // We do not enter this branch if table has no deleted slots - // and growth_left is positive. - // We enter this branch in the following cases listed in decreasing - // frequency: - // 1. Table without deleted slots (>95% cases) that needs to be resized. - // 2. Table with deleted slots that has space for the inserting element. - // 3. Table with deleted slots that needs to be rehashed or resized. - if (ABSL_PREDICT_TRUE(common.growth_info().HasNoGrowthLeftAndNoDeleted())) { - const size_t old_capacity = common.capacity(); - policy.resize(common, NextCapacity(old_capacity), HashtablezInfoHandle{}); - target = HashSetResizeHelper::FindFirstNonFullAfterResize( - common, old_capacity, hash); - } else { - // Note: the table may have no deleted slots here when generations - // are enabled. - const bool rehash_for_bug_detection = - common.should_rehash_for_bug_detection_on_insert(); - if (rehash_for_bug_detection) { - // Move to a different heap allocation in order to detect bugs. - const size_t cap = common.capacity(); - policy.resize(common, - common.growth_left() > 0 ? cap : NextCapacity(cap), - HashtablezInfoHandle{}); +void GrowFullSooTableToNextCapacityForceSampling( + CommonFields& common, const PolicyFunctions& __restrict policy) { + AssertFullSoo(common, policy); + ResizeFullSooTable( + common, policy, NextCapacity(SooCapacity()), + ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled); +} + +void Rehash(CommonFields& common, const PolicyFunctions& __restrict policy, + size_t n) { + const size_t cap = common.capacity(); + + auto clear_backing_array = [&]() { + ClearBackingArray(common, policy, policy.get_char_alloc(common), + /*reuse=*/false, policy.soo_enabled); + }; + + const size_t slot_size = policy.slot_size; + + if (n == 0) { + if (cap <= policy.soo_capacity()) return; + if (common.empty()) { + clear_backing_array(); + return; + } + if (common.size() <= policy.soo_capacity()) { + // When the table is already sampled, we keep it sampled. + if (common.infoz().IsSampled()) { + static constexpr size_t kInitialSampledCapacity = + NextCapacity(SooCapacity()); + if (cap > kInitialSampledCapacity) { + ResizeAllocatedTableWithSeedChange(common, policy, + kInitialSampledCapacity); + } + // This asserts that we didn't lose sampling coverage in `resize`. + ABSL_SWISSTABLE_ASSERT(common.infoz().IsSampled()); + return; } - if (ABSL_PREDICT_TRUE(common.growth_left() > 0)) { - target = find_first_non_full(common, hash); + ABSL_SWISSTABLE_ASSERT(slot_size <= sizeof(HeapOrSoo)); + ABSL_SWISSTABLE_ASSERT(policy.slot_align <= alignof(HeapOrSoo)); + HeapOrSoo tmp_slot(uninitialized_tag_t{}); + size_t begin_offset = FindFirstFullSlot(0, cap, common.control()); + policy.transfer_n( + &common, &tmp_slot, + SlotAddress(common.slot_array(), begin_offset, slot_size), 1); + clear_backing_array(); + policy.transfer_n(&common, common.soo_data(), &tmp_slot, 1); + common.set_full_soo(); + return; + } + } + + ValidateMaxSize(n, policy.slot_size); + // bitor is a faster way of doing `max` here. We will round up to the next + // power-of-2-minus-1, so bitor is good enough. + const size_t new_capacity = + NormalizeCapacity(n | SizeToCapacity(common.size())); + // n == 0 unconditionally rehashes as per the standard. + if (n == 0 || new_capacity > cap) { + if (cap == policy.soo_capacity()) { + if (common.empty()) { + ResizeEmptyNonAllocatedTableImpl(common, policy, new_capacity, + /*force_infoz=*/false); } else { - target = FindInsertPositionWithGrowthOrRehash(common, hash, policy); + ResizeFullSooTable(common, policy, new_capacity, + ResizeFullSooTableSamplingMode::kNoSampling); } + } else { + ResizeAllocatedTableWithSeedChange(common, policy, new_capacity); + } + // This is after resize, to ensure that we have completed the allocation + // and have potentially sampled the hashtable. + common.infoz().RecordReservation(n); + } +} + +void Copy(CommonFields& common, const PolicyFunctions& __restrict policy, + const CommonFields& other, + absl::FunctionRef copy_fn) { + const size_t size = other.size(); + ABSL_SWISSTABLE_ASSERT(size > 0); + const size_t soo_capacity = policy.soo_capacity(); + const size_t slot_size = policy.slot_size; + if (size <= soo_capacity) { + ABSL_SWISSTABLE_ASSERT(size == 1); + common.set_full_soo(); + const void* other_slot = + other.capacity() <= soo_capacity + ? other.soo_data() + : SlotAddress( + other.slot_array(), + FindFirstFullSlot(0, other.capacity(), other.control()), + slot_size); + copy_fn(common.soo_data(), other_slot); + + if (policy.is_hashtablez_eligible && ShouldSampleNextTable()) { + GrowFullSooTableToNextCapacityForceSampling(common, policy); } + return; + } + + ReserveTableToFitNewSize(common, policy, size); + auto infoz = common.infoz(); + ABSL_SWISSTABLE_ASSERT(other.capacity() > soo_capacity); + const size_t cap = common.capacity(); + ABSL_SWISSTABLE_ASSERT(cap > soo_capacity); + // Note about single group tables: + // 1. It is correct to have any order of elements. + // 2. Order has to be non deterministic. + // 3. We are assigning elements with arbitrary `shift` starting from + // `capacity + shift` position. + // 4. `shift` must be coprime with `capacity + 1` in order to be able to use + // modular arithmetic to traverse all positions, instead of cycling + // through a subset of positions. Odd numbers are coprime with any + // `capacity + 1` (2^N). + size_t offset = cap; + const size_t shift = is_single_group(cap) ? (common.seed().seed() | 1) : 0; + const void* hash_fn = policy.hash_fn(common); + auto hasher = policy.hash_slot; + IterateOverFullSlotsImpl( + other, slot_size, [&](const ctrl_t* that_ctrl, void* that_slot) { + if (shift == 0) { + // Big tables case. Position must be searched via probing. + // The table is guaranteed to be empty, so we can do faster than + // a full `insert`. + const size_t hash = (*hasher)(hash_fn, that_slot); + FindInfo target = find_first_non_full(common, hash); + infoz.RecordInsert(hash, target.probe_length); + offset = target.offset; + } else { + // Small tables case. Next position is computed via shift. + offset = (offset + shift) & cap; + } + const h2_t h2 = static_cast(*that_ctrl); + // We rely on the hash not changing for small tables. + ABSL_SWISSTABLE_ASSERT( + H2((*hasher)(hash_fn, that_slot)) == h2 && + "hash function value changed unexpectedly during the copy"); + SetCtrl(common, offset, h2, slot_size); + copy_fn(SlotAddress(common.slot_array(), offset, slot_size), that_slot); + common.maybe_increment_generation_on_insert(); + }); + if (shift != 0) { + // On small table copy we do not record individual inserts. + // RecordInsert requires hash, but it is unknown for small tables. + infoz.RecordStorageChanged(size, cap); + } + common.increment_size(size); + common.growth_info().OverwriteManyEmptyAsFull(size); +} + +void ReserveTableToFitNewSize(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t new_size) { + common.reset_reserved_growth(new_size); + common.set_reservation_size(new_size); + ABSL_SWISSTABLE_ASSERT(new_size > policy.soo_capacity()); + const size_t cap = common.capacity(); + if (ABSL_PREDICT_TRUE(common.empty() && cap <= policy.soo_capacity())) { + return ReserveEmptyNonAllocatedTableToFitNewSize(common, policy, new_size); + } + + ABSL_SWISSTABLE_ASSERT(!common.empty() || cap > policy.soo_capacity()); + ABSL_SWISSTABLE_ASSERT(cap > 0); + const size_t max_size_before_growth = + cap <= policy.soo_capacity() ? policy.soo_capacity() + : common.size() + common.growth_left(); + if (new_size <= max_size_before_growth) { + return; + } + ReserveAllocatedTable(common, policy, new_size); +} + +size_t PrepareInsertNonSoo(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t hash, FindInfo target) { + const bool rehash_for_bug_detection = + common.should_rehash_for_bug_detection_on_insert() && + // Required to allow use of ResizeAllocatedTable. + common.capacity() > 0; + if (rehash_for_bug_detection) { + // Move to a different heap allocation in order to detect bugs. + const size_t cap = common.capacity(); + ResizeAllocatedTableWithSeedChange( + common, policy, common.growth_left() > 0 ? cap : NextCapacity(cap)); + target = find_first_non_full(common, hash); + } + + const GrowthInfo growth_info = common.growth_info(); + // When there are no deleted slots in the table + // and growth_left is positive, we can insert at the first + // empty slot in the probe sequence (target). + if (ABSL_PREDICT_FALSE(!growth_info.HasNoDeletedAndGrowthLeft())) { + return PrepareInsertNonSooSlow(common, policy, hash); } PrepareInsertCommon(common); - common.growth_info().OverwriteControlAsFull(common.control()[target.offset]); + common.growth_info().OverwriteEmptyAsFull(); SetCtrl(common, target.offset, H2(hash), policy.slot_size); common.infoz().RecordInsert(hash, target.probe_length); return target.offset; } -void HashTableSizeOverflow() { - ABSL_RAW_LOG(FATAL, "Hash table size overflow"); +namespace { +// Returns true if the following is true +// 1. OptimalMemcpySizeForSooSlotTransfer(left) > +// OptimalMemcpySizeForSooSlotTransfer(left - 1) +// 2. OptimalMemcpySizeForSooSlotTransfer(left) are equal for all i in [left, +// right]. +// This function is used to verify that we have all the possible template +// instantiations for GrowFullSooTableToNextCapacity. +// With this verification the problem may be detected at compile time instead of +// link time. +constexpr bool VerifyOptimalMemcpySizeForSooSlotTransferRange(size_t left, + size_t right) { + size_t optimal_size_for_range = OptimalMemcpySizeForSooSlotTransfer(left); + if (optimal_size_for_range <= OptimalMemcpySizeForSooSlotTransfer(left - 1)) { + return false; + } + for (size_t i = left + 1; i <= right; ++i) { + if (OptimalMemcpySizeForSooSlotTransfer(i) != optimal_size_for_range) { + return false; + } + } + return true; } +} // namespace + +// Extern template instantiation for inline function. +template size_t TryFindNewIndexWithoutProbing(size_t h1, size_t old_index, + size_t old_capacity, + ctrl_t* new_ctrl, + size_t new_capacity); + +// We need to instantiate ALL possible template combinations because we define +// the function in the cc file. +template size_t GrowSooTableToNextCapacityAndPrepareInsert<0, false>( + CommonFields&, const PolicyFunctions&, size_t, ctrl_t); +template size_t GrowSooTableToNextCapacityAndPrepareInsert< + OptimalMemcpySizeForSooSlotTransfer(1), true>(CommonFields&, + const PolicyFunctions&, + size_t, ctrl_t); + +static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(2, 3)); +template size_t GrowSooTableToNextCapacityAndPrepareInsert< + OptimalMemcpySizeForSooSlotTransfer(3), true>(CommonFields&, + const PolicyFunctions&, + size_t, ctrl_t); + +static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(4, 8)); +template size_t GrowSooTableToNextCapacityAndPrepareInsert< + OptimalMemcpySizeForSooSlotTransfer(8), true>(CommonFields&, + const PolicyFunctions&, + size_t, ctrl_t); + +#if UINTPTR_MAX == UINT32_MAX +static_assert(MaxSooSlotSize() == 8); +#else +static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(9, 16)); +template size_t GrowSooTableToNextCapacityAndPrepareInsert< + OptimalMemcpySizeForSooSlotTransfer(16), true>(CommonFields&, + const PolicyFunctions&, + size_t, ctrl_t); +static_assert(MaxSooSlotSize() == 16); +#endif } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 79ccb59..3effc44 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -196,6 +196,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/endian.h" +#include "absl/base/internal/iterator_traits.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" #include "absl/base/optimization.h" @@ -208,30 +209,17 @@ #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" #include "absl/container/internal/hash_policy_traits.h" +#include "absl/container/internal/hashtable_control_bytes.h" #include "absl/container/internal/hashtable_debug_hooks.h" #include "absl/container/internal/hashtablez_sampler.h" +#include "absl/functional/function_ref.h" #include "absl/hash/hash.h" +#include "absl/hash/internal/weakly_mixed_integer.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" #include "absl/numeric/bits.h" #include "absl/utility/utility.h" -#ifdef ABSL_INTERNAL_HAVE_SSE2 -#include -#endif - -#ifdef ABSL_INTERNAL_HAVE_SSSE3 -#include -#endif - -#ifdef _MSC_VER -#include -#endif - -#ifdef ABSL_INTERNAL_HAVE_ARM_NEON -#include -#endif - namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { @@ -278,6 +266,15 @@ constexpr bool SwisstableGenerationsEnabled() { return false; } constexpr size_t NumGenerationBytes() { return 0; } #endif +// Returns true if we should assert that the table is not accessed after it has +// been destroyed or during the destruction of the table. +constexpr bool SwisstableAssertAccessToDestroyedTable() { +#ifndef NDEBUG + return true; +#endif + return SwisstableGenerationsEnabled(); +} + template void SwapAlloc(AllocType& lhs, AllocType& rhs, std::true_type /* propagate_on_container_swap */) { @@ -383,163 +380,6 @@ constexpr bool IsNoThrowSwappable(std::false_type /* is_swappable */) { return false; } -template -uint32_t TrailingZeros(T x) { - ABSL_ASSUME(x != 0); - return static_cast(countr_zero(x)); -} - -// 8 bytes bitmask with most significant bit set for every byte. -constexpr uint64_t kMsbs8Bytes = 0x8080808080808080ULL; - -// An abstract bitmask, such as that emitted by a SIMD instruction. -// -// Specifically, this type implements a simple bitset whose representation is -// controlled by `SignificantBits` and `Shift`. `SignificantBits` is the number -// of abstract bits in the bitset, while `Shift` is the log-base-two of the -// width of an abstract bit in the representation. -// This mask provides operations for any number of real bits set in an abstract -// bit. To add iteration on top of that, implementation must guarantee no more -// than the most significant real bit is set in a set abstract bit. -template -class NonIterableBitMask { - public: - explicit NonIterableBitMask(T mask) : mask_(mask) {} - - explicit operator bool() const { return this->mask_ != 0; } - - // Returns the index of the lowest *abstract* bit set in `self`. - uint32_t LowestBitSet() const { - return container_internal::TrailingZeros(mask_) >> Shift; - } - - // Returns the index of the highest *abstract* bit set in `self`. - uint32_t HighestBitSet() const { - return static_cast((bit_width(mask_) - 1) >> Shift); - } - - // Returns the number of trailing zero *abstract* bits. - uint32_t TrailingZeros() const { - return container_internal::TrailingZeros(mask_) >> Shift; - } - - // Returns the number of leading zero *abstract* bits. - uint32_t LeadingZeros() const { - constexpr int total_significant_bits = SignificantBits << Shift; - constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return static_cast( - countl_zero(static_cast(mask_ << extra_bits))) >> - Shift; - } - - T mask_; -}; - -// Mask that can be iterable -// -// For example, when `SignificantBits` is 16 and `Shift` is zero, this is just -// an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When -// `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as -// the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask. -// If NullifyBitsOnIteration is true (only allowed for Shift == 3), -// non zero abstract bit is allowed to have additional bits -// (e.g., `0xff`, `0x83` and `0x9c` are ok, but `0x6f` is not). -// -// For example: -// for (int i : BitMask(0b101)) -> yields 0, 2 -// for (int i : BitMask(0x0000000080800000)) -> yields 2, 3 -template -class BitMask : public NonIterableBitMask { - using Base = NonIterableBitMask; - static_assert(std::is_unsigned::value, ""); - static_assert(Shift == 0 || Shift == 3, ""); - static_assert(!NullifyBitsOnIteration || Shift == 3, ""); - - public: - explicit BitMask(T mask) : Base(mask) { - if (Shift == 3 && !NullifyBitsOnIteration) { - ABSL_SWISSTABLE_ASSERT(this->mask_ == (this->mask_ & kMsbs8Bytes)); - } - } - // BitMask is an iterator over the indices of its abstract bits. - using value_type = int; - using iterator = BitMask; - using const_iterator = BitMask; - - BitMask& operator++() { - if (Shift == 3 && NullifyBitsOnIteration) { - this->mask_ &= kMsbs8Bytes; - } - this->mask_ &= (this->mask_ - 1); - return *this; - } - - uint32_t operator*() const { return Base::LowestBitSet(); } - - BitMask begin() const { return *this; } - BitMask end() const { return BitMask(0); } - - private: - friend bool operator==(const BitMask& a, const BitMask& b) { - return a.mask_ == b.mask_; - } - friend bool operator!=(const BitMask& a, const BitMask& b) { - return a.mask_ != b.mask_; - } -}; - -using h2_t = uint8_t; - -// The values here are selected for maximum performance. See the static asserts -// below for details. - -// A `ctrl_t` is a single control byte, which can have one of four -// states: empty, deleted, full (which has an associated seven-bit h2_t value) -// and the sentinel. They have the following bit patterns: -// -// empty: 1 0 0 0 0 0 0 0 -// deleted: 1 1 1 1 1 1 1 0 -// full: 0 h h h h h h h // h represents the hash bits. -// sentinel: 1 1 1 1 1 1 1 1 -// -// These values are specifically tuned for SSE-flavored SIMD. -// The static_asserts below detail the source of these choices. -// -// We use an enum class so that when strict aliasing is enabled, the compiler -// knows ctrl_t doesn't alias other types. -enum class ctrl_t : int8_t { - kEmpty = -128, // 0b10000000 - kDeleted = -2, // 0b11111110 - kSentinel = -1, // 0b11111111 -}; -static_assert( - (static_cast(ctrl_t::kEmpty) & - static_cast(ctrl_t::kDeleted) & - static_cast(ctrl_t::kSentinel) & 0x80) != 0, - "Special markers need to have the MSB to make checking for them efficient"); -static_assert( - ctrl_t::kEmpty < ctrl_t::kSentinel && ctrl_t::kDeleted < ctrl_t::kSentinel, - "ctrl_t::kEmpty and ctrl_t::kDeleted must be smaller than " - "ctrl_t::kSentinel to make the SIMD test of IsEmptyOrDeleted() efficient"); -static_assert( - ctrl_t::kSentinel == static_cast(-1), - "ctrl_t::kSentinel must be -1 to elide loading it from memory into SIMD " - "registers (pcmpeqd xmm, xmm)"); -static_assert(ctrl_t::kEmpty == static_cast(-128), - "ctrl_t::kEmpty must be -128 to make the SIMD check for its " - "existence efficient (psignb xmm, xmm)"); -static_assert( - (~static_cast(ctrl_t::kEmpty) & - ~static_cast(ctrl_t::kDeleted) & - static_cast(ctrl_t::kSentinel) & 0x7F) != 0, - "ctrl_t::kEmpty and ctrl_t::kDeleted must share an unset bit that is not " - "shared by ctrl_t::kSentinel to make the scalar test for " - "MaskEmptyOrDeleted() efficient"); -static_assert(ctrl_t::kDeleted == static_cast(-2), - "ctrl_t::kDeleted must be -2 to make the implementation of " - "ConvertSpecialToEmptyAndFullToDeleted efficient"); - // See definition comment for why this is size 32. ABSL_DLL extern const ctrl_t kEmptyGroup[32]; @@ -585,360 +425,117 @@ inline bool IsEmptyGeneration(const GenerationType* generation) { return *generation == SentinelEmptyGeneration(); } -// Mixes a randomly generated per-process seed with `hash` and `ctrl` to -// randomize insertion order within groups. -bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, - const ctrl_t* ctrl); - -ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ShouldInsertBackwards( - ABSL_ATTRIBUTE_UNUSED size_t capacity, ABSL_ATTRIBUTE_UNUSED size_t hash, - ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) { -#if defined(NDEBUG) - return false; -#else - return ShouldInsertBackwardsForDebug(capacity, hash, ctrl); -#endif -} +// We only allow a maximum of 1 SOO element, which makes the implementation +// much simpler. Complications with multiple SOO elements include: +// - Satisfying the guarantee that erasing one element doesn't invalidate +// iterators to other elements means we would probably need actual SOO +// control bytes. +// - In order to prevent user code from depending on iteration order for small +// tables, we would need to randomize the iteration order somehow. +constexpr size_t SooCapacity() { return 1; } +// Sentinel type to indicate SOO CommonFields construction. +struct soo_tag_t {}; +// Sentinel type to indicate SOO CommonFields construction with full size. +struct full_soo_tag_t {}; +// Sentinel type to indicate non-SOO CommonFields construction. +struct non_soo_tag_t {}; +// Sentinel value to indicate an uninitialized value explicitly. +struct uninitialized_tag_t {}; +// Sentinel value to indicate creation of an empty table without a seed. +struct no_seed_empty_tag_t {}; -// Returns insert position for the given mask. -// We want to add entropy even when ASLR is not enabled. -// In debug build we will randomly insert in either the front or back of -// the group. -// TODO(kfm,sbenza): revisit after we do unconditional mixing -template -ABSL_ATTRIBUTE_ALWAYS_INLINE inline auto GetInsertionOffset( - Mask mask, ABSL_ATTRIBUTE_UNUSED size_t capacity, - ABSL_ATTRIBUTE_UNUSED size_t hash, - ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) { -#if defined(NDEBUG) - return mask.LowestBitSet(); -#else - return ShouldInsertBackwardsForDebug(capacity, hash, ctrl) - ? mask.HighestBitSet() - : mask.LowestBitSet(); -#endif -} +// Per table hash salt. This gets mixed into H1 to randomize iteration order +// per-table. +// The seed is needed to ensure non-determinism of iteration order. +class PerTableSeed { + public: + // The number of bits in the seed. + // It is big enough to ensure non-determinism of iteration order. + // We store the seed inside a uint64_t together with size and other metadata. + // Using 16 bits allows us to save one `and` instruction in H1 (we use movzwl + // instead of movq+and). + static constexpr size_t kBitCount = 16; -// Returns a per-table, hash salt, which changes on resize. This gets mixed into -// H1 to randomize iteration order per-table. -// -// The seed consists of the ctrl_ pointer, which adds enough entropy to ensure -// non-determinism of iteration order in most cases. -inline size_t PerTableSalt(const ctrl_t* ctrl) { - // The low bits of the pointer have little or no entropy because of - // alignment. We shift the pointer to try to use higher entropy bits. A - // good number seems to be 12 bits, because that aligns with page size. - return reinterpret_cast(ctrl) >> 12; -} -// Extracts the H1 portion of a hash: 57 bits mixed with a per-table salt. -inline size_t H1(size_t hash, const ctrl_t* ctrl) { - return (hash >> 7) ^ PerTableSalt(ctrl); -} + // Returns the seed for the table. Only the lowest kBitCount are non zero. + size_t seed() const { return seed_; } -// Extracts the H2 portion of a hash: the 7 bits not used for H1. -// -// These are used as an occupied control byte. -inline h2_t H2(size_t hash) { return hash & 0x7F; } + private: + friend class HashtableSize; + explicit PerTableSeed(size_t seed) : seed_(seed) {} -// Helpers for checking the state of a control byte. -inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } -inline bool IsFull(ctrl_t c) { - // Cast `c` to the underlying type instead of casting `0` to `ctrl_t` as `0` - // is not a value in the enum. Both ways are equivalent, but this way makes - // linters happier. - return static_cast>(c) >= 0; -} -inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } -inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } + const size_t seed_; +}; -#ifdef ABSL_INTERNAL_HAVE_SSE2 -// Quick reference guide for intrinsics used below: -// -// * __m128i: An XMM (128-bit) word. -// -// * _mm_setzero_si128: Returns a zero vector. -// * _mm_set1_epi8: Returns a vector with the same i8 in each lane. -// -// * _mm_subs_epi8: Saturating-subtracts two i8 vectors. -// * _mm_and_si128: Ands two i128s together. -// * _mm_or_si128: Ors two i128s together. -// * _mm_andnot_si128: And-nots two i128s together. -// -// * _mm_cmpeq_epi8: Component-wise compares two i8 vectors for equality, -// filling each lane with 0x00 or 0xff. -// * _mm_cmpgt_epi8: Same as above, but using > rather than ==. -// -// * _mm_loadu_si128: Performs an unaligned load of an i128. -// * _mm_storeu_si128: Performs an unaligned store of an i128. -// -// * _mm_sign_epi8: Retains, negates, or zeroes each i8 lane of the first -// argument if the corresponding lane of the second -// argument is positive, negative, or zero, respectively. -// * _mm_movemask_epi8: Selects the sign bit out of each i8 lane and produces a -// bitmask consisting of those bits. -// * _mm_shuffle_epi8: Selects i8s from the first argument, using the low -// four bits of each i8 lane in the second argument as -// indices. - -// https://github.com/abseil/abseil-cpp/issues/209 -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 -// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char -// Work around this by using the portable implementation of Group -// when using -funsigned-char under GCC. -inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { -#if defined(__GNUC__) && !defined(__clang__) - if (std::is_unsigned::value) { - const __m128i mask = _mm_set1_epi8(0x80); - const __m128i diff = _mm_subs_epi8(b, a); - return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); - } -#endif - return _mm_cmpgt_epi8(a, b); +// Returns next per-table seed. +inline uint16_t NextSeed() { + static_assert(PerTableSeed::kBitCount == 16); + thread_local uint16_t seed = + static_cast(reinterpret_cast(&seed)); + seed += uint16_t{0xad53}; + return seed; } -struct GroupSse2Impl { - static constexpr size_t kWidth = 16; // the number of slots per group +// The size and also has additionally +// 1) one bit that stores whether we have infoz. +// 2) PerTableSeed::kBitCount bits for the seed. +class HashtableSize { + public: + static constexpr size_t kSizeBitCount = 64 - PerTableSeed::kBitCount - 1; - explicit GroupSse2Impl(const ctrl_t* pos) { - ctrl = _mm_loadu_si128(reinterpret_cast(pos)); - } + explicit HashtableSize(uninitialized_tag_t) {} + explicit HashtableSize(no_seed_empty_tag_t) : data_(0) {} + explicit HashtableSize(full_soo_tag_t) : data_(kSizeOneNoMetadata) {} - // Returns a bitmask representing the positions of slots that match hash. - BitMask Match(h2_t hash) const { - auto match = _mm_set1_epi8(static_cast(hash)); - return BitMask( - static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); + // Returns actual size of the table. + size_t size() const { return static_cast(data_ >> kSizeShift); } + void increment_size() { data_ += kSizeOneNoMetadata; } + void increment_size(size_t size) { + data_ += static_cast(size) * kSizeOneNoMetadata; } + void decrement_size() { data_ -= kSizeOneNoMetadata; } + // Returns true if the table is empty. + bool empty() const { return data_ < kSizeOneNoMetadata; } + // Sets the size to zero, but keeps all the metadata bits. + void set_size_to_zero_keep_metadata() { data_ = data_ & kMetadataMask; } - // Returns a bitmask representing the positions of empty slots. - NonIterableBitMask MaskEmpty() const { -#ifdef ABSL_INTERNAL_HAVE_SSSE3 - // This only works because ctrl_t::kEmpty is -128. - return NonIterableBitMask( - static_cast(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)))); -#else - auto match = _mm_set1_epi8(static_cast(ctrl_t::kEmpty)); - return NonIterableBitMask( - static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); -#endif + PerTableSeed seed() const { + return PerTableSeed(static_cast(data_) & kSeedMask); } - // Returns a bitmask representing the positions of full slots. - // Note: for `is_small()` tables group may contain the "same" slot twice: - // original and mirrored. - BitMask MaskFull() const { - return BitMask( - static_cast(_mm_movemask_epi8(ctrl) ^ 0xffff)); + void generate_new_seed() { + data_ = (data_ & ~kSeedMask) ^ uint64_t{NextSeed()}; } - // Returns a bitmask representing the positions of non full slots. - // Note: this includes: kEmpty, kDeleted, kSentinel. - // It is useful in contexts when kSentinel is not present. - auto MaskNonFull() const { - return BitMask( - static_cast(_mm_movemask_epi8(ctrl))); - } - - // Returns a bitmask representing the positions of empty or deleted slots. - NonIterableBitMask MaskEmptyOrDeleted() const { - auto special = _mm_set1_epi8(static_cast(ctrl_t::kSentinel)); - return NonIterableBitMask(static_cast( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)))); + // Returns true if the table has infoz. + bool has_infoz() const { + return ABSL_PREDICT_FALSE((data_ & kHasInfozMask) != 0); } - // Returns the number of trailing empty or deleted elements in the group. - uint32_t CountLeadingEmptyOrDeleted() const { - auto special = _mm_set1_epi8(static_cast(ctrl_t::kSentinel)); - return TrailingZeros(static_cast( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1)); - } + // Sets the has_infoz bit. + void set_has_infoz() { data_ |= kHasInfozMask; } - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - auto msbs = _mm_set1_epi8(static_cast(-128)); - auto x126 = _mm_set1_epi8(126); -#ifdef ABSL_INTERNAL_HAVE_SSSE3 - auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); -#else - auto zero = _mm_setzero_si128(); - auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); - auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); -#endif - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); - } + void set_no_seed_for_testing() { data_ &= ~kSeedMask; } - __m128i ctrl; -}; -#endif // ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 - -#if defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) -struct GroupAArch64Impl { - static constexpr size_t kWidth = 8; - - explicit GroupAArch64Impl(const ctrl_t* pos) { - ctrl = vld1_u8(reinterpret_cast(pos)); - } - - auto Match(h2_t hash) const { - uint8x8_t dup = vdup_n_u8(hash); - auto mask = vceq_u8(ctrl, dup); - return BitMask( - vget_lane_u64(vreinterpret_u64_u8(mask), 0)); - } - - NonIterableBitMask MaskEmpty() const { - uint64_t mask = - vget_lane_u64(vreinterpret_u64_u8(vceq_s8( - vdup_n_s8(static_cast(ctrl_t::kEmpty)), - vreinterpret_s8_u8(ctrl))), - 0); - return NonIterableBitMask(mask); - } - - // Returns a bitmask representing the positions of full slots. - // Note: for `is_small()` tables group may contain the "same" slot twice: - // original and mirrored. - auto MaskFull() const { - uint64_t mask = vget_lane_u64( - vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl), - vdup_n_s8(static_cast(0)))), - 0); - return BitMask(mask); - } - - // Returns a bitmask representing the positions of non full slots. - // Note: this includes: kEmpty, kDeleted, kSentinel. - // It is useful in contexts when kSentinel is not present. - auto MaskNonFull() const { - uint64_t mask = vget_lane_u64( - vreinterpret_u64_u8(vclt_s8(vreinterpret_s8_u8(ctrl), - vdup_n_s8(static_cast(0)))), - 0); - return BitMask(mask); - } - - NonIterableBitMask MaskEmptyOrDeleted() const { - uint64_t mask = - vget_lane_u64(vreinterpret_u64_u8(vcgt_s8( - vdup_n_s8(static_cast(ctrl_t::kSentinel)), - vreinterpret_s8_u8(ctrl))), - 0); - return NonIterableBitMask(mask); - } - - uint32_t CountLeadingEmptyOrDeleted() const { - uint64_t mask = - vget_lane_u64(vreinterpret_u64_u8(vcle_s8( - vdup_n_s8(static_cast(ctrl_t::kSentinel)), - vreinterpret_s8_u8(ctrl))), - 0); - // Similar to MaskEmptyorDeleted() but we invert the logic to invert the - // produced bitfield. We then count number of trailing zeros. - // Clang and GCC optimize countr_zero to rbit+clz without any check for 0, - // so we should be fine. - return static_cast(countr_zero(mask)) >> 3; - } - - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0); - constexpr uint64_t slsbs = 0x0202020202020202ULL; - constexpr uint64_t midbs = 0x7e7e7e7e7e7e7e7eULL; - auto x = slsbs & (mask >> 6); - auto res = (x + midbs) | kMsbs8Bytes; - little_endian::Store64(dst, res); - } - - uint8x8_t ctrl; + private: + static constexpr size_t kSizeShift = 64 - kSizeBitCount; + static constexpr uint64_t kSizeOneNoMetadata = uint64_t{1} << kSizeShift; + static constexpr uint64_t kMetadataMask = kSizeOneNoMetadata - 1; + static constexpr uint64_t kSeedMask = + (uint64_t{1} << PerTableSeed::kBitCount) - 1; + // The next bit after the seed. + static constexpr uint64_t kHasInfozMask = kSeedMask + 1; + uint64_t data_; }; -#endif // ABSL_INTERNAL_HAVE_ARM_NEON && ABSL_IS_LITTLE_ENDIAN - -struct GroupPortableImpl { - static constexpr size_t kWidth = 8; - - explicit GroupPortableImpl(const ctrl_t* pos) - : ctrl(little_endian::Load64(pos)) {} - - BitMask Match(h2_t hash) const { - // For the technique, see: - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord - // (Determine if a word has a byte equal to n). - // - // Caveat: there are false positives but: - // - they only occur if there is a real match - // - they never occur on ctrl_t::kEmpty, ctrl_t::kDeleted, ctrl_t::kSentinel - // - they will be handled gracefully by subsequent checks in code - // - // Example: - // v = 0x1716151413121110 - // hash = 0x12 - // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl ^ (lsbs * hash); - return BitMask((x - lsbs) & ~x & kMsbs8Bytes); - } - - NonIterableBitMask MaskEmpty() const { - return NonIterableBitMask((ctrl & ~(ctrl << 6)) & - kMsbs8Bytes); - } - - // Returns a bitmask representing the positions of full slots. - // Note: for `is_small()` tables group may contain the "same" slot twice: - // original and mirrored. - BitMask MaskFull() const { - return BitMask((ctrl ^ kMsbs8Bytes) & kMsbs8Bytes); - } - - // Returns a bitmask representing the positions of non full slots. - // Note: this includes: kEmpty, kDeleted, kSentinel. - // It is useful in contexts when kSentinel is not present. - auto MaskNonFull() const { - return BitMask(ctrl & kMsbs8Bytes); - } - - NonIterableBitMask MaskEmptyOrDeleted() const { - return NonIterableBitMask((ctrl & ~(ctrl << 7)) & - kMsbs8Bytes); - } - - uint32_t CountLeadingEmptyOrDeleted() const { - // ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and - // kDeleted. We lower all other bits and count number of trailing zeros. - constexpr uint64_t bits = 0x0101010101010101ULL; - return static_cast(countr_zero((ctrl | ~(ctrl >> 7)) & bits) >> - 3); - } - - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & kMsbs8Bytes; - auto res = (~x + (x >> 7)) & ~lsbs; - little_endian::Store64(dst, res); - } - uint64_t ctrl; -}; +// Extracts the H1 portion of a hash: 57 bits mixed with a per-table seed. +inline size_t H1(size_t hash, PerTableSeed seed) { + return (hash >> 7) ^ seed.seed(); +} -#ifdef ABSL_INTERNAL_HAVE_SSE2 -using Group = GroupSse2Impl; -using GroupFullEmptyOrDeleted = GroupSse2Impl; -#elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) -using Group = GroupAArch64Impl; -// For Aarch64, we use the portable implementation for counting and masking -// full, empty or deleted group elements. This is to avoid the latency of moving -// between data GPRs and Neon registers when it does not provide a benefit. -// Using Neon is profitable when we call Match(), but is not when we don't, -// which is the case when we do *EmptyOrDeleted and MaskFull operations. -// It is difficult to make a similar approach beneficial on other architectures -// such as x86 since they have much lower GPR <-> vector register transfer -// latency and 16-wide Groups. -using GroupFullEmptyOrDeleted = GroupPortableImpl; -#else -using Group = GroupPortableImpl; -using GroupFullEmptyOrDeleted = GroupPortableImpl; -#endif +// Extracts the H2 portion of a hash: the 7 bits not used for H1. +// +// These are used as an occupied control byte. +inline h2_t H2(size_t hash) { return hash & 0x7F; } // When there is an insertion with no reserved growth, we rehash with // probability `min(1, RehashProbabilityConstant() / capacity())`. Using a @@ -974,10 +571,10 @@ class CommonFieldsGenerationInfoEnabled { // references. We rehash on the first insertion after reserved_growth_ reaches // 0 after a call to reserve. We also do a rehash with low probability // whenever reserved_growth_ is zero. - bool should_rehash_for_bug_detection_on_insert(const ctrl_t* ctrl, + bool should_rehash_for_bug_detection_on_insert(PerTableSeed seed, size_t capacity) const; // Similar to above, except that we don't depend on reserved_growth_. - bool should_rehash_for_bug_detection_on_move(const ctrl_t* ctrl, + bool should_rehash_for_bug_detection_on_move(PerTableSeed seed, size_t capacity) const; void maybe_increment_generation_on_insert() { if (reserved_growth_ == kReservedGrowthJustRanOut) reserved_growth_ = 0; @@ -1031,10 +628,10 @@ class CommonFieldsGenerationInfoDisabled { CommonFieldsGenerationInfoDisabled& operator=( CommonFieldsGenerationInfoDisabled&&) = default; - bool should_rehash_for_bug_detection_on_insert(const ctrl_t*, size_t) const { + bool should_rehash_for_bug_detection_on_insert(PerTableSeed, size_t) const { return false; } - bool should_rehash_for_bug_detection_on_move(const ctrl_t*, size_t) const { + bool should_rehash_for_bug_detection_on_move(PerTableSeed, size_t) const { return false; } void maybe_increment_generation_on_insert() {} @@ -1127,9 +724,9 @@ class GrowthInfo { } // Overwrites several empty slots with full slots. - void OverwriteManyEmptyAsFull(size_t cnt) { - ABSL_SWISSTABLE_ASSERT(GetGrowthLeft() >= cnt); - growth_left_info_ -= cnt; + void OverwriteManyEmptyAsFull(size_t count) { + ABSL_SWISSTABLE_ASSERT(GetGrowthLeft() >= count); + growth_left_info_ -= count; } // Overwrites specified control element with full slot. @@ -1154,7 +751,14 @@ class GrowthInfo { // 2. There is no growth left. bool HasNoGrowthLeftAndNoDeleted() const { return growth_left_info_ == 0; } - // Returns true if table guaranteed to have no k + // Returns true if GetGrowthLeft() == 0, but must be called only if + // HasNoDeleted() is false. It is slightly more efficient. + bool HasNoGrowthLeftAssumingMayHaveDeleted() const { + ABSL_SWISSTABLE_ASSERT(!HasNoDeleted()); + return growth_left_info_ == kDeletedBit; + } + + // Returns true if table guaranteed to have no kDeleted slots. bool HasNoDeleted() const { return static_cast>(growth_left_info_) >= 0; } @@ -1175,7 +779,7 @@ static_assert(alignof(GrowthInfo) == alignof(size_t), ""); // Returns whether `n` is a valid capacity (i.e., number of slots). // // A valid capacity is a non-zero integer `2^m - 1`. -inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } +constexpr bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } // Returns the number of "cloned control bytes". // @@ -1191,26 +795,32 @@ constexpr size_t NumControlBytes(size_t capacity) { // Computes the offset from the start of the backing allocation of control. // infoz and growth_info are stored at the beginning of the backing array. -inline static size_t ControlOffset(bool has_infoz) { +constexpr size_t ControlOffset(bool has_infoz) { return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(GrowthInfo); } +// Returns the offset of the next item after `offset` that is aligned to `align` +// bytes. `align` must be a power of two. +constexpr size_t AlignUpTo(size_t offset, size_t align) { + return (offset + align - 1) & (~align + 1); +} + // Helper class for computing offsets and allocation size of hash set fields. class RawHashSetLayout { public: - explicit RawHashSetLayout(size_t capacity, size_t slot_align, bool has_infoz) - : capacity_(capacity), - control_offset_(ControlOffset(has_infoz)), + explicit RawHashSetLayout(size_t capacity, size_t slot_size, + size_t slot_align, bool has_infoz) + : control_offset_(ControlOffset(has_infoz)), generation_offset_(control_offset_ + NumControlBytes(capacity)), slot_offset_( - (generation_offset_ + NumGenerationBytes() + slot_align - 1) & - (~slot_align + 1)) { + AlignUpTo(generation_offset_ + NumGenerationBytes(), slot_align)), + alloc_size_(slot_offset_ + capacity * slot_size) { ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); + ABSL_SWISSTABLE_ASSERT( + slot_size <= + ((std::numeric_limits::max)() - slot_offset_) / capacity); } - // Returns the capacity of a table. - size_t capacity() const { return capacity_; } - // Returns precomputed offset from the start of the backing allocation of // control. size_t control_offset() const { return control_offset_; } @@ -1225,39 +835,17 @@ class RawHashSetLayout { // Given the capacity of a table, computes the total size of the backing // array. - size_t alloc_size(size_t slot_size) const { - ABSL_SWISSTABLE_ASSERT( - slot_size <= - ((std::numeric_limits::max)() - slot_offset_) / capacity_); - return slot_offset_ + capacity_ * slot_size; - } + size_t alloc_size() const { return alloc_size_; } private: - size_t capacity_; size_t control_offset_; size_t generation_offset_; size_t slot_offset_; + size_t alloc_size_; }; struct HashtableFreeFunctionsAccess; -// We only allow a maximum of 1 SOO element, which makes the implementation -// much simpler. Complications with multiple SOO elements include: -// - Satisfying the guarantee that erasing one element doesn't invalidate -// iterators to other elements means we would probably need actual SOO -// control bytes. -// - In order to prevent user code from depending on iteration order for small -// tables, we would need to randomize the iteration order somehow. -constexpr size_t SooCapacity() { return 1; } -// Sentinel type to indicate SOO CommonFields construction. -struct soo_tag_t {}; -// Sentinel type to indicate SOO CommonFields construction with full size. -struct full_soo_tag_t {}; -// Sentinel type to indicate non-SOO CommonFields construction. -struct non_soo_tag_t {}; -// Sentinel value to indicate an uninitialized CommonFields for use in swapping. -struct uninitialized_tag_t {}; - // Suppress erroneous uninitialized memory errors on GCC. For example, GCC // thinks that the call to slot_array() in find_or_prepare_insert() is reading // uninitialized memory, but slot_array is only called there when the table is @@ -1285,7 +873,7 @@ union MaybeInitializedPtr { }; struct HeapPtrs { - HeapPtrs() = default; + explicit HeapPtrs(uninitialized_tag_t) {} explicit HeapPtrs(ctrl_t* c) : control(c) {} // The control bytes (and, also, a pointer near to the base of the backing @@ -1304,10 +892,13 @@ struct HeapPtrs { MaybeInitializedPtr slot_array; }; +// Returns the maximum size of the SOO slot. +constexpr size_t MaxSooSlotSize() { return sizeof(HeapPtrs); } + // Manages the backing array pointers or the SOO slot. When raw_hash_set::is_soo // is true, the SOO slot is stored in `soo_data`. Otherwise, we use `heap`. union HeapOrSoo { - HeapOrSoo() = default; + explicit HeapOrSoo(uninitialized_tag_t) : heap(uninitialized_tag_t{}) {} explicit HeapOrSoo(ctrl_t* c) : heap(c) {} ctrl_t*& control() { @@ -1330,26 +921,50 @@ union HeapOrSoo { } HeapPtrs heap; - unsigned char soo_data[sizeof(HeapPtrs)]; + unsigned char soo_data[MaxSooSlotSize()]; }; +// Returns a reference to the GrowthInfo object stored immediately before +// `control`. +inline GrowthInfo& GetGrowthInfoFromControl(ctrl_t* control) { + auto* gl_ptr = reinterpret_cast(control) - 1; + ABSL_SWISSTABLE_ASSERT( + reinterpret_cast(gl_ptr) % alignof(GrowthInfo) == 0); + return *gl_ptr; +} + // CommonFields hold the fields in raw_hash_set that do not depend // on template parameters. This allows us to conveniently pass all // of this state to helper functions as a single argument. class CommonFields : public CommonFieldsGenerationInfo { public: - explicit CommonFields(soo_tag_t) : capacity_(SooCapacity()), size_(0) {} + explicit CommonFields(soo_tag_t) + : capacity_(SooCapacity()), + size_(no_seed_empty_tag_t{}), + heap_or_soo_(uninitialized_tag_t{}) {} explicit CommonFields(full_soo_tag_t) - : capacity_(SooCapacity()), size_(size_t{1} << HasInfozShift()) {} + : capacity_(SooCapacity()), + size_(full_soo_tag_t{}), + heap_or_soo_(uninitialized_tag_t{}) {} explicit CommonFields(non_soo_tag_t) - : capacity_(0), size_(0), heap_or_soo_(EmptyGroup()) {} + : capacity_(0), + size_(no_seed_empty_tag_t{}), + heap_or_soo_(EmptyGroup()) {} // For use in swapping. - explicit CommonFields(uninitialized_tag_t) {} + explicit CommonFields(uninitialized_tag_t) + : size_(uninitialized_tag_t{}), heap_or_soo_(uninitialized_tag_t{}) {} // Not copyable CommonFields(const CommonFields&) = delete; CommonFields& operator=(const CommonFields&) = delete; + // Copy with guarantee that it is not SOO. + CommonFields(non_soo_tag_t, const CommonFields& that) + : capacity_(that.capacity_), + size_(that.size_), + heap_or_soo_(that.heap_or_soo_) { + } + // Movable CommonFields(CommonFields&& that) = default; CommonFields& operator=(CommonFields&&) = default; @@ -1364,11 +979,21 @@ class CommonFields : public CommonFieldsGenerationInfo { const void* soo_data() const { return heap_or_soo_.get_soo_data(); } void* soo_data() { return heap_or_soo_.get_soo_data(); } - HeapOrSoo heap_or_soo() const { return heap_or_soo_; } - const HeapOrSoo& heap_or_soo_ref() const { return heap_or_soo_; } - ctrl_t* control() const { return heap_or_soo_.control(); } - void set_control(ctrl_t* c) { heap_or_soo_.control() = c; } + + // When we set the control bytes, we also often want to generate a new seed. + // So we bundle these two operations together to make sure we don't forget to + // generate a new seed. + // The table will be invalidated if + // `kGenerateSeed && !empty() && !is_single_group(capacity())` because H1 is + // being changed. In such cases, we will need to rehash the table. + template + void set_control(ctrl_t* c) { + heap_or_soo_.control() = c; + if constexpr (kGenerateSeed) { + generate_new_seed(); + } + } void* backing_array_start() const { // growth_info (and maybe infoz) is stored before control bytes. ABSL_SWISSTABLE_ASSERT( @@ -1382,26 +1007,39 @@ class CommonFields : public CommonFieldsGenerationInfo { void set_slots(void* s) { heap_or_soo_.slot_array().set(s); } // The number of filled slots. - size_t size() const { return size_ >> HasInfozShift(); } - void set_size(size_t s) { - size_ = (s << HasInfozShift()) | (size_ & HasInfozMask()); - } + size_t size() const { return size_.size(); } + // Sets the size to zero, but keeps hashinfoz bit and seed. + void set_size_to_zero() { size_.set_size_to_zero_keep_metadata(); } void set_empty_soo() { AssertInSooMode(); - size_ = 0; + size_ = HashtableSize(no_seed_empty_tag_t{}); } void set_full_soo() { AssertInSooMode(); - size_ = size_t{1} << HasInfozShift(); + size_ = HashtableSize(full_soo_tag_t{}); } void increment_size() { ABSL_SWISSTABLE_ASSERT(size() < capacity()); - size_ += size_t{1} << HasInfozShift(); + size_.increment_size(); + } + void increment_size(size_t n) { + ABSL_SWISSTABLE_ASSERT(size() + n <= capacity()); + size_.increment_size(n); } void decrement_size() { - ABSL_SWISSTABLE_ASSERT(size() > 0); - size_ -= size_t{1} << HasInfozShift(); + ABSL_SWISSTABLE_ASSERT(!empty()); + size_.decrement_size(); } + bool empty() const { return size_.empty(); } + + // The seed used for the H1 part of the hash function. + PerTableSeed seed() const { return size_.seed(); } + // Generates a new seed for the H1 part of the hash function. + // The table will be invalidated if + // `kGenerateSeed && !empty() && !is_single_group(capacity())` because H1 is + // being changed. In such cases, we will need to rehash the table. + void generate_new_seed() { size_.generate_new_seed(); } + void set_no_seed_for_testing() { size_.set_no_seed_for_testing(); } // The total number of available slots. size_t capacity() const { return capacity_; } @@ -1419,21 +1057,14 @@ class CommonFields : public CommonFieldsGenerationInfo { size_t growth_left() const { return growth_info().GetGrowthLeft(); } GrowthInfo& growth_info() { - auto* gl_ptr = reinterpret_cast(control()) - 1; - ABSL_SWISSTABLE_ASSERT( - reinterpret_cast(gl_ptr) % alignof(GrowthInfo) == 0); - return *gl_ptr; + return GetGrowthInfoFromControl(control()); } GrowthInfo growth_info() const { return const_cast(this)->growth_info(); } - bool has_infoz() const { - return ABSL_PREDICT_FALSE((size_ & HasInfozMask()) != 0); - } - void set_has_infoz(bool has_infoz) { - size_ = (size() << HasInfozShift()) | static_cast(has_infoz); - } + bool has_infoz() const { return size_.has_infoz(); } + void set_has_infoz() { size_.set_has_infoz(); } HashtablezInfoHandle infoz() { return has_infoz() @@ -1446,12 +1077,18 @@ class CommonFields : public CommonFieldsGenerationInfo { } bool should_rehash_for_bug_detection_on_insert() const { + if constexpr (!SwisstableGenerationsEnabled()) { + return false; + } + // As an optimization, we avoid calling ShouldRehashForBugDetection if we + // will end up rehashing anyways. + if (growth_left() == 0) return false; return CommonFieldsGenerationInfo:: - should_rehash_for_bug_detection_on_insert(control(), capacity()); + should_rehash_for_bug_detection_on_insert(seed(), capacity()); } bool should_rehash_for_bug_detection_on_move() const { return CommonFieldsGenerationInfo::should_rehash_for_bug_detection_on_move( - control(), capacity()); + seed(), capacity()); } void reset_reserved_growth(size_t reservation) { CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size()); @@ -1459,8 +1096,8 @@ class CommonFields : public CommonFieldsGenerationInfo { // The size of the backing array allocation. size_t alloc_size(size_t slot_size, size_t slot_align) const { - return RawHashSetLayout(capacity(), slot_align, has_infoz()) - .alloc_size(slot_size); + return RawHashSetLayout(capacity(), slot_size, slot_align, has_infoz()) + .alloc_size(); } // Move fields other than heap_or_soo_. @@ -1513,11 +1150,10 @@ class CommonFields : public CommonFieldsGenerationInfo { // regressions, presumably because we need capacity to do find operations. size_t capacity_; - // The size and also has one bit that stores whether we have infoz. // TODO(b/289225379): we could put size_ into HeapOrSoo and make capacity_ // encode the size in SOO case. We would be making size()/capacity() more // expensive in order to have more SOO space. - size_t size_; + HashtableSize size_; // Either the control/slots pointers or the SOO slot. HeapOrSoo heap_or_soo_; @@ -1527,11 +1163,17 @@ template class raw_hash_set; // Returns the next valid capacity after `n`. -inline size_t NextCapacity(size_t n) { +constexpr size_t NextCapacity(size_t n) { ABSL_SWISSTABLE_ASSERT(IsValidCapacity(n) || n == 0); return n * 2 + 1; } +// Returns the previous valid capacity before `n`. +constexpr size_t PreviousCapacity(size_t n) { + ABSL_SWISSTABLE_ASSERT(IsValidCapacity(n)); + return n / 2; +} + // Applies the following mapping to every byte in the control array: // * kDeleted -> kEmpty // * kEmpty -> kEmpty @@ -1543,19 +1185,10 @@ inline size_t NextCapacity(size_t n) { void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity); // Converts `n` into the next valid capacity, per `IsValidCapacity`. -inline size_t NormalizeCapacity(size_t n) { +constexpr size_t NormalizeCapacity(size_t n) { return n ? ~size_t{} >> countl_zero(n) : 1; } -template -size_t MaxValidCapacity() { - return NormalizeCapacity((std::numeric_limits::max)() / 4 / - kSlotSize); -} - -// Use a non-inlined function to avoid code bloat. -[[noreturn]] void HashTableSizeOverflow(); - // General notes on capacity/growth methods below: // - We use 7/8th as maximum load factor. For 16-wide groups, that gives an // average of two empty slots per group. @@ -1566,7 +1199,7 @@ size_t MaxValidCapacity() { // Given `capacity`, applies the load factor; i.e., it returns the maximum // number of values we should put into the table before a resizing rehash. -inline size_t CapacityToGrowth(size_t capacity) { +constexpr size_t CapacityToGrowth(size_t capacity) { ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); // `capacity*7/8` if (Group::kWidth == 8 && capacity == 7) { @@ -1576,18 +1209,28 @@ inline size_t CapacityToGrowth(size_t capacity) { return capacity - capacity / 8; } -// Given `growth`, "unapplies" the load factor to find how large the capacity +// Given `size`, "unapplies" the load factor to find how large the capacity // should be to stay within the load factor. // -// This might not be a valid capacity and `NormalizeCapacity()` should be -// called on this. -inline size_t GrowthToLowerboundCapacity(size_t growth) { - // `growth*8/7` - if (Group::kWidth == 8 && growth == 7) { - // x+(x-1)/7 does not work when x==7. - return 8; - } - return growth + static_cast((static_cast(growth) - 1) / 7); +// For size == 0, returns 0. +// For other values, returns the same as `NormalizeCapacity(size*8/7)`. +constexpr size_t SizeToCapacity(size_t size) { + if (size == 0) { + return 0; + } + // The minimum possible capacity is NormalizeCapacity(size). + // Shifting right `~size_t{}` by `leading_zeros` yields + // NormalizeCapacity(size). + int leading_zeros = absl::countl_zero(size); + constexpr size_t kLast3Bits = size_t{7} << (sizeof(size_t) * 8 - 3); + size_t max_size_for_next_capacity = kLast3Bits >> leading_zeros; + // Decrease shift if size is too big for the minimum capacity. + leading_zeros -= static_cast(size > max_size_for_next_capacity); + if constexpr (Group::kWidth == 8) { + // Formula doesn't work when size==7 for 8-wide groups. + leading_zeros -= (size == 7); + } + return (~size_t{}) >> leading_zeros; } template @@ -1596,12 +1239,9 @@ size_t SelectBucketCountForIterRange(InputIter first, InputIter last, if (bucket_count != 0) { return bucket_count; } - using InputIterCategory = - typename std::iterator_traits::iterator_category; - if (std::is_base_of::value) { - return GrowthToLowerboundCapacity( - static_cast(std::distance(first, last))); + if (base_internal::IsAtLeastIterator()) { + return SizeToCapacity(static_cast(std::distance(first, last))); } return 0; } @@ -1674,7 +1314,7 @@ inline void AssertIsValidForComparison(const ctrl_t* ctrl, FATAL, "Invalid iterator comparison. The element was likely erased."); } } else { - ABSL_HARDENING_ASSERT( + ABSL_HARDENING_ASSERT_SLOW( ctrl_is_valid_for_comparison && "Invalid iterator comparison. The element might have been erased or " "the table might have rehashed. Consider running with --config=asan to " @@ -1772,33 +1412,22 @@ struct FindInfo { size_t probe_length; }; -// Whether a table is "small". A small table fits entirely into a probing -// group, i.e., has a capacity < `Group::kWidth`. -// -// In small mode we are able to use the whole capacity. The extra control -// bytes give us at least one "empty" control byte to stop the iteration. -// This is important to make 1 a valid capacity. -// -// In small mode only the first `capacity` control bytes after the sentinel -// are valid. The rest contain dummy ctrl_t::kEmpty values that do not -// represent a real slot. This is important to take into account on -// `find_first_non_full()`, where we never try -// `ShouldInsertBackwards()` for small tables. -inline bool is_small(size_t capacity) { return capacity < Group::kWidth - 1; } - // Whether a table fits entirely into a probing group. // Arbitrary order of elements in such tables is correct. -inline bool is_single_group(size_t capacity) { +constexpr bool is_single_group(size_t capacity) { return capacity <= Group::kWidth; } // Begins a probing operation on `common.control`, using `hash`. -inline probe_seq probe(const ctrl_t* ctrl, const size_t capacity, +inline probe_seq probe(size_t h1, size_t capacity) { + return probe_seq(h1, capacity); +} +inline probe_seq probe(PerTableSeed seed, size_t capacity, size_t hash) { - return probe_seq(H1(hash, ctrl), capacity); + return probe(H1(hash, seed), capacity); } inline probe_seq probe(const CommonFields& common, size_t hash) { - return probe(common.control(), common.capacity(), hash); + return probe(common.seed(), common.capacity(), hash); } // Probes an array of control bits using a probe sequence derived from `hash`, @@ -1808,51 +1437,70 @@ inline probe_seq probe(const CommonFields& common, size_t hash) { // // NOTE: this function must work with tables having both empty and deleted // slots in the same group. Such tables appear during `erase()`. +FindInfo find_first_non_full(const CommonFields& common, size_t hash); + +constexpr size_t kProbedElementIndexSentinel = ~size_t{}; + +// Implementation detail of transfer_unprobed_elements_to_next_capacity_fn. +// Tries to find the new index for an element whose hash corresponds to +// `h1` for growth to the next capacity. +// Returns kProbedElementIndexSentinel if full probing is required. +// +// If element is located in the first probing group in the table before growth, +// returns one of two positions: `old_index` or `old_index + old_capacity + 1`. +// +// Otherwise, we will try to insert it into the first probe group of the new +// table. We only attempt to do so if the first probe group is already +// initialized. template -inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { - auto seq = probe(common, hash); - const ctrl_t* ctrl = common.control(); - if (IsEmptyOrDeleted(ctrl[seq.offset()]) && - !ShouldInsertBackwards(common.capacity(), hash, ctrl)) { - return {seq.offset(), /*probe_length=*/0}; - } - while (true) { - GroupFullEmptyOrDeleted g{ctrl + seq.offset()}; - auto mask = g.MaskEmptyOrDeleted(); - if (mask) { - return { - seq.offset(GetInsertionOffset(mask, common.capacity(), hash, ctrl)), - seq.index()}; - } - seq.next(); - ABSL_SWISSTABLE_ASSERT(seq.index() <= common.capacity() && "full table!"); - } +inline size_t TryFindNewIndexWithoutProbing(size_t h1, size_t old_index, + size_t old_capacity, + ctrl_t* new_ctrl, + size_t new_capacity) { + size_t index_diff = old_index - h1; + // The first probe group starts with h1 & capacity. + // All following groups start at (h1 + Group::kWidth * K) & capacity. + // We can find an index within the floating group as index_diff modulo + // Group::kWidth. + // Both old and new capacity are larger than Group::kWidth so we can avoid + // computing `& capacity`. + size_t in_floating_group_index = index_diff & (Group::kWidth - 1); + // By subtracting we will get the difference between the first probe group + // and the probe group corresponding to old_index. + index_diff -= in_floating_group_index; + if (ABSL_PREDICT_TRUE((index_diff & old_capacity) == 0)) { + size_t new_index = (h1 + in_floating_group_index) & new_capacity; + ABSL_ASSUME(new_index != kProbedElementIndexSentinel); + return new_index; + } + ABSL_SWISSTABLE_ASSERT(((old_index - h1) & old_capacity) >= Group::kWidth); + // Try to insert element into the first probe group. + // new_ctrl is not yet fully initialized so we can't use regular search via + // find_first_non_full. + + // We can search in the first probe group only if it is located in already + // initialized part of the table. + if (ABSL_PREDICT_FALSE((h1 & old_capacity) >= old_index)) { + return kProbedElementIndexSentinel; + } + size_t offset = h1 & new_capacity; + Group new_g(new_ctrl + offset); + if (auto mask = new_g.MaskNonFull(); ABSL_PREDICT_TRUE(mask)) { + size_t result = offset + mask.LowestBitSet(); + ABSL_ASSUME(result != kProbedElementIndexSentinel); + return result; + } + return kProbedElementIndexSentinel; } -// Extern template for inline function keep possibility of inlining. +// Extern template for inline function keeps possibility of inlining. // When compiler decided to not inline, no symbols will be added to the // corresponding translation unit. -extern template FindInfo find_first_non_full(const CommonFields&, size_t); - -// Non-inlined version of find_first_non_full for use in less -// performance critical routines. -FindInfo find_first_non_full_outofline(const CommonFields&, size_t); - -inline void ResetGrowthLeft(CommonFields& common) { - common.growth_info().InitGrowthLeftNoDeleted( - CapacityToGrowth(common.capacity()) - common.size()); -} - -// Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire -// array as marked as empty. -inline void ResetCtrl(CommonFields& common, size_t slot_size) { - const size_t capacity = common.capacity(); - ctrl_t* ctrl = common.control(); - std::memset(ctrl, static_cast(ctrl_t::kEmpty), - capacity + 1 + NumClonedBytes()); - ctrl[capacity] = ctrl_t::kSentinel; - SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity); -} +extern template size_t TryFindNewIndexWithoutProbing(size_t h1, + size_t old_index, + size_t old_capacity, + ctrl_t* new_ctrl, + size_t new_capacity); // Sets sanitizer poisoning for slot corresponding to control byte being set. inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h, @@ -1899,6 +1547,22 @@ inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h, SetCtrlInSingleGroupTable(c, i, static_cast(h), slot_size); } +// Like SetCtrl, but in a table with capacity >= Group::kWidth - 1, +// we can save some operations when setting the cloned control byte. +inline void SetCtrlInLargeTable(const CommonFields& c, size_t i, ctrl_t h, + size_t slot_size) { + ABSL_SWISSTABLE_ASSERT(c.capacity() >= Group::kWidth - 1); + DoSanitizeOnSetCtrl(c, i, h, slot_size); + ctrl_t* ctrl = c.control(); + ctrl[i] = h; + ctrl[((i - NumClonedBytes()) & c.capacity()) + NumClonedBytes()] = h; +} +// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. +inline void SetCtrlInLargeTable(const CommonFields& c, size_t i, h2_t h, + size_t slot_size) { + SetCtrlInLargeTable(c, i, static_cast(h), slot_size); +} + // growth_info (which is a size_t) is stored with the backing array. constexpr size_t BackingArrayAlignment(size_t align_of_slot) { return (std::max)(align_of_slot, alignof(GrowthInfo)); @@ -1911,423 +1575,283 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { (slot * slot_size)); } -// Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`. -// No insertion to the table allowed during Callback call. +// Iterates over all full slots and calls `cb(const ctrl_t*, void*)`. +// No insertion to the table is allowed during `cb` call. // Erasure is allowed only for the element passed to the callback. -template -ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( - const CommonFields& c, SlotType* slot, Callback cb) { - const size_t cap = c.capacity(); - const ctrl_t* ctrl = c.control(); - if (is_small(cap)) { - // Mirrored/cloned control bytes in small table are also located in the - // first group (starting from position 0). We are taking group from position - // `capacity` in order to avoid duplicates. - - // Small tables capacity fits into portable group, where - // GroupPortableImpl::MaskFull is more efficient for the - // capacity <= GroupPortableImpl::kWidth. - ABSL_SWISSTABLE_ASSERT(cap <= GroupPortableImpl::kWidth && - "unexpectedly large small capacity"); - static_assert(Group::kWidth >= GroupPortableImpl::kWidth, - "unexpected group width"); - // Group starts from kSentinel slot, so indices in the mask will - // be increased by 1. - const auto mask = GroupPortableImpl(ctrl + cap).MaskFull(); - --ctrl; - --slot; - for (uint32_t i : mask) { - cb(ctrl + i, slot + i); - } - return; - } - size_t remaining = c.size(); - ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining; - while (remaining != 0) { - for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { - ABSL_SWISSTABLE_ASSERT(IsFull(ctrl[i]) && - "hash table was modified unexpectedly"); - cb(ctrl + i, slot + i); - --remaining; - } - ctrl += Group::kWidth; - slot += Group::kWidth; - ABSL_SWISSTABLE_ASSERT( - (remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && - "hash table was modified unexpectedly"); - } - // NOTE: erasure of the current element is allowed in callback for - // absl::erase_if specialization. So we use `>=`. - ABSL_SWISSTABLE_ASSERT(original_size_for_assert >= c.size() && - "hash table was modified unexpectedly"); -} +// The table must not be in SOO mode. +void IterateOverFullSlots(const CommonFields& c, size_t slot_size, + absl::FunctionRef cb); template -constexpr bool ShouldSampleHashtablezInfo() { +constexpr bool ShouldSampleHashtablezInfoForAlloc() { // Folks with custom allocators often make unwarranted assumptions about the // behavior of their classes vis-a-vis trivial destructability and what // calls they will or won't make. Avoid sampling for people with custom // allocators to get us out of this mess. This is not a hard guarantee but // a workaround while we plan the exact guarantee we want to provide. - return std::is_same>::value; + return std::is_same_v>; } template -HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, size_t sizeof_key, - size_t sizeof_value, - size_t old_capacity, bool was_soo, - HashtablezInfoHandle forced_infoz, - CommonFields& c) { - if (forced_infoz.IsSampled()) return forced_infoz; +bool ShouldSampleHashtablezInfoOnResize(bool force_sampling, + bool is_hashtablez_eligible, + size_t old_capacity, CommonFields& c) { + if (!is_hashtablez_eligible) return false; + // Force sampling is only allowed for SOO tables. + ABSL_SWISSTABLE_ASSERT(kSooEnabled || !force_sampling); + if (kSooEnabled && force_sampling) { + return true; + } // In SOO, we sample on the first insertion so if this is an empty SOO case // (e.g. when reserve is called), then we still need to sample. - if (kSooEnabled && was_soo && c.size() == 0) { - return Sample(sizeof_slot, sizeof_key, sizeof_value, SooCapacity()); + if (kSooEnabled && old_capacity == SooCapacity() && c.empty()) { + return ShouldSampleNextTable(); } - // For non-SOO cases, we sample whenever the capacity is increasing from zero - // to non-zero. if (!kSooEnabled && old_capacity == 0) { - return Sample(sizeof_slot, sizeof_key, sizeof_value, 0); + return ShouldSampleNextTable(); } - return c.infoz(); + return false; } -// Helper class to perform resize of the hash set. -// -// It contains special optimizations for small group resizes. -// See GrowIntoSingleGroupShuffleControlBytes for details. -class HashSetResizeHelper { - public: - explicit HashSetResizeHelper(CommonFields& c, bool was_soo, bool had_soo_slot, - HashtablezInfoHandle forced_infoz) - : old_capacity_(c.capacity()), - had_infoz_(c.has_infoz()), - was_soo_(was_soo), - had_soo_slot_(had_soo_slot), - forced_infoz_(forced_infoz) {} - - // Optimized for small groups version of `find_first_non_full`. - // Beneficial only right after calling `raw_hash_set::resize`. - // It is safe to call in case capacity is big or was not changed, but there - // will be no performance benefit. - // It has implicit assumption that `resize` will call - // `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`. - // Falls back to `find_first_non_full` in case of big groups. - static FindInfo FindFirstNonFullAfterResize(const CommonFields& c, - size_t old_capacity, size_t hash); - - HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; } - void* old_soo_data() { return old_heap_or_soo_.get_soo_data(); } - ctrl_t* old_ctrl() const { - ABSL_SWISSTABLE_ASSERT(!was_soo_); - return old_heap_or_soo_.control(); - } - void* old_slots() const { - ABSL_SWISSTABLE_ASSERT(!was_soo_); - return old_heap_or_soo_.slot_array().get(); - } - size_t old_capacity() const { return old_capacity_; } - - // Returns the index of the SOO slot when growing from SOO to non-SOO in a - // single group. See also InitControlBytesAfterSoo(). It's important to use - // index 1 so that when resizing from capacity 1 to 3, we can still have - // random iteration order between the first two inserted elements. - // I.e. it allows inserting the second element at either index 0 or 2. - static size_t SooSlotIndex() { return 1; } - - // Allocates a backing array for the hashtable. - // Reads `capacity` and updates all other fields based on the result of - // the allocation. - // - // It also may do the following actions: - // 1. initialize control bytes - // 2. initialize slots - // 3. deallocate old slots. - // - // We are bundling a lot of functionality - // in one ABSL_ATTRIBUTE_NOINLINE function in order to minimize binary code - // duplication in raw_hash_set<>::resize. - // - // `c.capacity()` must be nonzero. - // POSTCONDITIONS: - // 1. CommonFields is initialized. - // - // if IsGrowingIntoSingleGroupApplicable && TransferUsesMemcpy - // Both control bytes and slots are fully initialized. - // old_slots are deallocated. - // infoz.RecordRehash is called. - // - // if IsGrowingIntoSingleGroupApplicable && !TransferUsesMemcpy - // Control bytes are fully initialized. - // infoz.RecordRehash is called. - // GrowSizeIntoSingleGroup must be called to finish slots initialization. - // - // if !IsGrowingIntoSingleGroupApplicable - // Control bytes are initialized to empty table via ResetCtrl. - // raw_hash_set<>::resize must insert elements regularly. - // infoz.RecordRehash is called if old_capacity == 0. - // - // Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation. - template - ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc, - ctrl_t soo_slot_h2, - size_t key_size, - size_t value_size) { - ABSL_SWISSTABLE_ASSERT(c.capacity()); - HashtablezInfoHandle infoz = - ShouldSampleHashtablezInfo() - ? SampleHashtablezInfo(SizeOfSlot, key_size, value_size, - old_capacity_, was_soo_, - forced_infoz_, c) - : HashtablezInfoHandle{}; - - const bool has_infoz = infoz.IsSampled(); - RawHashSetLayout layout(c.capacity(), AlignOfSlot, has_infoz); - char* mem = static_cast(Allocate( - &alloc, layout.alloc_size(SizeOfSlot))); - const GenerationType old_generation = c.generation(); - c.set_generation_ptr( - reinterpret_cast(mem + layout.generation_offset())); - c.set_generation(NextGeneration(old_generation)); - c.set_control(reinterpret_cast(mem + layout.control_offset())); - c.set_slots(mem + layout.slot_offset()); - ResetGrowthLeft(c); - - const bool grow_single_group = - IsGrowingIntoSingleGroupApplicable(old_capacity_, layout.capacity()); - if (SooEnabled && was_soo_ && grow_single_group) { - InitControlBytesAfterSoo(c.control(), soo_slot_h2, layout.capacity()); - if (TransferUsesMemcpy && had_soo_slot_) { - TransferSlotAfterSoo(c, SizeOfSlot); - } - // SooEnabled implies that old_capacity_ != 0. - } else if ((SooEnabled || old_capacity_ != 0) && grow_single_group) { - if (TransferUsesMemcpy) { - GrowSizeIntoSingleGroupTransferable(c, SizeOfSlot); - DeallocateOld(alloc, SizeOfSlot); - } else { - GrowIntoSingleGroupShuffleControlBytes(c.control(), layout.capacity()); - } - } else { - ResetCtrl(c, SizeOfSlot); - } - - c.set_has_infoz(has_infoz); - if (has_infoz) { - infoz.RecordStorageChanged(c.size(), layout.capacity()); - if ((SooEnabled && was_soo_) || grow_single_group || old_capacity_ == 0) { - infoz.RecordRehash(0); - } - c.set_infoz(infoz); - } - return grow_single_group; - } - - // Relocates slots into new single group consistent with - // GrowIntoSingleGroupShuffleControlBytes. - // - // PRECONDITIONS: - // 1. GrowIntoSingleGroupShuffleControlBytes was already called. - template - void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref) { - ABSL_SWISSTABLE_ASSERT(old_capacity_ < Group::kWidth / 2); - ABSL_SWISSTABLE_ASSERT( - IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); - using slot_type = typename PolicyTraits::slot_type; - ABSL_SWISSTABLE_ASSERT(is_single_group(c.capacity())); - - auto* new_slots = static_cast(c.slot_array()) + 1; - auto* old_slots_ptr = static_cast(old_slots()); - auto* old_ctrl_ptr = old_ctrl(); - - for (size_t i = 0; i < old_capacity_; ++i, ++new_slots) { - if (IsFull(old_ctrl_ptr[i])) { - SanitizerUnpoisonMemoryRegion(new_slots, sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref, new_slots, old_slots_ptr + i); - } - } - PoisonSingleGroupEmptySlots(c, sizeof(slot_type)); - } - - // Deallocates old backing array. - template - void DeallocateOld(CharAlloc alloc_ref, size_t slot_size) { - SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_); - auto layout = RawHashSetLayout(old_capacity_, AlignOfSlot, had_infoz_); - Deallocate( - &alloc_ref, old_ctrl() - layout.control_offset(), - layout.alloc_size(slot_size)); - } - - private: - // Returns true if `GrowSizeIntoSingleGroup` can be used for resizing. - static bool IsGrowingIntoSingleGroupApplicable(size_t old_capacity, - size_t new_capacity) { - // NOTE that `old_capacity < new_capacity` in order to have - // `old_capacity < Group::kWidth / 2` to make faster copies of 8 bytes. - return is_single_group(new_capacity) && old_capacity < new_capacity; - } - - // Relocates control bytes and slots into new single group for - // transferable objects. - // Must be called only if IsGrowingIntoSingleGroupApplicable returned true. - void GrowSizeIntoSingleGroupTransferable(CommonFields& c, size_t slot_size); - - // If there was an SOO slot and slots are transferable, transfers the SOO slot - // into the new heap allocation. Must be called only if - // IsGrowingIntoSingleGroupApplicable returned true. - void TransferSlotAfterSoo(CommonFields& c, size_t slot_size); - - // Shuffle control bits deterministically to the next capacity. - // Returns offset for newly added element with given hash. - // - // PRECONDITIONs: - // 1. new_ctrl is allocated for new_capacity, - // but not initialized. - // 2. new_capacity is a single group. - // 3. old_capacity > 0. - // - // All elements are transferred into the first `old_capacity + 1` positions - // of the new_ctrl. Elements are shifted by 1 in order to keep a space at the - // beginning for the new element. - // Position of the new added element will be based on `H1` and is not - // deterministic. - // - // Examples: - // S = kSentinel, E = kEmpty - // - // old_ctrl = 0SEEEEEEE... - // new_ctrl = E0ESE0EEE... - // - // old_ctrl = 012S012EEEEEEEEE... - // new_ctrl = E012EEESE012EEE... - // - // old_ctrl = 0123456S0123456EEEEEEEEEEE... - // new_ctrl = E0123456EEEEEESE0123456EEE... - void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* new_ctrl, - size_t new_capacity) const; - - // If the table was SOO, initializes new control bytes. `h2` is the control - // byte corresponding to the full slot. Must be called only if - // IsGrowingIntoSingleGroupApplicable returned true. - // Requires: `had_soo_slot_ || h2 == ctrl_t::kEmpty`. - void InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2, - size_t new_capacity); - - // Shuffle trivially transferable slots in the way consistent with - // GrowIntoSingleGroupShuffleControlBytes. - // - // PRECONDITIONs: - // 1. old_capacity must be non-zero. - // 2. new_ctrl is fully initialized using - // GrowIntoSingleGroupShuffleControlBytes. - // 3. new_slots is allocated and *not* poisoned. - // - // POSTCONDITIONS: - // 1. new_slots are transferred from old_slots_ consistent with - // GrowIntoSingleGroupShuffleControlBytes. - // 2. Empty new_slots are *not* poisoned. - void GrowIntoSingleGroupShuffleTransferableSlots(void* new_slots, - size_t slot_size) const; - - // Poison empty slots that were transferred using the deterministic algorithm - // described above. - // PRECONDITIONs: - // 1. new_ctrl is fully initialized using - // GrowIntoSingleGroupShuffleControlBytes. - // 2. new_slots is fully initialized consistent with - // GrowIntoSingleGroupShuffleControlBytes. - void PoisonSingleGroupEmptySlots(CommonFields& c, size_t slot_size) const { - // poison non full items - for (size_t i = 0; i < c.capacity(); ++i) { - if (!IsFull(c.control()[i])) { - SanitizerPoisonMemoryRegion(SlotAddress(c.slot_array(), i, slot_size), - slot_size); - } - } - } - - HeapOrSoo old_heap_or_soo_; - size_t old_capacity_; - bool had_infoz_; - bool was_soo_; - bool had_soo_slot_; - // Either null infoz or a pre-sampled forced infoz for SOO tables. - HashtablezInfoHandle forced_infoz_; -}; - -inline void PrepareInsertCommon(CommonFields& common) { - common.increment_size(); - common.maybe_increment_generation_on_insert(); +// Allocates `n` bytes for a backing array. +template +ABSL_ATTRIBUTE_NOINLINE void* AllocateBackingArray(void* alloc, size_t n) { + return Allocate(static_cast(alloc), n); } -// Like prepare_insert, but for the case of inserting into a full SOO table. -size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, - CommonFields& common); +template +ABSL_ATTRIBUTE_NOINLINE void DeallocateBackingArray( + void* alloc, size_t capacity, ctrl_t* ctrl, size_t slot_size, + size_t slot_align, bool had_infoz) { + RawHashSetLayout layout(capacity, slot_size, slot_align, had_infoz); + void* backing_array = ctrl - layout.control_offset(); + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(backing_array, layout.alloc_size()); + Deallocate(static_cast(alloc), backing_array, + layout.alloc_size()); +} // PolicyFunctions bundles together some information for a particular // raw_hash_set instantiation. This information is passed to // type-erased functions that want to do small amounts of type-specific // work. struct PolicyFunctions { - size_t slot_size; + uint32_t key_size; + uint32_t value_size; + uint32_t slot_size; + uint16_t slot_align; + bool soo_enabled; + bool is_hashtablez_eligible; // Returns the pointer to the hash function stored in the set. - const void* (*hash_fn)(const CommonFields& common); + void* (*hash_fn)(CommonFields& common); // Returns the hash of the pointed-to slot. size_t (*hash_slot)(const void* hash_fn, void* slot); - // Transfers the contents of src_slot to dst_slot. - void (*transfer)(void* set, void* dst_slot, void* src_slot); + // Transfers the contents of `count` slots from src_slot to dst_slot. + // We use ability to transfer several slots in single group table growth. + void (*transfer_n)(void* set, void* dst_slot, void* src_slot, size_t count); - // Deallocates the backing store from common. - void (*dealloc)(CommonFields& common, const PolicyFunctions& policy); + // Returns the pointer to the CharAlloc stored in the set. + void* (*get_char_alloc)(CommonFields& common); + + // Allocates n bytes for the backing store for common. + void* (*alloc)(void* alloc, size_t n); - // Resizes set to the new capacity. - // Arguments are used as in raw_hash_set::resize_impl. - void (*resize)(CommonFields& common, size_t new_capacity, - HashtablezInfoHandle forced_infoz); + // Deallocates the backing store from common. + void (*dealloc)(void* alloc, size_t capacity, ctrl_t* ctrl, size_t slot_size, + size_t slot_align, bool had_infoz); + + // Implementation detail of GrowToNextCapacity. + // Iterates over all full slots and transfers unprobed elements. + // Initializes the new control bytes except mirrored bytes and kSentinel. + // Caller must finish the initialization. + // All slots corresponding to the full control bytes are transferred. + // Probed elements are reported by `encode_probed_element` callback. + // encode_probed_element may overwrite old_ctrl buffer till source_offset. + // Different encoding is used depending on the capacity of the table. + // See ProbedItem*Bytes classes for details. + void (*transfer_unprobed_elements_to_next_capacity)( + CommonFields& common, const ctrl_t* old_ctrl, void* old_slots, + // TODO(b/382423690): Try to use absl::FunctionRef here. + void* probed_storage, + void (*encode_probed_element)(void* probed_storage, h2_t h2, + size_t source_offset, size_t h1)); + + uint8_t soo_capacity() const { + return static_cast(soo_enabled ? SooCapacity() : 0); + } }; +// Returns the maximum valid size for a table with 1-byte slots. +// This function is an utility shared by MaxValidSize and IsAboveValidSize. +// Template parameter is only used to enable testing. +template +constexpr size_t MaxValidSizeFor1ByteSlot() { + if constexpr (kSizeOfSizeT == 8) { + return CapacityToGrowth( + static_cast(uint64_t{1} << HashtableSize::kSizeBitCount) - 1); + } else { + static_assert(kSizeOfSizeT == 4); + return CapacityToGrowth((size_t{1} << (kSizeOfSizeT * 8 - 2)) - 1); + } +} + +// Returns the maximum valid size for a table with provided slot size. +// Template parameter is only used to enable testing. +template +constexpr size_t MaxValidSize(size_t slot_size) { + if constexpr (kSizeOfSizeT == 8) { + // For small slot sizes we are limited by HashtableSize::kSizeBitCount. + if (slot_size < size_t{1} << (64 - HashtableSize::kSizeBitCount)) { + return MaxValidSizeFor1ByteSlot(); + } + return (size_t{1} << (kSizeOfSizeT * 8 - 2)) / slot_size; + } else { + return MaxValidSizeFor1ByteSlot() / slot_size; + } +} + +// Returns true if size is larger than the maximum valid size. +// It is an optimization to avoid the division operation in the common case. +// Template parameter is only used to enable testing. +template +constexpr bool IsAboveValidSize(size_t size, size_t slot_size) { + if constexpr (kSizeOfSizeT == 8) { + // For small slot sizes we are limited by HashtableSize::kSizeBitCount. + if (ABSL_PREDICT_TRUE(slot_size < + (size_t{1} << (64 - HashtableSize::kSizeBitCount)))) { + return size > MaxValidSizeFor1ByteSlot(); + } + return size > MaxValidSize(slot_size); + } else { + return uint64_t{size} * slot_size > + MaxValidSizeFor1ByteSlot(); + } +} + +// Returns the index of the SOO slot when growing from SOO to non-SOO in a +// single group. See also InitializeSmallControlBytesAfterSoo(). It's important +// to use index 1 so that when resizing from capacity 1 to 3, we can still have +// random iteration order between the first two inserted elements. +// I.e. it allows inserting the second element at either index 0 or 2. +constexpr size_t SooSlotIndex() { return 1; } + +// Maximum capacity for the algorithm for small table after SOO. +// Note that typical size after SOO is 3, but we allow up to 7. +// Allowing till 16 would require additional store that can be avoided. +constexpr size_t MaxSmallAfterSooCapacity() { return 7; } + +// Type erased version of raw_hash_set::reserve. +// Requires: `new_size > policy.soo_capacity`. +void ReserveTableToFitNewSize(CommonFields& common, + const PolicyFunctions& policy, size_t new_size); + +// Resizes empty non-allocated table to the next valid capacity after +// `bucket_count`. Requires: +// 1. `c.capacity() == policy.soo_capacity`. +// 2. `c.empty()`. +// 3. `new_size > policy.soo_capacity`. +// The table will be attempted to be sampled. +void ReserveEmptyNonAllocatedTableToFitBucketCount( + CommonFields& common, const PolicyFunctions& policy, size_t bucket_count); + +// Type erased version of raw_hash_set::rehash. +void Rehash(CommonFields& common, const PolicyFunctions& policy, size_t n); + +// Type erased version of copy constructor. +void Copy(CommonFields& common, const PolicyFunctions& policy, + const CommonFields& other, + absl::FunctionRef copy_fn); + +// Returns the optimal size for memcpy when transferring SOO slot. +// Otherwise, returns the optimal size for memcpy SOO slot transfer +// to SooSlotIndex(). +// At the destination we are allowed to copy upto twice more bytes, +// because there is at least one more slot after SooSlotIndex(). +// The result must not exceed MaxSooSlotSize(). +// Some of the cases are merged to minimize the number of function +// instantiations. +constexpr size_t OptimalMemcpySizeForSooSlotTransfer( + size_t slot_size, size_t max_soo_slot_size = MaxSooSlotSize()) { + static_assert(MaxSooSlotSize() >= 8, "unexpectedly small SOO slot size"); + if (slot_size == 1) { + return 1; + } + if (slot_size <= 3) { + return 4; + } + // We are merging 4 and 8 into one case because we expect them to be the + // hottest cases. Copying 8 bytes is as fast on common architectures. + if (slot_size <= 8) { + return 8; + } + if (max_soo_slot_size <= 16) { + return max_soo_slot_size; + } + if (slot_size <= 16) { + return 16; + } + if (max_soo_slot_size <= 24) { + return max_soo_slot_size; + } + static_assert(MaxSooSlotSize() <= 24, "unexpectedly large SOO slot size"); + return 24; +} + +// Resizes SOO table to the NextCapacity(SooCapacity()) and prepares insert for +// the given new_hash. Returns the offset of the new element. +// `soo_slot_ctrl` is the control byte of the SOO slot. +// If soo_slot_ctrl is kEmpty +// 1. The table must be empty. +// 2. Table will be forced to be sampled. +// All possible template combinations are defined in cc file to improve +// compilation time. +template +size_t GrowSooTableToNextCapacityAndPrepareInsert(CommonFields& common, + const PolicyFunctions& policy, + size_t new_hash, + ctrl_t soo_slot_ctrl); + +// As `ResizeFullSooTableToNextCapacity`, except that we also force the SOO +// table to be sampled. SOO tables need to switch from SOO to heap in order to +// store the infoz. No-op if sampling is disabled or not possible. +void GrowFullSooTableToNextCapacityForceSampling(CommonFields& common, + const PolicyFunctions& policy); + +// Resizes table with allocated slots and change the table seed. +// Tables with SOO enabled must have capacity > policy.soo_capacity. +// No sampling will be performed since table is already allocated. +void ResizeAllocatedTableWithSeedChange(CommonFields& common, + const PolicyFunctions& policy, + size_t new_capacity); + // ClearBackingArray clears the backing array, either modifying it in place, // or creating a new one based on the value of "reuse". // REQUIRES: c.capacity > 0 void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, - bool reuse, bool soo_enabled); + void* alloc, bool reuse, bool soo_enabled); // Type-erased version of raw_hash_set::erase_meta_only. void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size); -// Function to place in PolicyFunctions::dealloc for raw_hash_sets -// that are using std::allocator. This allows us to share the same -// function body for raw_hash_set instantiations that have the -// same slot alignment. -template -ABSL_ATTRIBUTE_NOINLINE void DeallocateStandard(CommonFields& common, - const PolicyFunctions& policy) { - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(common.slot_array(), - policy.slot_size * common.capacity()); - - std::allocator alloc; - common.infoz().Unregister(); - Deallocate( - &alloc, common.backing_array_start(), - common.alloc_size(policy.slot_size, AlignOfSlot)); -} - // For trivially relocatable types we use memcpy directly. This allows us to // share the same function body for raw_hash_set instantiations that have the // same slot size as long as they are relocatable. +// Separate function for relocating single slot cause significant binary bloat. template -ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) { - memcpy(dst, src, SizeOfSlot); +ABSL_ATTRIBUTE_NOINLINE void TransferNRelocatable(void*, void* dst, void* src, + size_t count) { + // TODO(b/382423690): Experiment with making specialization for power of 2 and + // non power of 2. This would require passing the size of the slot. + memcpy(dst, src, SizeOfSlot * count); } -// Type erased raw_hash_set::get_hash_ref_fn for the empty hash function case. -const void* GetHashRefForEmptyHasher(const CommonFields& common); +// Returns a pointer to `common`. This is used to implement type erased +// raw_hash_set::get_hash_ref_fn and raw_hash_set::get_alloc_ref_fn for the +// empty class cases. +void* GetRefForEmptyClass(CommonFields& common); // Given the hash of a value not currently in the table and the first empty // slot in the probe sequence, finds a viable slot index to insert it at. @@ -2344,8 +1868,8 @@ const void* GetHashRefForEmptyHasher(const CommonFields& common); // REQUIRES: Table is not SOO. // REQUIRES: At least one non-full slot available. // REQUIRES: `target` is a valid empty position to insert. -size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target, - const PolicyFunctions& policy); +size_t PrepareInsertNonSoo(CommonFields& common, const PolicyFunctions& policy, + size_t hash, FindInfo target); // A SwissTable. // @@ -2376,9 +1900,6 @@ class raw_hash_set { public: using init_type = typename PolicyTraits::init_type; using key_type = typename PolicyTraits::key_type; - // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user - // code fixes! - using slot_type = typename PolicyTraits::slot_type; using allocator_type = Alloc; using size_type = size_t; using difference_type = ptrdiff_t; @@ -2393,6 +1914,7 @@ class raw_hash_set { using const_pointer = typename absl::allocator_traits< allocator_type>::template rebind_traits::const_pointer; + private: // Alias used for heterogeneous lookup functions. // `key_arg` evaluates to `K` when the functors are transparent and to // `key_type` otherwise. It permits template argument deduction on `K` for the @@ -2400,7 +1922,8 @@ class raw_hash_set { template using key_arg = typename KeyArgImpl::template type; - private: + using slot_type = typename PolicyTraits::slot_type; + // TODO(b/289225379): we could add extra SOO space inside raw_hash_set // after CommonFields to allow inlining larger slot_types (e.g. std::string), // but it's a bit complicated if we want to support incomplete mapped_type in @@ -2650,18 +2173,15 @@ class raw_hash_set { std::is_nothrow_default_constructible::value && std::is_nothrow_default_constructible::value) {} - ABSL_ATTRIBUTE_NOINLINE explicit raw_hash_set( + explicit raw_hash_set( size_t bucket_count, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) : settings_(CommonFields::CreateDefault(), hash, eq, alloc) { if (bucket_count > DefaultCapacity()) { - if (ABSL_PREDICT_FALSE(bucket_count > - MaxValidCapacity())) { - HashTableSizeOverflow(); - } - resize(NormalizeCapacity(bucket_count)); + ReserveEmptyNonAllocatedTableToFitBucketCount( + common(), GetPolicyFunctions(), bucket_count); } } @@ -2762,74 +2282,20 @@ class raw_hash_set { raw_hash_set(const raw_hash_set& that) : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} + allocator_type(that.char_alloc_ref()))) {} raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(GrowthToLowerboundCapacity(that.size()), that.hash_ref(), - that.eq_ref(), a) { + : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { that.AssertNotDebugCapacity(); - const size_t size = that.size(); - if (size == 0) { - return; - } - // We don't use `that.is_soo()` here because `that` can have non-SOO - // capacity but have a size that fits into SOO capacity. - if (fits_in_soo(size)) { - ABSL_SWISSTABLE_ASSERT(size == 1); - common().set_full_soo(); - emplace_at(soo_iterator(), *that.begin()); - const HashtablezInfoHandle infoz = try_sample_soo(); - if (infoz.IsSampled()) resize_with_soo_infoz(infoz); - return; - } - ABSL_SWISSTABLE_ASSERT(!that.is_soo()); - const size_t cap = capacity(); - // Note about single group tables: - // 1. It is correct to have any order of elements. - // 2. Order has to be non deterministic. - // 3. We are assigning elements with arbitrary `shift` starting from - // `capacity + shift` position. - // 4. `shift` must be coprime with `capacity + 1` in order to be able to use - // modular arithmetic to traverse all positions, instead if cycling - // through a subset of positions. Odd numbers are coprime with any - // `capacity + 1` (2^N). - size_t offset = cap; - const size_t shift = - is_single_group(cap) ? (PerTableSalt(control()) | 1) : 0; - IterateOverFullSlots( - that.common(), that.slot_array(), - [&](const ctrl_t* that_ctrl, - slot_type* that_slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { - if (shift == 0) { - // Big tables case. Position must be searched via probing. - // The table is guaranteed to be empty, so we can do faster than - // a full `insert`. - const size_t hash = PolicyTraits::apply( - HashElement{hash_ref()}, PolicyTraits::element(that_slot)); - FindInfo target = find_first_non_full_outofline(common(), hash); - infoz().RecordInsert(hash, target.probe_length); - offset = target.offset; - } else { - // Small tables case. Next position is computed via shift. - offset = (offset + shift) & cap; - } - const h2_t h2 = static_cast(*that_ctrl); - ABSL_SWISSTABLE_ASSERT( // We rely that hash is not changed for small - // tables. - H2(PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(that_slot))) == h2 && - "hash function value changed unexpectedly during the copy"); - SetCtrl(common(), offset, h2, sizeof(slot_type)); - emplace_at(iterator_at(offset), PolicyTraits::element(that_slot)); - common().maybe_increment_generation_on_insert(); - }); - if (shift != 0) { - // On small table copy we do not record individual inserts. - // RecordInsert requires hash, but it is unknown for small tables. - infoz().RecordStorageChanged(size, cap); - } - common().set_size(size); - growth_info().OverwriteManyEmptyAsFull(size); + if (that.empty()) return; + Copy(common(), GetPolicyFunctions(), that.common(), + [this](void* dst, const void* src) { + // TODO(b/413598253): type erase for trivially copyable types via + // PolicyTraits. + construct(to_slot(dst), + PolicyTraits::element( + static_cast(const_cast(src)))); + }); } ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept( @@ -2843,7 +2309,7 @@ class raw_hash_set { settings_(PolicyTraits::transfer_uses_memcpy() || !that.is_full_soo() ? std::move(that.common()) : CommonFields{full_soo_tag_t{}}, - that.hash_ref(), that.eq_ref(), that.alloc_ref()) { + that.hash_ref(), that.eq_ref(), that.char_alloc_ref()) { if (!PolicyTraits::transfer_uses_memcpy() && that.is_full_soo()) { transfer(soo_slot(), that.soo_slot()); } @@ -2854,7 +2320,7 @@ class raw_hash_set { raw_hash_set(raw_hash_set&& that, const allocator_type& a) : settings_(CommonFields::CreateDefault(), that.hash_ref(), that.eq_ref(), a) { - if (a == that.alloc_ref()) { + if (CharAlloc(a) == that.char_alloc_ref()) { swap_common(that); annotate_for_bug_detection_on_move(that); } else { @@ -2871,7 +2337,9 @@ class raw_hash_set { // is an exact match for that.size(). If this->capacity() is too big, then // it would make iteration very slow to reuse the allocation. Maybe we can // do the same heuristic as clear() and reuse if it's small enough. - raw_hash_set tmp(that, propagate_alloc ? that.alloc_ref() : alloc_ref()); + allocator_type alloc(propagate_alloc ? that.char_alloc_ref() + : char_alloc_ref()); + raw_hash_set tmp(that, alloc); // NOLINTNEXTLINE: not returning *this for performance. return assign_impl(std::move(tmp)); } @@ -2890,14 +2358,14 @@ class raw_hash_set { ~raw_hash_set() { destructor_impl(); -#ifndef NDEBUG - common().set_capacity(InvalidCapacity::kDestroyed); -#endif + if constexpr (SwisstableAssertAccessToDestroyedTable()) { + common().set_capacity(InvalidCapacity::kDestroyed); + } } iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (ABSL_PREDICT_FALSE(empty())) return end(); - if (is_soo()) return soo_iterator(); + if (capacity() == 1) return single_iterator(); iterator it = {control(), common().slots_union(), common().generation_ptr()}; it.skip_empty_or_deleted(); @@ -2933,9 +2401,7 @@ class raw_hash_set { ABSL_ASSUME(cap >= kDefaultCapacity); return cap; } - size_t max_size() const { - return CapacityToGrowth(MaxValidCapacity()); - } + size_t max_size() const { return MaxValidSize(sizeof(slot_type)); } ABSL_ATTRIBUTE_REINITIALIZES void clear() { if (SwisstableGenerationsEnabled() && @@ -2958,8 +2424,7 @@ class raw_hash_set { common().set_empty_soo(); } else { destroy_slots(); - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128, - SooEnabled()); + clear_backing_array(/*reuse=*/cap < 128); } common().set_reserved_growth(0); common().set_reservation_size(0); @@ -2971,15 +2436,15 @@ class raw_hash_set { // flat_hash_map m; // m.insert(std::make_pair("abc", 42)); template ::value && - IsNotBitField::value && - !IsLifetimeBoundAssignmentFrom::value, - int> = 0> + int = std::enable_if_t::value && + IsNotBitField::value && + !IsLifetimeBoundAssignmentFrom::value, + int>()> std::pair insert(T&& value) ABSL_ATTRIBUTE_LIFETIME_BOUND { return emplace(std::forward(value)); } - template ::value && IsNotBitField::value && IsLifetimeBoundAssignmentFrom::value, @@ -2987,7 +2452,7 @@ class raw_hash_set { std::pair insert( T&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return emplace(std::forward(value)); + return this->template insert(std::forward(value)); } // This overload kicks in when the argument is a bitfield or an lvalue of @@ -3001,22 +2466,22 @@ class raw_hash_set { // const char* p = "hello"; // s.insert(p); // - template ::value && !IsLifetimeBoundAssignmentFrom::value, - int> = 0> + int>()> std::pair insert(const T& value) ABSL_ATTRIBUTE_LIFETIME_BOUND { return emplace(value); } - template ::value && IsLifetimeBoundAssignmentFrom::value, int> = 0> std::pair insert( const T& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return emplace(value); + return this->template insert(value); } // This overload kicks in when the argument is an rvalue of init_type. Its @@ -3043,21 +2508,22 @@ class raw_hash_set { #endif template ::value && - IsNotBitField::value && - !IsLifetimeBoundAssignmentFrom::value, - int> = 0> + int = std::enable_if_t::value && + IsNotBitField::value && + !IsLifetimeBoundAssignmentFrom::value, + int>()> iterator insert(const_iterator, T&& value) ABSL_ATTRIBUTE_LIFETIME_BOUND { return insert(std::forward(value)).first; } - template ::value && IsNotBitField::value && IsLifetimeBoundAssignmentFrom::value, int> = 0> - iterator insert(const_iterator, T&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY( - this)) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return insert(std::forward(value)).first; + iterator insert(const_iterator hint, + T&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return this->template insert(hint, std::forward(value)); } template (f)(constructor(&alloc_ref(), &slot)); + allocator_type alloc(char_alloc_ref()); + std::forward(f)(constructor(&alloc, &slot)); ABSL_SWISSTABLE_ASSERT(!slot); } return res.first; @@ -3243,7 +2710,7 @@ class raw_hash_set { iterator erase(const_iterator first, const_iterator last) ABSL_ATTRIBUTE_LIFETIME_BOUND { AssertNotDebugCapacity(); - // We check for empty first because ClearBackingArray requires that + // We check for empty first because clear_backing_array requires that // capacity() > 0 as a precondition. if (empty()) return end(); if (first == last) return last.inner_; @@ -3254,11 +2721,10 @@ class raw_hash_set { } if (first == begin() && last == end()) { // TODO(ezb): we access control bytes in destroy_slots so it could make - // sense to combine destroy_slots and ClearBackingArray to avoid cache + // sense to combine destroy_slots and clear_backing_array to avoid cache // misses when the table is large. Note that we also do this in clear(). destroy_slots(); - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true, - SooEnabled()); + clear_backing_array(/*reuse=*/true); common().set_reserved_growth(common().reservation_size()); return end(); } @@ -3303,7 +2769,8 @@ class raw_hash_set { AssertNotDebugCapacity(); AssertIsFull(position.control(), position.inner_.generation(), position.inner_.generation_ptr(), "extract()"); - auto node = CommonAccess::Transfer(alloc_ref(), position.slot()); + allocator_type alloc(char_alloc_ref()); + auto node = CommonAccess::Transfer(alloc, position.slot()); if (is_soo()) { common().set_empty_soo(); } else { @@ -3329,73 +2796,16 @@ class raw_hash_set { swap_common(that); swap(hash_ref(), that.hash_ref()); swap(eq_ref(), that.eq_ref()); - SwapAlloc(alloc_ref(), that.alloc_ref(), + SwapAlloc(char_alloc_ref(), that.char_alloc_ref(), typename AllocTraits::propagate_on_container_swap{}); } - void rehash(size_t n) { - const size_t cap = capacity(); - if (n == 0) { - if (cap == 0 || is_soo()) return; - if (empty()) { - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false, - SooEnabled()); - return; - } - if (fits_in_soo(size())) { - // When the table is already sampled, we keep it sampled. - if (infoz().IsSampled()) { - const size_t kInitialSampledCapacity = NextCapacity(SooCapacity()); - if (capacity() > kInitialSampledCapacity) { - resize(kInitialSampledCapacity); - } - // This asserts that we didn't lose sampling coverage in `resize`. - ABSL_SWISSTABLE_ASSERT(infoz().IsSampled()); - return; - } - alignas(slot_type) unsigned char slot_space[sizeof(slot_type)]; - slot_type* tmp_slot = to_slot(slot_space); - transfer(tmp_slot, begin().slot()); - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false, - SooEnabled()); - transfer(soo_slot(), tmp_slot); - common().set_full_soo(); - return; - } - } - - // bitor is a faster way of doing `max` here. We will round up to the next - // power-of-2-minus-1, so bitor is good enough. - auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); - // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > cap) { - if (ABSL_PREDICT_FALSE(m > MaxValidCapacity())) { - HashTableSizeOverflow(); - } - resize(m); - - // This is after resize, to ensure that we have completed the allocation - // and have potentially sampled the hashtable. - infoz().RecordReservation(n); - } - } + void rehash(size_t n) { Rehash(common(), GetPolicyFunctions(), n); } void reserve(size_t n) { - const size_t max_size_before_growth = - is_soo() ? SooCapacity() : size() + growth_left(); - if (n > max_size_before_growth) { - if (ABSL_PREDICT_FALSE(n > max_size())) { - HashTableSizeOverflow(); - } - size_t m = GrowthToLowerboundCapacity(n); - resize(NormalizeCapacity(m)); - - // This is after resize, to ensure that we have completed the allocation - // and have potentially sampled the hashtable. - infoz().RecordReservation(n); + if (ABSL_PREDICT_TRUE(n > DefaultCapacity())) { + ReserveTableToFitNewSize(common(), GetPolicyFunctions(), n); } - common().reset_reserved_growth(n); - common().set_reservation_size(n); } // Extension API: support for heterogeneous keys. @@ -3424,7 +2834,7 @@ class raw_hash_set { // Avoid probing if we won't be able to prefetch the addresses received. #ifdef ABSL_HAVE_PREFETCH prefetch_heap_block(); - auto seq = probe(common(), hash_ref()(key)); + auto seq = probe(common(), hash_of(key)); PrefetchToLocalCache(control() + seq.offset()); PrefetchToLocalCache(slot_array() + seq.offset()); #endif // ABSL_HAVE_PREFETCH @@ -3441,9 +2851,9 @@ class raw_hash_set { template iterator find(const key_arg& key) ABSL_ATTRIBUTE_LIFETIME_BOUND { AssertOnFind(key); - if (is_soo()) return find_soo(key); + if (capacity() <= 1) return find_small(key); prefetch_heap_block(); - return find_non_soo(key, hash_ref()(key)); + return find_large(key, hash_of(key)); } template @@ -3493,7 +2903,9 @@ class raw_hash_set { hasher hash_function() const { return hash_ref(); } key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } + allocator_type get_allocator() const { + return allocator_type(char_alloc_ref()); + } friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { if (a.size() != b.size()) return false; @@ -3525,7 +2937,7 @@ class raw_hash_set { H>::type AbslHashValue(H h, const raw_hash_set& s) { return H::combine(H::combine_unordered(std::move(h), s.begin(), s.end()), - s.size()); + hash_internal::WeaklyMixedInteger{s.size()}); } friend void swap(raw_hash_set& a, @@ -3560,7 +2972,7 @@ class raw_hash_set { struct EqualElement { template bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(eq(lhs, rhs)); } const K1& rhs; const key_equal& eq; @@ -3598,37 +3010,48 @@ class raw_hash_set { template inline void construct(slot_type* slot, Args&&... args) { common().RunWithReentrancyGuard([&] { - PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + allocator_type alloc(char_alloc_ref()); + PolicyTraits::construct(&alloc, slot, std::forward(args)...); }); } inline void destroy(slot_type* slot) { - common().RunWithReentrancyGuard( - [&] { PolicyTraits::destroy(&alloc_ref(), slot); }); + common().RunWithReentrancyGuard([&] { + allocator_type alloc(char_alloc_ref()); + PolicyTraits::destroy(&alloc, slot); + }); } inline void transfer(slot_type* to, slot_type* from) { - common().RunWithReentrancyGuard( - [&] { PolicyTraits::transfer(&alloc_ref(), to, from); }); + common().RunWithReentrancyGuard([&] { + allocator_type alloc(char_alloc_ref()); + PolicyTraits::transfer(&alloc, to, from); + }); } // TODO(b/289225379): consider having a helper class that has the impls for // SOO functionality. template - iterator find_soo(const key_arg& key) { - ABSL_SWISSTABLE_ASSERT(is_soo()); - return empty() || !PolicyTraits::apply(EqualElement{key, eq_ref()}, - PolicyTraits::element(soo_slot())) + iterator find_small(const key_arg& key) { + ABSL_SWISSTABLE_ASSERT(capacity() <= 1); + return empty() || !PolicyTraits::apply( + EqualElement{key, eq_ref()}, + PolicyTraits::element(single_slot())) ? end() - : soo_iterator(); + : single_iterator(); } template - iterator find_non_soo(const key_arg& key, size_t hash) { + iterator find_large(const key_arg& key, size_t hash) { + ABSL_SWISSTABLE_ASSERT(capacity() > 1); ABSL_SWISSTABLE_ASSERT(!is_soo()); auto seq = probe(common(), hash); + const h2_t h2 = H2(hash); const ctrl_t* ctrl = control(); while (true) { +#ifndef ABSL_HAVE_MEMORY_SANITIZER + absl::PrefetchToLocalCache(slot_array() + seq.offset()); +#endif Group g{ctrl + seq.offset()}; - for (uint32_t i : g.Match(H2(hash))) { + for (uint32_t i : g.Match(h2)) { if (ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement{key, eq_ref()}, PolicyTraits::element(slot_array() + seq.offset(i))))) @@ -3640,36 +3063,49 @@ class raw_hash_set { } } - // Conditionally samples hashtablez for SOO tables. This should be called on - // insertion into an empty SOO table and in copy construction when the size - // can fit in SOO capacity. - inline HashtablezInfoHandle try_sample_soo() { + // Returns true if the table needs to be sampled. + // This should be called on insertion into an empty SOO table and in copy + // construction when the size can fit in SOO capacity. + bool should_sample_soo() const { ABSL_SWISSTABLE_ASSERT(is_soo()); - if (!ShouldSampleHashtablezInfo()) return HashtablezInfoHandle{}; - return Sample(sizeof(slot_type), sizeof(key_type), sizeof(value_type), - SooCapacity()); + if (!ShouldSampleHashtablezInfoForAlloc()) return false; + return ABSL_PREDICT_FALSE(ShouldSampleNextTable()); + } + + void clear_backing_array(bool reuse) { + ABSL_SWISSTABLE_ASSERT(capacity() > DefaultCapacity()); + ClearBackingArray(common(), GetPolicyFunctions(), &char_alloc_ref(), reuse, + SooEnabled()); } - inline void destroy_slots() { + void destroy_slots() { ABSL_SWISSTABLE_ASSERT(!is_soo()); if (PolicyTraits::template destroy_is_trivial()) return; - IterateOverFullSlots( - common(), slot_array(), - [&](const ctrl_t*, slot_type* slot) - ABSL_ATTRIBUTE_ALWAYS_INLINE { this->destroy(slot); }); + auto destroy_slot = [&](const ctrl_t*, void* slot) { + this->destroy(static_cast(slot)); + }; + if constexpr (SwisstableAssertAccessToDestroyedTable()) { + CommonFields common_copy(non_soo_tag_t{}, this->common()); + common().set_capacity(InvalidCapacity::kDestroyed); + IterateOverFullSlots(common_copy, sizeof(slot_type), destroy_slot); + common().set_capacity(common_copy.capacity()); + } else { + IterateOverFullSlots(common(), sizeof(slot_type), destroy_slot); + } } - inline void dealloc() { - ABSL_SWISSTABLE_ASSERT(capacity() != 0); + void dealloc() { + ABSL_SWISSTABLE_ASSERT(capacity() > DefaultCapacity()); // Unpoison before returning the memory to the allocator. SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * capacity()); infoz().Unregister(); - Deallocate( - &alloc_ref(), common().backing_array_start(), - common().alloc_size(sizeof(slot_type), alignof(slot_type))); + DeallocateBackingArray(&char_alloc_ref(), capacity(), control(), + sizeof(slot_type), alignof(slot_type), + common().has_infoz()); } - inline void destructor_impl() { + void destructor_impl() { if (SwisstableGenerationsEnabled() && capacity() >= InvalidCapacity::kMovedFrom) { return; @@ -3695,128 +3131,21 @@ class raw_hash_set { sizeof(slot_type)); } + template + size_t hash_of(const K& key) const { + return hash_ref()(key); + } size_t hash_of(slot_type* slot) const { return PolicyTraits::apply(HashElement{hash_ref()}, PolicyTraits::element(slot)); } - // Resizes table to the new capacity and move all elements to the new - // positions accordingly. - // - // Note that for better performance instead of - // find_first_non_full(common(), hash), - // HashSetResizeHelper::FindFirstNonFullAfterResize( - // common(), old_capacity, hash) - // can be called right after `resize`. - void resize(size_t new_capacity) { - raw_hash_set::resize_impl(common(), new_capacity, HashtablezInfoHandle{}); - } - - // As above, except that we also accept a pre-sampled, forced infoz for - // SOO tables, since they need to switch from SOO to heap in order to - // store the infoz. - void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) { - ABSL_SWISSTABLE_ASSERT(forced_infoz.IsSampled()); - raw_hash_set::resize_impl(common(), NextCapacity(SooCapacity()), - forced_infoz); - } - - // Resizes set to the new capacity. - // It is a static function in order to use its pointer in GetPolicyFunctions. - ABSL_ATTRIBUTE_NOINLINE static void resize_impl( - CommonFields& common, size_t new_capacity, - HashtablezInfoHandle forced_infoz) { - raw_hash_set* set = reinterpret_cast(&common); - ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity)); - ABSL_SWISSTABLE_ASSERT(!set->fits_in_soo(new_capacity)); - const bool was_soo = set->is_soo(); - const bool had_soo_slot = was_soo && !set->empty(); - const ctrl_t soo_slot_h2 = - had_soo_slot ? static_cast(H2(set->hash_of(set->soo_slot()))) - : ctrl_t::kEmpty; - HashSetResizeHelper resize_helper(common, was_soo, had_soo_slot, - forced_infoz); - // Initialize HashSetResizeHelper::old_heap_or_soo_. We can't do this in - // HashSetResizeHelper constructor because it can't transfer slots when - // transfer_uses_memcpy is false. - // TODO(b/289225379): try to handle more of the SOO cases inside - // InitializeSlots. See comment on cl/555990034 snapshot #63. - if (PolicyTraits::transfer_uses_memcpy() || !had_soo_slot) { - resize_helper.old_heap_or_soo() = common.heap_or_soo(); - } else { - set->transfer(set->to_slot(resize_helper.old_soo_data()), - set->soo_slot()); - } - common.set_capacity(new_capacity); - // Note that `InitializeSlots` does different number initialization steps - // depending on the values of `transfer_uses_memcpy` and capacities. - // Refer to the comment in `InitializeSlots` for more details. - const bool grow_single_group = - resize_helper.InitializeSlots( - common, CharAlloc(set->alloc_ref()), soo_slot_h2, sizeof(key_type), - sizeof(value_type)); - - // In the SooEnabled() case, capacity is never 0 so we don't check. - if (!SooEnabled() && resize_helper.old_capacity() == 0) { - // InitializeSlots did all the work including infoz().RecordRehash(). - return; - } - ABSL_SWISSTABLE_ASSERT(resize_helper.old_capacity() > 0); - // Nothing more to do in this case. - if (was_soo && !had_soo_slot) return; - - slot_type* new_slots = set->slot_array(); - if (grow_single_group) { - if (PolicyTraits::transfer_uses_memcpy()) { - // InitializeSlots did all the work. - return; - } - if (was_soo) { - set->transfer(new_slots + resize_helper.SooSlotIndex(), - to_slot(resize_helper.old_soo_data())); - return; - } else { - // We want GrowSizeIntoSingleGroup to be called here in order to make - // InitializeSlots not depend on PolicyTraits. - resize_helper.GrowSizeIntoSingleGroup(common, - set->alloc_ref()); - } - } else { - // InitializeSlots prepares control bytes to correspond to empty table. - const auto insert_slot = [&](slot_type* slot) { - size_t hash = PolicyTraits::apply(HashElement{set->hash_ref()}, - PolicyTraits::element(slot)); - auto target = find_first_non_full(common, hash); - SetCtrl(common, target.offset, H2(hash), sizeof(slot_type)); - set->transfer(new_slots + target.offset, slot); - return target.probe_length; - }; - if (was_soo) { - insert_slot(to_slot(resize_helper.old_soo_data())); - return; - } else { - auto* old_slots = static_cast(resize_helper.old_slots()); - size_t total_probe_length = 0; - for (size_t i = 0; i != resize_helper.old_capacity(); ++i) { - if (IsFull(resize_helper.old_ctrl()[i])) { - total_probe_length += insert_slot(old_slots + i); - } - } - common.infoz().RecordRehash(total_probe_length); - } - } - resize_helper.DeallocateOld(CharAlloc(set->alloc_ref()), - sizeof(slot_type)); - } - // Casting directly from e.g. char* to slot_type* can cause compilation errors // on objective-C. This function converts to void* first, avoiding the issue. static slot_type* to_slot(void* buf) { return static_cast(buf); } // Requires that lhs does not have a full SOO slot. - static void move_common(bool rhs_is_full_soo, allocator_type& rhs_alloc, + static void move_common(bool rhs_is_full_soo, CharAlloc& rhs_alloc, CommonFields& lhs, CommonFields&& rhs) { if (PolicyTraits::transfer_uses_memcpy() || !rhs_is_full_soo) { lhs = std::move(rhs); @@ -3841,10 +3170,12 @@ class raw_hash_set { } CommonFields tmp = CommonFields(uninitialized_tag_t{}); const bool that_is_full_soo = that.is_full_soo(); - move_common(that_is_full_soo, that.alloc_ref(), tmp, + move_common(that_is_full_soo, that.char_alloc_ref(), tmp, std::move(that.common())); - move_common(is_full_soo(), alloc_ref(), that.common(), std::move(common())); - move_common(that_is_full_soo, that.alloc_ref(), common(), std::move(tmp)); + move_common(is_full_soo(), char_alloc_ref(), that.common(), + std::move(common())); + move_common(that_is_full_soo, that.char_alloc_ref(), common(), + std::move(tmp)); } void annotate_for_bug_detection_on_move( @@ -3862,7 +3193,8 @@ class raw_hash_set { } common().increment_generation(); if (!empty() && common().should_rehash_for_bug_detection_on_move()) { - resize(capacity()); + ResizeAllocatedTableWithSeedChange(common(), GetPolicyFunctions(), + capacity()); } } @@ -3871,11 +3203,11 @@ class raw_hash_set { // We don't bother checking for this/that aliasing. We just need to avoid // breaking the invariants in that case. destructor_impl(); - move_common(that.is_full_soo(), that.alloc_ref(), common(), + move_common(that.is_full_soo(), that.char_alloc_ref(), common(), std::move(that.common())); hash_ref() = that.hash_ref(); eq_ref() = that.eq_ref(); - CopyAlloc(alloc_ref(), that.alloc_ref(), + CopyAlloc(char_alloc_ref(), that.char_alloc_ref(), std::integral_constant()); that.common() = CommonFields::CreateDefault(); annotate_for_bug_detection_on_move(that); @@ -3902,7 +3234,7 @@ class raw_hash_set { } raw_hash_set& move_assign(raw_hash_set&& that, std::false_type /*propagate_alloc*/) { - if (alloc_ref() == that.alloc_ref()) { + if (char_alloc_ref() == that.char_alloc_ref()) { return assign_impl(std::move(that)); } // Aliasing can't happen here because allocs would compare equal above. @@ -3918,22 +3250,25 @@ class raw_hash_set { template std::pair find_or_prepare_insert_soo(const K& key) { + ctrl_t soo_slot_ctrl; if (empty()) { - const HashtablezInfoHandle infoz = try_sample_soo(); - if (infoz.IsSampled()) { - resize_with_soo_infoz(infoz); - } else { + if (!should_sample_soo()) { common().set_full_soo(); return {soo_iterator(), true}; } + soo_slot_ctrl = ctrl_t::kEmpty; } else if (PolicyTraits::apply(EqualElement{key, eq_ref()}, PolicyTraits::element(soo_slot()))) { return {soo_iterator(), false}; } else { - resize(NextCapacity(SooCapacity())); - } - const size_t index = - PrepareInsertAfterSoo(hash_ref()(key), sizeof(slot_type), common()); + soo_slot_ctrl = static_cast(H2(hash_of(soo_slot()))); + } + constexpr bool kUseMemcpy = + PolicyTraits::transfer_uses_memcpy() && SooEnabled(); + size_t index = GrowSooTableToNextCapacityAndPrepareInsert< + kUseMemcpy ? OptimalMemcpySizeForSooSlotTransfer(sizeof(slot_type)) : 0, + kUseMemcpy>(common(), GetPolicyFunctions(), hash_of(key), + soo_slot_ctrl); return {iterator_at(index), true}; } @@ -3941,12 +3276,16 @@ class raw_hash_set { std::pair find_or_prepare_insert_non_soo(const K& key) { ABSL_SWISSTABLE_ASSERT(!is_soo()); prefetch_heap_block(); - auto hash = hash_ref()(key); + const size_t hash = hash_of(key); auto seq = probe(common(), hash); + const h2_t h2 = H2(hash); const ctrl_t* ctrl = control(); while (true) { +#ifndef ABSL_HAVE_MEMORY_SANITIZER + absl::PrefetchToLocalCache(slot_array() + seq.offset()); +#endif Group g{ctrl + seq.offset()}; - for (uint32_t i : g.Match(H2(hash))) { + for (uint32_t i : g.Match(h2)) { if (ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement{key, eq_ref()}, PolicyTraits::element(slot_array() + seq.offset(i))))) @@ -3954,11 +3293,10 @@ class raw_hash_set { } auto mask_empty = g.MaskEmpty(); if (ABSL_PREDICT_TRUE(mask_empty)) { - size_t target = seq.offset( - GetInsertionOffset(mask_empty, capacity(), hash, control())); - return {iterator_at(PrepareInsertNonSoo(common(), hash, - FindInfo{target, seq.index()}, - GetPolicyFunctions())), + size_t target = seq.offset(mask_empty.LowestBitSet()); + return {iterator_at(PrepareInsertNonSoo(common(), GetPolicyFunctions(), + hash, + FindInfo{target, seq.index()})), true}; } seq.next(); @@ -3976,6 +3314,11 @@ class raw_hash_set { // Asserts that the capacity is not a sentinel invalid value. void AssertNotDebugCapacity() const { +#ifdef NDEBUG + if (!SwisstableGenerationsEnabled()) { + return; + } +#endif if (ABSL_PREDICT_TRUE(capacity() < InvalidCapacity::kAboveMaxValidCapacity)) { return; @@ -3983,8 +3326,11 @@ class raw_hash_set { assert(capacity() != InvalidCapacity::kReentrance && "Reentrant container access during element construction/destruction " "is not allowed."); - assert(capacity() != InvalidCapacity::kDestroyed && - "Use of destroyed hash table."); + if constexpr (SwisstableAssertAccessToDestroyedTable()) { + if (capacity() == InvalidCapacity::kDestroyed) { + ABSL_RAW_LOG(FATAL, "Use of destroyed hash table."); + } + } if (SwisstableGenerationsEnabled() && ABSL_PREDICT_FALSE(capacity() >= InvalidCapacity::kMovedFrom)) { if (capacity() == InvalidCapacity::kSelfMovedFrom) { @@ -4015,9 +3361,10 @@ class raw_hash_set { } if (empty()) return; - const size_t hash_of_arg = hash_ref()(key); - const auto assert_consistent = [&](const ctrl_t*, slot_type* slot) { - const value_type& element = PolicyTraits::element(slot); + const size_t hash_of_arg = hash_of(key); + const auto assert_consistent = [&](const ctrl_t*, void* slot) { + const value_type& element = + PolicyTraits::element(static_cast(slot)); const bool is_key_equal = PolicyTraits::apply(EqualElement{key, eq_ref()}, element); if (!is_key_equal) return; @@ -4037,7 +3384,7 @@ class raw_hash_set { } // We only do validation for small tables so that it's constant time. if (capacity() > 16) return; - IterateOverFullSlots(common(), slot_array(), assert_consistent); + IterateOverFullSlots(common(), sizeof(slot_type), assert_consistent); } // Attempts to find `key` in the table; if it isn't found, returns an iterator @@ -4062,7 +3409,10 @@ class raw_hash_set { void emplace_at(iterator iter, Args&&... args) { construct(iter.slot(), std::forward(args)...); - assert(PolicyTraits::apply(FindElement{*this}, *iter) == iter && + // When capacity is 1, find calls find_small and if size is 0, then it will + // return an end iterator. This can happen in the raw_hash_set copy ctor. + assert((capacity() == 1 || + PolicyTraits::apply(FindElement{*this}, *iter) == iter) && "constructed value does not match the lookup key"); } @@ -4125,10 +3475,12 @@ class raw_hash_set { } slot_type* soo_slot() { ABSL_SWISSTABLE_ASSERT(is_soo()); - return static_cast(common().soo_data()); + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN( + static_cast(common().soo_data())); } const slot_type* soo_slot() const { - return const_cast(this)->soo_slot(); + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN( + const_cast(this)->soo_slot()); } iterator soo_iterator() { return {SooControl(), soo_slot(), common().generation_ptr()}; @@ -4136,6 +3488,20 @@ class raw_hash_set { const_iterator soo_iterator() const { return const_cast(this)->soo_iterator(); } + slot_type* single_slot() { + ABSL_SWISSTABLE_ASSERT(capacity() <= 1); + return SooEnabled() ? soo_slot() : slot_array(); + } + const slot_type* single_slot() const { + return const_cast(this)->single_slot(); + } + iterator single_iterator() { + return {SooEnabled() ? SooControl() : control(), single_slot(), + common().generation_ptr()}; + } + const_iterator single_iterator() const { + return const_cast(this)->single_iterator(); + } HashtablezInfoHandle infoz() { ABSL_SWISSTABLE_ASSERT(!is_soo()); return common().infoz(); @@ -4145,49 +3511,118 @@ class raw_hash_set { const hasher& hash_ref() const { return settings_.template get<1>(); } key_equal& eq_ref() { return settings_.template get<2>(); } const key_equal& eq_ref() const { return settings_.template get<2>(); } - allocator_type& alloc_ref() { return settings_.template get<3>(); } - const allocator_type& alloc_ref() const { + CharAlloc& char_alloc_ref() { return settings_.template get<3>(); } + const CharAlloc& char_alloc_ref() const { return settings_.template get<3>(); } - static const void* get_hash_ref_fn(const CommonFields& common) { - auto* h = reinterpret_cast(&common); - return &h->hash_ref(); + static void* get_char_alloc_ref_fn(CommonFields& common) { + auto* h = reinterpret_cast(&common); + return &h->char_alloc_ref(); } - static void transfer_slot_fn(void* set, void* dst, void* src) { + static void* get_hash_ref_fn(CommonFields& common) { + auto* h = reinterpret_cast(&common); + // TODO(b/397453582): Remove support for const hasher. + return const_cast*>(&h->hash_ref()); + } + static void transfer_n_slots_fn(void* set, void* dst, void* src, + size_t count) { + auto* src_slot = to_slot(src); + auto* dst_slot = to_slot(dst); + auto* h = static_cast(set); - h->transfer(static_cast(dst), static_cast(src)); + for (; count > 0; --count, ++src_slot, ++dst_slot) { + h->transfer(dst_slot, src_slot); + } } - // Note: dealloc_fn will only be used if we have a non-standard allocator. - static void dealloc_fn(CommonFields& common, const PolicyFunctions&) { - auto* set = reinterpret_cast(&common); - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(common.slot_array(), - sizeof(slot_type) * common.capacity()); + // TODO(b/382423690): Try to type erase entire function or at least type erase + // by GetKey + Hash for memcpyable types. + // TODO(b/382423690): Try to type erase for big slots: sizeof(slot_type) > 16. + static void transfer_unprobed_elements_to_next_capacity_fn( + CommonFields& common, const ctrl_t* old_ctrl, void* old_slots, + void* probed_storage, + void (*encode_probed_element)(void* probed_storage, h2_t h2, + size_t source_offset, size_t h1)) { + const size_t new_capacity = common.capacity(); + const size_t old_capacity = PreviousCapacity(new_capacity); + ABSL_ASSUME(old_capacity + 1 >= Group::kWidth); + ABSL_ASSUME((old_capacity + 1) % Group::kWidth == 0); + + auto* set = reinterpret_cast(&common); + slot_type* old_slots_ptr = to_slot(old_slots); + ctrl_t* new_ctrl = common.control(); + slot_type* new_slots = set->slot_array(); - common.infoz().Unregister(); - Deallocate( - &set->alloc_ref(), common.backing_array_start(), - common.alloc_size(sizeof(slot_type), alignof(slot_type))); + const PerTableSeed seed = common.seed(); + + for (size_t group_index = 0; group_index < old_capacity; + group_index += Group::kWidth) { + GroupFullEmptyOrDeleted old_g(old_ctrl + group_index); + std::memset(new_ctrl + group_index, static_cast(ctrl_t::kEmpty), + Group::kWidth); + std::memset(new_ctrl + group_index + old_capacity + 1, + static_cast(ctrl_t::kEmpty), Group::kWidth); + // TODO(b/382423690): try to type erase everything outside of the loop. + // We will share a lot of code in expense of one function call per group. + for (auto in_fixed_group_index : old_g.MaskFull()) { + size_t old_index = group_index + in_fixed_group_index; + slot_type* old_slot = old_slots_ptr + old_index; + // TODO(b/382423690): try to avoid entire hash calculation since we need + // only one new bit of h1. + size_t hash = set->hash_of(old_slot); + size_t h1 = H1(hash, seed); + h2_t h2 = H2(hash); + size_t new_index = TryFindNewIndexWithoutProbing( + h1, old_index, old_capacity, new_ctrl, new_capacity); + // Note that encode_probed_element is allowed to use old_ctrl buffer + // till and included the old_index. + if (ABSL_PREDICT_FALSE(new_index == kProbedElementIndexSentinel)) { + encode_probed_element(probed_storage, h2, old_index, h1); + continue; + } + ABSL_SWISSTABLE_ASSERT((new_index & old_capacity) <= old_index); + ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[new_index])); + new_ctrl[new_index] = static_cast(h2); + auto* new_slot = new_slots + new_index; + SanitizerUnpoisonMemoryRegion(new_slot, sizeof(slot_type)); + set->transfer(new_slot, old_slot); + SanitizerPoisonMemoryRegion(old_slot, sizeof(slot_type)); + } + } } static const PolicyFunctions& GetPolicyFunctions() { + static_assert(sizeof(slot_type) <= (std::numeric_limits::max)(), + "Slot size is too large. Use std::unique_ptr for value type " + "or use absl::node_hash_{map,set}."); + static_assert(alignof(slot_type) <= + size_t{(std::numeric_limits::max)()}); + static_assert(sizeof(key_type) <= + size_t{(std::numeric_limits::max)()}); + static_assert(sizeof(value_type) <= + size_t{(std::numeric_limits::max)()}); + static constexpr size_t kBackingArrayAlignment = + BackingArrayAlignment(alignof(slot_type)); static constexpr PolicyFunctions value = { - sizeof(slot_type), + static_cast(sizeof(key_type)), + static_cast(sizeof(value_type)), + static_cast(sizeof(slot_type)), + static_cast(alignof(slot_type)), SooEnabled(), + ShouldSampleHashtablezInfoForAlloc(), // TODO(b/328722020): try to type erase // for standard layout and alignof(Hash) <= alignof(CommonFields). - std::is_empty::value ? &GetHashRefForEmptyHasher - : &raw_hash_set::get_hash_ref_fn, + std::is_empty_v ? &GetRefForEmptyClass + : &raw_hash_set::get_hash_ref_fn, PolicyTraits::template get_hash_slot_fn(), PolicyTraits::transfer_uses_memcpy() - ? TransferRelocatable - : &raw_hash_set::transfer_slot_fn, - (std::is_same>::value - ? &DeallocateStandard - : &raw_hash_set::dealloc_fn), - &raw_hash_set::resize_impl - }; + ? TransferNRelocatable + : &raw_hash_set::transfer_n_slots_fn, + std::is_empty_v ? &GetRefForEmptyClass + : &raw_hash_set::get_char_alloc_ref_fn, + &AllocateBackingArray, + &DeallocateBackingArray, + &raw_hash_set::transfer_unprobed_elements_to_next_capacity_fn}; return value; } @@ -4195,9 +3630,9 @@ class raw_hash_set { // CompressedTuple will ensure that sizeof is not affected by any of the empty // fields that occur after CommonFields. absl::container_internal::CompressedTuple + CharAlloc> settings_{CommonFields::CreateDefault(), hasher{}, - key_equal{}, allocator_type{}}; + key_equal{}, CharAlloc{}}; }; // Friend access for free functions in raw_hash_set.h. @@ -4220,8 +3655,11 @@ struct HashtableFreeFunctionsAccess { } ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = c->size(); size_t num_deleted = 0; + using SlotType = typename Set::slot_type; IterateOverFullSlots( - c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) { + c->common(), sizeof(SlotType), + [&](const ctrl_t* ctrl, void* slot_void) { + auto* slot = static_cast(slot_void); if (pred(Set::PolicyTraits::element(slot))) { c->destroy(slot); EraseMetaOnly(c->common(), static_cast(ctrl - c->control()), @@ -4246,10 +3684,12 @@ struct HashtableFreeFunctionsAccess { cb(*c->soo_iterator()); return; } + using SlotType = typename Set::slot_type; using ElementTypeWithConstness = decltype(*c->begin()); IterateOverFullSlots( - c->common(), c->slot_array(), [&cb](const ctrl_t*, auto* slot) { - ElementTypeWithConstness& element = Set::PolicyTraits::element(slot); + c->common(), sizeof(SlotType), [&cb](const ctrl_t*, void* slot) { + ElementTypeWithConstness& element = + Set::PolicyTraits::element(static_cast(slot)); cb(element); }); } @@ -4282,12 +3722,13 @@ struct HashtableDebugAccess> { const typename Set::key_type& key) { if (set.is_soo()) return 0; size_t num_probes = 0; - size_t hash = set.hash_ref()(key); + const size_t hash = set.hash_of(key); auto seq = probe(set.common(), hash); + const h2_t h2 = H2(hash); const ctrl_t* ctrl = set.control(); while (true) { container_internal::Group g{ctrl + seq.offset()}; - for (uint32_t i : g.Match(container_internal::H2(hash))) { + for (uint32_t i : g.Match(h2)) { if (Traits::apply( typename Set::template EqualElement{ key, set.eq_ref()}, @@ -4320,6 +3761,22 @@ struct HashtableDebugAccess> { }; } // namespace hashtable_debug_internal + +// Extern template instantiations reduce binary size and linker input size. +// Function definition is in raw_hash_set.cc. +extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<0, false>( + CommonFields&, const PolicyFunctions&, size_t, ctrl_t); +extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<1, true>( + CommonFields&, const PolicyFunctions&, size_t, ctrl_t); +extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<4, true>( + CommonFields&, const PolicyFunctions&, size_t, ctrl_t); +extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<8, true>( + CommonFields&, const PolicyFunctions&, size_t, ctrl_t); +#if UINTPTR_MAX == UINT64_MAX +extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<16, true>( + CommonFields&, const PolicyFunctions&, size_t, ctrl_t); +#endif + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/raw_hash_set_resize_impl.h b/absl/container/internal/raw_hash_set_resize_impl.h new file mode 100644 index 0000000..149d9e8 --- /dev/null +++ b/absl/container/internal/raw_hash_set_resize_impl.h @@ -0,0 +1,80 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This is a private implementation detail of resize algorithm of +// raw_hash_set. It is exposed in a separate file for testing purposes. + +#ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_RESIZE_IMPL_H_ +#define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_RESIZE_IMPL_H_ + +#include +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// Encoding for probed elements used for smaller tables. +// Data is encoded into single integer. +// Storage format for 4 bytes: +// - 7 bits for h2 +// - 12 bits for source_offset +// - 13 bits for h1 +// Storage format for 8 bytes: +// - 7 bits for h2 +// - 28 bits for source_offset +// - 29 bits for h1 +// Storage format for 16 bytes: +// - 7 bits for h2 +// - 57 bits for source_offset +// - 58 bits for h1 +template +struct ProbedItemImpl { + static constexpr IntType kH2Bits = 7; + + static constexpr IntType kMaxOldBits = (kTotalBits - kH2Bits) / 2; + static constexpr IntType kMaxOldCapacity = (IntType{1} << kMaxOldBits) - 1; + + // We always have one bit more for h1. + static constexpr IntType kMaxNewBits = kMaxOldBits + 1; + static constexpr IntType kMaxNewCapacity = (IntType{1} << kMaxNewBits) - 1; + + static constexpr IntType kH2Shift = (kTotalBits - kH2Bits); + static_assert(kMaxNewBits + kMaxOldBits + kH2Bits == kTotalBits); + + ProbedItemImpl() = default; + ProbedItemImpl(uint8_t h2_arg, size_t source_offset_arg, size_t h1_arg) + : h2(h2_arg), + source_offset(static_cast(source_offset_arg)), + h1(static_cast(h1_arg)) {} + + IntType h2 : kH2Bits; + IntType source_offset : kMaxOldBits; + IntType h1 : kMaxNewBits; +}; + +using ProbedItem4Bytes = ProbedItemImpl; +static_assert(sizeof(ProbedItem4Bytes) == 4); +using ProbedItem8Bytes = ProbedItemImpl; +static_assert(sizeof(ProbedItem8Bytes) == 8); +using ProbedItem16Bytes = ProbedItemImpl; +static_assert(sizeof(ProbedItem16Bytes) == 16); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_RESIZE_IMPL_H_ diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index 127c893..8aed18b 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -99,6 +99,11 @@ class NodeHashMapPolicy; // In most cases `T` needs only to provide the `absl_container_hash`. In this // case `std::equal_to` will be used instead of `eq` part. // +// PERFORMANCE WARNING: Erasure & sparsity can negatively affect performance: +// * Iteration takes O(capacity) time, not O(size). +// * erase() slows down begin() and ++iterator. +// * Capacity only shrinks on rehash() or clear() -- not on erase(). +// // Example: // // // Create a node hash map of three strings (that map to strings) diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index cffa50e..6240e2d 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -97,6 +97,11 @@ struct NodeHashSetPolicy; // In most cases `T` needs only to provide the `absl_container_hash`. In this // case `std::equal_to` will be used instead of `eq` part. // +// PERFORMANCE WARNING: Erasure & sparsity can negatively affect performance: +// * Iteration takes O(capacity) time, not O(size). +// * erase() slows down begin() and ++iterator. +// * Capacity only shrinks on rehash() or clear() -- not on erase(). +// // Example: // // // Create a node hash set of three strings diff --git a/absl/crc/crc32c.cc b/absl/crc/crc32c.cc index 468c1b3..9b1ef7e 100644 --- a/absl/crc/crc32c.cc +++ b/absl/crc/crc32c.cc @@ -54,10 +54,6 @@ crc32c_t ExtendCrc32cInternal(crc32c_t initial_crc, } // namespace crc_internal -crc32c_t ComputeCrc32c(absl::string_view buf) { - return ExtendCrc32c(crc32c_t{0}, buf); -} - crc32c_t ExtendCrc32cByZeroes(crc32c_t initial_crc, size_t length) { uint32_t crc = static_cast(initial_crc) ^ kCRC32Xor; CrcEngine()->ExtendByZeroes(&crc, length); diff --git a/absl/crc/crc32c.h b/absl/crc/crc32c.h index 362861e..5ecc6b3 100644 --- a/absl/crc/crc32c.h +++ b/absl/crc/crc32c.h @@ -83,11 +83,6 @@ crc32c_t ExtendCrc32cInternal(crc32c_t initial_crc, // CRC32C Computation Functions // ----------------------------------------------------------------------------- -// ComputeCrc32c() -// -// Returns the CRC32C value of the provided string. -crc32c_t ComputeCrc32c(absl::string_view buf); - // ExtendCrc32c() // // Computes a CRC32C value from an `initial_crc` CRC32C value including the @@ -112,6 +107,13 @@ inline crc32c_t ExtendCrc32c(crc32c_t initial_crc, return crc_internal::ExtendCrc32cInternal(initial_crc, buf_to_add); } +// ComputeCrc32c() +// +// Returns the CRC32C value of the provided string. +inline crc32c_t ComputeCrc32c(absl::string_view buf) { + return ExtendCrc32c(crc32c_t{0}, buf); +} + // ExtendCrc32cByZeroes() // // Computes a CRC32C value for a buffer with an `initial_crc` CRC32C value, diff --git a/absl/crc/internal/crc32_x86_arm_combined_simd.h b/absl/crc/internal/crc32_x86_arm_combined_simd.h index 0f6e347..5a9b61a 100644 --- a/absl/crc/internal/crc32_x86_arm_combined_simd.h +++ b/absl/crc/internal/crc32_x86_arm_combined_simd.h @@ -99,19 +99,12 @@ V128 V128_PMul10(const V128 l, const V128 r); // Produces a XOR operation of |l| and |r|. V128 V128_Xor(const V128 l, const V128 r); -// Produces an AND operation of |l| and |r|. -V128 V128_And(const V128 l, const V128 r); - // Sets the lower half of a 128 bit register to the given 64-bit value and // zeroes the upper half. // dst[63:0] := |r| // dst[127:64] := |0| V128 V128_From64WithZeroFill(const uint64_t r); -// Shift |l| right by |imm| bytes while shifting in zeros. -template -V128 V128_ShiftRight(const V128 l); - // Extracts a 32-bit integer from |l|, selected with |imm|. template int V128_Extract32(const V128 l); @@ -170,17 +163,10 @@ inline V128 V128_PMul10(const V128 l, const V128 r) { inline V128 V128_Xor(const V128 l, const V128 r) { return _mm_xor_si128(l, r); } -inline V128 V128_And(const V128 l, const V128 r) { return _mm_and_si128(l, r); } - inline V128 V128_From64WithZeroFill(const uint64_t r) { return _mm_set_epi64x(static_cast(0), static_cast(r)); } -template -inline V128 V128_ShiftRight(const V128 l) { - return _mm_srli_si128(l, imm); -} - template inline int V128_Extract32(const V128 l) { return _mm_extract_epi32(l, imm); @@ -261,20 +247,12 @@ inline V128 V128_PMul10(const V128 l, const V128 r) { inline V128 V128_Xor(const V128 l, const V128 r) { return veorq_u64(l, r); } -inline V128 V128_And(const V128 l, const V128 r) { return vandq_u64(l, r); } - inline V128 V128_From64WithZeroFill(const uint64_t r){ constexpr uint64x2_t kZero = {0, 0}; return vsetq_lane_u64(r, kZero, 0); } -template -inline V128 V128_ShiftRight(const V128 l) { - return vreinterpretq_u64_s8( - vextq_s8(vreinterpretq_s8_u64(l), vdupq_n_s8(0), imm)); -} - template inline int V128_Extract32(const V128 l) { return vgetq_lane_s32(vreinterpretq_s32_u64(l), imm); diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 79dace3..3194bec 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc @@ -64,27 +64,27 @@ class CRC32AcceleratedX86ARMCombined : public CRC32 { constexpr size_t kSmallCutoff = 256; constexpr size_t kMediumCutoff = 2048; -#define ABSL_INTERNAL_STEP1(crc) \ +#define ABSL_INTERNAL_STEP1(crc, p) \ do { \ crc = CRC32_u8(static_cast(crc), *p++); \ } while (0) -#define ABSL_INTERNAL_STEP2(crc) \ +#define ABSL_INTERNAL_STEP2(crc, p) \ do { \ crc = \ CRC32_u16(static_cast(crc), absl::little_endian::Load16(p)); \ p += 2; \ } while (0) -#define ABSL_INTERNAL_STEP4(crc) \ +#define ABSL_INTERNAL_STEP4(crc, p) \ do { \ crc = \ CRC32_u32(static_cast(crc), absl::little_endian::Load32(p)); \ p += 4; \ } while (0) -#define ABSL_INTERNAL_STEP8(crc, data) \ - do { \ - crc = CRC32_u64(static_cast(crc), \ - absl::little_endian::Load64(data)); \ - data += 8; \ +#define ABSL_INTERNAL_STEP8(crc, p) \ + do { \ + crc = \ + CRC32_u64(static_cast(crc), absl::little_endian::Load64(p)); \ + p += 8; \ } while (0) #define ABSL_INTERNAL_STEP8BY2(crc0, crc1, p0, p1) \ do { \ @@ -221,7 +221,8 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreamsBase // We are applying it to CRC32C polynomial. ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul( const uint8_t* p, V128* partialCRC) const { - V128 loopMultiplicands = V128_Load(reinterpret_cast(k1k2)); + V128 loopMultiplicands = + V128_Load(reinterpret_cast(kFoldAcross512Bits)); V128 partialCRC1 = partialCRC[0]; V128 partialCRC2 = partialCRC[1]; @@ -265,53 +266,33 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreamsBase // Combine 4 vectors of partial crc into a single vector. V128 reductionMultiplicands = - V128_Load(reinterpret_cast(k5k6)); + V128_Load(reinterpret_cast(kFoldAcross256Bits)); V128 low = V128_PMulLow(reductionMultiplicands, partialCRC1); V128 high = V128_PMulHi(reductionMultiplicands, partialCRC1); partialCRC1 = V128_Xor(low, high); - partialCRC1 = V128_Xor(partialCRC1, partialCRC2); + partialCRC1 = V128_Xor(partialCRC1, partialCRC3); - low = V128_PMulLow(reductionMultiplicands, partialCRC3); - high = V128_PMulHi(reductionMultiplicands, partialCRC3); + low = V128_PMulLow(reductionMultiplicands, partialCRC2); + high = V128_PMulHi(reductionMultiplicands, partialCRC2); - partialCRC3 = V128_Xor(low, high); - partialCRC3 = V128_Xor(partialCRC3, partialCRC4); + partialCRC2 = V128_Xor(low, high); + partialCRC2 = V128_Xor(partialCRC2, partialCRC4); - reductionMultiplicands = V128_Load(reinterpret_cast(k3k4)); + reductionMultiplicands = + V128_Load(reinterpret_cast(kFoldAcross128Bits)); low = V128_PMulLow(reductionMultiplicands, partialCRC1); high = V128_PMulHi(reductionMultiplicands, partialCRC1); V128 fullCRC = V128_Xor(low, high); - fullCRC = V128_Xor(fullCRC, partialCRC3); + fullCRC = V128_Xor(fullCRC, partialCRC2); // Reduce fullCRC into scalar value. - reductionMultiplicands = V128_Load(reinterpret_cast(k5k6)); - - V128 mask = V128_Load(reinterpret_cast(kMask)); - - V128 tmp = V128_PMul01(reductionMultiplicands, fullCRC); - fullCRC = V128_ShiftRight<8>(fullCRC); - fullCRC = V128_Xor(fullCRC, tmp); - - reductionMultiplicands = V128_Load(reinterpret_cast(k7k0)); - - tmp = V128_ShiftRight<4>(fullCRC); - fullCRC = V128_And(fullCRC, mask); - fullCRC = V128_PMulLow(reductionMultiplicands, fullCRC); - fullCRC = V128_Xor(tmp, fullCRC); - - reductionMultiplicands = V128_Load(reinterpret_cast(kPoly)); - - tmp = V128_And(fullCRC, mask); - tmp = V128_PMul01(reductionMultiplicands, tmp); - tmp = V128_And(tmp, mask); - tmp = V128_PMulLow(reductionMultiplicands, tmp); - - fullCRC = V128_Xor(tmp, fullCRC); - - return static_cast(V128_Extract32<1>(fullCRC)); + uint32_t crc = 0; + crc = CRC32_u64(crc, V128_Extract64<0>(fullCRC)); + crc = CRC32_u64(crc, V128_Extract64<1>(fullCRC)); + return crc; } // Update crc with 64 bytes of data from p. @@ -325,15 +306,23 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreamsBase return crc; } - // Generated by crc32c_x86_test --crc32c_generate_constants=true - // and verified against constants in linux kernel for S390: - // https://github.com/torvalds/linux/blob/master/arch/s390/crypto/crc32le-vx.S - alignas(16) static constexpr uint64_t k1k2[2] = {0x0740eef02, 0x09e4addf8}; - alignas(16) static constexpr uint64_t k3k4[2] = {0x1384aa63a, 0x0ba4fc28e}; - alignas(16) static constexpr uint64_t k5k6[2] = {0x0f20c0dfe, 0x14cd00bd6}; - alignas(16) static constexpr uint64_t k7k0[2] = {0x0dd45aab8, 0x000000000}; - alignas(16) static constexpr uint64_t kPoly[2] = {0x105ec76f0, 0x0dea713f1}; - alignas(16) static constexpr uint32_t kMask[4] = {~0u, 0u, ~0u, 0u}; + // Constants generated by './scripts/gen-crc-consts.py x86_pclmul + // crc32_lsb_0x82f63b78' from the Linux kernel. + alignas(16) static constexpr uint64_t kFoldAcross512Bits[2] = { + // (x^543 mod G) * x^32 + 0x00000000740eef02, + // (x^479 mod G) * x^32 + 0x000000009e4addf8}; + alignas(16) static constexpr uint64_t kFoldAcross256Bits[2] = { + // (x^287 mod G) * x^32 + 0x000000003da6d0cb, + // (x^223 mod G) * x^32 + 0x00000000ba4fc28e}; + alignas(16) static constexpr uint64_t kFoldAcross128Bits[2] = { + // (x^159 mod G) * x^32 + 0x00000000f20c0dfe, + // (x^95 mod G) * x^32 + 0x00000000493c7d27}; // Medium runs of bytes are broken into groups of kGroupsSmall blocks of same // size. Each group is CRCed in parallel then combined at the end of the @@ -345,24 +334,6 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreamsBase static constexpr size_t kMaxStreams = 3; }; -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -alignas(16) constexpr uint64_t - CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::k1k2[2]; -alignas(16) constexpr uint64_t - CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::k3k4[2]; -alignas(16) constexpr uint64_t - CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::k5k6[2]; -alignas(16) constexpr uint64_t - CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::k7k0[2]; -alignas(16) constexpr uint64_t - CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::kPoly[2]; -alignas(16) constexpr uint32_t - CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::kMask[4]; -constexpr size_t - CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::kGroupsSmall; -constexpr size_t CRC32AcceleratedX86ARMCombinedMultipleStreamsBase::kMaxStreams; -#endif // ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL - template class CRC32AcceleratedX86ARMCombinedMultipleStreams @@ -384,15 +355,15 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreams length &= ~size_t{8}; } if (length & 4) { - ABSL_INTERNAL_STEP4(l); + ABSL_INTERNAL_STEP4(l, p); length &= ~size_t{4}; } if (length & 2) { - ABSL_INTERNAL_STEP2(l); + ABSL_INTERNAL_STEP2(l, p); length &= ~size_t{2}; } if (length & 1) { - ABSL_INTERNAL_STEP1(l); + ABSL_INTERNAL_STEP1(l, p); length &= ~size_t{1}; } if (length == 0) { @@ -478,7 +449,7 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreams const uint8_t* x = RoundUp<8>(p); // Process bytes until p is 8-byte aligned, if that isn't past the end. while (p != x) { - ABSL_INTERNAL_STEP1(l); + ABSL_INTERNAL_STEP1(l, p); } size_t bs = static_cast(e - p) / @@ -597,7 +568,7 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreams } // Process the last few bytes while (p != e) { - ABSL_INTERNAL_STEP1(l); + ABSL_INTERNAL_STEP1(l, p); } #undef ABSL_INTERNAL_STEP8BY3 diff --git a/absl/debugging/internal/addresses.h b/absl/debugging/internal/addresses.h new file mode 100644 index 0000000..504fd6f --- /dev/null +++ b/absl/debugging/internal/addresses.h @@ -0,0 +1,57 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_DEBUGGING_INTERNAL_ADDRESSES_H_ +#define ABSL_DEBUGGING_INTERNAL_ADDRESSES_H_ + +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Removes any metadata (tag bits) from the given pointer, converting it into a +// user-readable address. +inline uintptr_t StripPointerMetadata(uintptr_t ptr) { +#if defined(__aarch64__) + // When PAC-RET (-mbranch-protection=pac-ret) is enabled, return addresses + // stored on the stack will be signed, which means that pointer bits outside + // of the virtual address range are potentially set. Since the stacktrace code + // is expected to return normal code pointers, this function clears those + // bits. + register uintptr_t x30 __asm__("x30") = ptr; + // The normal instruction for clearing PAC bits is XPACI, but for + // compatibility with ARM platforms that do not support pointer + // authentication, we use the hint space instruction XPACLRI instead. Hint + // space instructions behave as NOPs on unsupported platforms. +#define ABSL_XPACLRI_HINT "hint #0x7;" + asm(ABSL_XPACLRI_HINT : "+r"(x30)); // asm("xpaclri" : "+r"(x30)); +#undef ABSL_XPACLRI_HINT + return x30; +#else + return ptr; +#endif +} + +inline uintptr_t StripPointerMetadata(void* ptr) { + return StripPointerMetadata(reinterpret_cast(ptr)); +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_ADDRESSES_H_ diff --git a/absl/debugging/internal/decode_rust_punycode.cc b/absl/debugging/internal/decode_rust_punycode.cc index 43b46bf..6652dc2 100644 --- a/absl/debugging/internal/decode_rust_punycode.cc +++ b/absl/debugging/internal/decode_rust_punycode.cc @@ -172,7 +172,7 @@ bool ScanNextDelta(const char*& punycode_begin, const char* const punycode_end, } // namespace -absl::Nullable DecodeRustPunycode(DecodeRustPunycodeOptions options) { +char* absl_nullable DecodeRustPunycode(DecodeRustPunycodeOptions options) { const char* punycode_begin = options.punycode_begin; const char* const punycode_end = options.punycode_end; char* const out_begin = options.out_begin; diff --git a/absl/debugging/internal/decode_rust_punycode.h b/absl/debugging/internal/decode_rust_punycode.h index 0ae53ff..44aad8a 100644 --- a/absl/debugging/internal/decode_rust_punycode.h +++ b/absl/debugging/internal/decode_rust_punycode.h @@ -23,10 +23,10 @@ ABSL_NAMESPACE_BEGIN namespace debugging_internal { struct DecodeRustPunycodeOptions { - const char* punycode_begin; - const char* punycode_end; - char* out_begin; - char* out_end; + const char* absl_nonnull punycode_begin; + const char* absl_nonnull punycode_end; + char* absl_nonnull out_begin; + char* absl_nonnull out_end; }; // Given Rust Punycode in `punycode_begin .. punycode_end`, writes the @@ -46,7 +46,7 @@ struct DecodeRustPunycodeOptions { // DecodeRustPunycode is async-signal-safe with bounded runtime and a small // stack footprint, making it suitable for use in demangling Rust symbol names // from a signal handler. -absl::Nullable DecodeRustPunycode(DecodeRustPunycodeOptions options); +char* absl_nullable DecodeRustPunycode(DecodeRustPunycodeOptions options); } // namespace debugging_internal ABSL_NAMESPACE_END diff --git a/absl/debugging/internal/demangle.cc b/absl/debugging/internal/demangle.cc index f7de117..5f62ebb 100644 --- a/absl/debugging/internal/demangle.cc +++ b/absl/debugging/internal/demangle.cc @@ -484,36 +484,6 @@ static bool IsAlpha(char c) { static bool IsDigit(char c) { return c >= '0' && c <= '9'; } -// Returns true if "str" is a function clone suffix. These suffixes are used -// by GCC 4.5.x and later versions (and our locally-modified version of GCC -// 4.4.x) to indicate functions which have been cloned during optimization. -// We treat any sequence (.+.+)+ as a function clone suffix. -// Additionally, '_' is allowed along with the alphanumeric sequence. -static bool IsFunctionCloneSuffix(const char *str) { - size_t i = 0; - while (str[i] != '\0') { - bool parsed = false; - // Consume a single [. | _]*[.]* sequence. - if (str[i] == '.' && (IsAlpha(str[i + 1]) || str[i + 1] == '_')) { - parsed = true; - i += 2; - while (IsAlpha(str[i]) || str[i] == '_') { - ++i; - } - } - if (str[i] == '.' && IsDigit(str[i + 1])) { - parsed = true; - i += 2; - while (IsDigit(str[i])) { - ++i; - } - } - if (!parsed) - return false; - } - return true; // Consumed everything in "str". -} - static bool EndsWith(State *state, const char chr) { return state->parse_state.out_cur_idx > 0 && state->parse_state.out_cur_idx < state->out_end_idx && @@ -1039,7 +1009,8 @@ static bool ParseNumber(State *state, int *number_out) { number = ~number + 1; } if (p != RemainingInput(state)) { // Conversion succeeded. - state->parse_state.mangled_idx += p - RemainingInput(state); + state->parse_state.mangled_idx += + static_cast(p - RemainingInput(state)); UpdateHighWaterMark(state); if (number_out != nullptr) { // Note: possibly truncate "number". @@ -1062,7 +1033,8 @@ static bool ParseFloatNumber(State *state) { } } if (p != RemainingInput(state)) { // Conversion succeeded. - state->parse_state.mangled_idx += p - RemainingInput(state); + state->parse_state.mangled_idx += + static_cast(p - RemainingInput(state)); UpdateHighWaterMark(state); return true; } @@ -1081,7 +1053,8 @@ static bool ParseSeqId(State *state) { } } if (p != RemainingInput(state)) { // Conversion succeeded. - state->parse_state.mangled_idx += p - RemainingInput(state); + state->parse_state.mangled_idx += + static_cast(p - RemainingInput(state)); UpdateHighWaterMark(state); return true; } @@ -1100,7 +1073,7 @@ static bool ParseIdentifier(State *state, size_t length) { } else { MaybeAppendWithLength(state, RemainingInput(state), length); } - state->parse_state.mangled_idx += length; + state->parse_state.mangled_idx += static_cast(length); UpdateHighWaterMark(state); return true; } @@ -2929,7 +2902,7 @@ static bool ParseTopLevelMangledName(State *state) { if (ParseMangledName(state)) { if (RemainingInput(state)[0] != '\0') { // Drop trailing function clone suffix, if any. - if (IsFunctionCloneSuffix(RemainingInput(state))) { + if (RemainingInput(state)[0] == '.') { return true; } // Append trailing version suffix if any. diff --git a/absl/debugging/internal/demangle_rust.cc b/absl/debugging/internal/demangle_rust.cc index 4309bd8..f7f6713 100644 --- a/absl/debugging/internal/demangle_rust.cc +++ b/absl/debugging/internal/demangle_rust.cc @@ -84,7 +84,7 @@ class RustSymbolParser { // structure was not recognized or exceeded implementation limits, such as by // nesting structures too deep. In either case *this should not be used // again. - ABSL_MUST_USE_RESULT bool Parse() && { + [[nodiscard]] bool Parse() && { // Recursively parses the grammar production named by callee, then resumes // execution at the next statement. // @@ -564,7 +564,7 @@ class RustSymbolParser { // If the next input character is the given character, consumes it and returns // true; otherwise returns false without consuming a character. - ABSL_MUST_USE_RESULT bool Eat(char want) { + [[nodiscard]] bool Eat(char want) { if (encoding_[pos_] != want) return false; ++pos_; return true; @@ -573,7 +573,7 @@ class RustSymbolParser { // Provided there is enough remaining output space, appends c to the output, // writing a fresh NUL terminator afterward, and returns true. Returns false // if the output buffer had less than two bytes free. - ABSL_MUST_USE_RESULT bool EmitChar(char c) { + [[nodiscard]] bool EmitChar(char c) { if (silence_depth_ > 0) return true; if (out_end_ - out_ < 2) return false; *out_++ = c; @@ -584,7 +584,7 @@ class RustSymbolParser { // Provided there is enough remaining output space, appends the C string token // to the output, followed by a NUL character, and returns true. Returns // false if not everything fit into the output buffer. - ABSL_MUST_USE_RESULT bool Emit(const char* token) { + [[nodiscard]] bool Emit(const char* token) { if (silence_depth_ > 0) return true; const size_t token_length = std::strlen(token); const size_t bytes_to_copy = token_length + 1; // token and final NUL @@ -598,7 +598,7 @@ class RustSymbolParser { // of disambiguator (if it's nonnegative) or "?" (if it's negative) to the // output, followed by a NUL character, and returns true. Returns false if // not everything fit into the output buffer. - ABSL_MUST_USE_RESULT bool EmitDisambiguator(int disambiguator) { + [[nodiscard]] bool EmitDisambiguator(int disambiguator) { if (disambiguator < 0) return EmitChar('?'); // parsed but too large if (disambiguator == 0) return EmitChar('0'); // Convert disambiguator to decimal text. Three digits per byte is enough @@ -618,7 +618,7 @@ class RustSymbolParser { // On success returns true and fills value with the encoded value if it was // not too big, otherwise with -1. If the optional disambiguator was omitted, // value is 0. On parse failure returns false and sets value to -1. - ABSL_MUST_USE_RESULT bool ParseDisambiguator(int& value) { + [[nodiscard]] bool ParseDisambiguator(int& value) { value = -1; // disambiguator = s base-62-number @@ -639,7 +639,7 @@ class RustSymbolParser { // On success returns true and fills value with the encoded value if it was // not too big, otherwise with -1. On parse failure returns false and sets // value to -1. - ABSL_MUST_USE_RESULT bool ParseBase62Number(int& value) { + [[nodiscard]] bool ParseBase62Number(int& value) { value = -1; // base-62-number = (digit | lower | upper)* _ @@ -686,7 +686,7 @@ class RustSymbolParser { // A nonzero uppercase_namespace specifies the character after the N in a // nested-identifier, e.g., 'C' for a closure, allowing ParseIdentifier to // write out the name with the conventional decoration for that namespace. - ABSL_MUST_USE_RESULT bool ParseIdentifier(char uppercase_namespace = '\0') { + [[nodiscard]] bool ParseIdentifier(char uppercase_namespace = '\0') { // identifier -> disambiguator? undisambiguated-identifier int disambiguator = 0; if (!ParseDisambiguator(disambiguator)) return false; @@ -703,7 +703,7 @@ class RustSymbolParser { // // At other appearances of undisambiguated-identifier in the grammar, this // treatment is not applicable, and the call site omits both arguments. - ABSL_MUST_USE_RESULT bool ParseUndisambiguatedIdentifier( + [[nodiscard]] bool ParseUndisambiguatedIdentifier( char uppercase_namespace = '\0', int disambiguator = 0) { // undisambiguated-identifier -> u? decimal-number _? bytes const bool is_punycoded = Eat('u'); @@ -766,7 +766,7 @@ class RustSymbolParser { // Consumes a decimal number like 0 or 123 from the input. On success returns // true and fills value with the encoded value. If the encoded value is too // large or otherwise unparsable, returns false and sets value to -1. - ABSL_MUST_USE_RESULT bool ParseDecimalNumber(int& value) { + [[nodiscard]] bool ParseDecimalNumber(int& value) { value = -1; if (!IsDigit(Peek())) return false; int encoded_number = Take() - '0'; @@ -788,7 +788,7 @@ class RustSymbolParser { // Consumes a binder of higher-ranked lifetimes if one is present. On success // returns true and discards the encoded lifetime count. On parse failure // returns false. - ABSL_MUST_USE_RESULT bool ParseOptionalBinder() { + [[nodiscard]] bool ParseOptionalBinder() { // binder -> G base-62-number if (!Eat('G')) return true; int ignored_binding_count; @@ -802,7 +802,7 @@ class RustSymbolParser { // things we omit from output, such as the entire contents of generic-args. // // On parse failure returns false. - ABSL_MUST_USE_RESULT bool ParseOptionalLifetime() { + [[nodiscard]] bool ParseOptionalLifetime() { // lifetime -> L base-62-number if (!Eat('L')) return true; int ignored_de_bruijn_index; @@ -811,14 +811,14 @@ class RustSymbolParser { // Consumes a lifetime just like ParseOptionalLifetime, but returns false if // there is no lifetime here. - ABSL_MUST_USE_RESULT bool ParseRequiredLifetime() { + [[nodiscard]] bool ParseRequiredLifetime() { if (Peek() != 'L') return false; return ParseOptionalLifetime(); } // Pushes ns onto the namespace stack and returns true if the stack is not // full, else returns false. - ABSL_MUST_USE_RESULT bool PushNamespace(char ns) { + [[nodiscard]] bool PushNamespace(char ns) { if (namespace_depth_ == kNamespaceStackSize) return false; namespace_stack_[namespace_depth_++] = ns; return true; @@ -830,7 +830,7 @@ class RustSymbolParser { // Pushes position onto the position stack and returns true if the stack is // not full, else returns false. - ABSL_MUST_USE_RESULT bool PushPosition(int position) { + [[nodiscard]] bool PushPosition(int position) { if (position_depth_ == kPositionStackSize) return false; position_stack_[position_depth_++] = position; return true; @@ -845,7 +845,7 @@ class RustSymbolParser { // beginning of the backref target. Returns true on success. Returns false // if parsing failed, the stack is exhausted, or the backref target position // is out of range. - ABSL_MUST_USE_RESULT bool BeginBackref() { + [[nodiscard]] bool BeginBackref() { // backref = B base-62-number (B already consumed) // // Reject backrefs that don't parse, overflow int, or don't point backward. diff --git a/absl/debugging/internal/stacktrace_aarch64-inl.inc b/absl/debugging/internal/stacktrace_aarch64-inl.inc index dccadae..1746b5d 100644 --- a/absl/debugging/internal/stacktrace_aarch64-inl.inc +++ b/absl/debugging/internal/stacktrace_aarch64-inl.inc @@ -18,6 +18,7 @@ #include "absl/base/attributes.h" #include "absl/debugging/internal/address_is_readable.h" +#include "absl/debugging/internal/addresses.h" #include "absl/debugging/internal/vdso_support.h" // a no-op on non-elf or non-glibc systems #include "absl/debugging/stacktrace.h" @@ -101,7 +102,8 @@ static bool InsideSignalStack(void** ptr, const StackInfo* stack_info) { // "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. template ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. -ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. static void **NextStackFrame(void **old_frame_pointer, const void *uc, const StackInfo *stack_info) { void **new_frame_pointer = reinterpret_cast(*old_frame_pointer); @@ -124,6 +126,7 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, if (pre_signal_frame_pointer >= old_frame_pointer) { new_frame_pointer = pre_signal_frame_pointer; } + } } #endif @@ -131,17 +134,13 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, if ((reinterpret_cast(new_frame_pointer) & 7) != 0) return nullptr; - // Check that alleged frame pointer is actually readable. This is to - // prevent "double fault" in case we hit the first fault due to e.g. - // stack corruption. - if (!absl::debugging_internal::AddressIsReadable( - new_frame_pointer)) - return nullptr; - } - + uintptr_t new_fp_comparable = reinterpret_cast(new_frame_pointer); // Only check the size if both frames are in the same stack. - if (InsideSignalStack(new_frame_pointer, stack_info) == - InsideSignalStack(old_frame_pointer, stack_info)) { + const bool old_inside_signal_stack = + InsideSignalStack(old_frame_pointer, stack_info); + const bool new_inside_signal_stack = + InsideSignalStack(new_frame_pointer, stack_info); + if (new_inside_signal_stack == old_inside_signal_stack) { // Check frame size. In strict mode, we assume frames to be under // 100,000 bytes. In non-strict mode, we relax the limit to 1MB. const size_t max_size = STRICT_UNWINDING ? 100000 : 1000000; @@ -155,16 +154,15 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, if (frame_size > max_size) { size_t stack_low = stack_info->stack_low; size_t stack_high = stack_info->stack_high; - if (InsideSignalStack(new_frame_pointer, stack_info)) { + if (new_inside_signal_stack) { stack_low = stack_info->sig_stack_low; stack_high = stack_info->sig_stack_high; } if (stack_high < kUnknownStackEnd && static_cast(getpagesize()) < stack_low) { - const uintptr_t new_fp_u = - reinterpret_cast(new_frame_pointer); // Stack bounds are known. - if (!(stack_low < new_fp_u && new_fp_u <= stack_high)) { + if (!(stack_low < new_fp_comparable && + new_fp_comparable <= stack_high)) { // new_frame_pointer is not within a known stack. return nullptr; } @@ -174,24 +172,19 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, } } } + // New frame pointer is valid if it is inside either known stack or readable. + // This assumes that everything within either known stack is readable. Outside + // either known stack but readable is unexpected, and possibly corrupt, but + // for now assume it is valid. If it isn't actually valid, the next frame will + // be corrupt and we will detect that next iteration. + if (new_inside_signal_stack || + (new_fp_comparable >= stack_info->stack_low && + new_fp_comparable < stack_info->stack_high) || + absl::debugging_internal::AddressIsReadable(new_frame_pointer)) { + return new_frame_pointer; + } - return new_frame_pointer; -} - -// When PAC-RET (-mbranch-protection=pac-ret) is enabled, return addresses -// stored on the stack will be signed, which means that pointer bits outside of -// the VA range are potentially set. Since the stacktrace code is expected to -// return normal code pointers, this function clears those bits. -inline void* ClearPacBits(void* ptr) { - register void* x30 __asm__("x30") = ptr; - // The normal instruction for clearing PAC bits is XPACI, but for - // compatibility with ARM platforms that do not support pointer - // authentication, we use the hint space instruction XPACLRI instead. Hint - // space instructions behave as NOPs on unsupported platforms. -#define ABSL_XPACLRI_HINT "hint #0x7;" - asm(ABSL_XPACLRI_HINT : "+r"(x30)); // asm("xpaclri" : "+r"(x30)); -#undef ABSL_XPACLRI_HINT - return x30; + return nullptr; } template @@ -200,8 +193,10 @@ template ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. -static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, - const void *ucp, int *min_dropped_frames) { +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. +static int UnwindImpl(void **result, uintptr_t *frames, int *sizes, + int max_depth, int skip_count, const void *ucp, + int *min_dropped_frames) { #ifdef __GNUC__ void **frame_pointer = reinterpret_cast(__builtin_frame_address(0)); #else @@ -235,10 +230,18 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, if (skip_count > 0) { skip_count--; } else { - result[n] = ClearPacBits(prev_return_address); + result[n] = reinterpret_cast( + absl::debugging_internal::StripPointerMetadata(prev_return_address)); if (IS_STACK_FRAMES) { - sizes[n] = static_cast( - ComputeStackFrameSize(prev_frame_pointer, frame_pointer)); + if (frames != nullptr) { + frames[n] = absl::debugging_internal::StripPointerMetadata( + prev_frame_pointer) + + 2 * sizeof(void *) /* go past the return address */; + } + if (sizes != nullptr) { + sizes[n] = static_cast( + ComputeStackFrameSize(prev_frame_pointer, frame_pointer)); + } } n++; } diff --git a/absl/debugging/internal/stacktrace_arm-inl.inc b/absl/debugging/internal/stacktrace_arm-inl.inc index 102a2a1..3feb521 100644 --- a/absl/debugging/internal/stacktrace_arm-inl.inc +++ b/absl/debugging/internal/stacktrace_arm-inl.inc @@ -19,6 +19,7 @@ #include +#include "absl/debugging/internal/addresses.h" #include "absl/debugging/stacktrace.h" // WARNING: @@ -67,8 +68,9 @@ void StacktraceArmDummyFunction() { __asm__ volatile(""); } #endif template -static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, - const void * /* ucp */, int *min_dropped_frames) { +static int UnwindImpl(void **result, uintptr_t *frames, int *sizes, + int max_depth, int skip_count, const void * /* ucp */, + int *min_dropped_frames) { #ifdef __GNUC__ void **sp = reinterpret_cast(__builtin_frame_address(0)); #else @@ -97,11 +99,18 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, result[n] = *sp; if (IS_STACK_FRAMES) { - if (next_sp > sp) { - sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; - } else { - // A frame-size of 0 is used to indicate unknown frame size. - sizes[n] = 0; + if (frames != nullptr) { + frames[n] = absl::debugging_internal::StripPointerMetadata(sp) + + 1 * sizeof(void *) /* go past the return address */; + } + if (sizes != nullptr) { + if (next_sp > sp) { + sizes[n] = absl::debugging_internal::StripPointerMetadata(next_sp) - + absl::debugging_internal::StripPointerMetadata(sp); + } else { + // A frame-size of 0 is used to indicate unknown frame size. + sizes[n] = 0; + } } } n++; diff --git a/absl/debugging/internal/stacktrace_emscripten-inl.inc b/absl/debugging/internal/stacktrace_emscripten-inl.inc index 0f44451..2f39c70 100644 --- a/absl/debugging/internal/stacktrace_emscripten-inl.inc +++ b/absl/debugging/internal/stacktrace_emscripten-inl.inc @@ -21,8 +21,10 @@ #define ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ #include +#include #include +#include #include #include "absl/base/attributes.h" @@ -62,8 +64,9 @@ ABSL_ATTRIBUTE_UNUSED static int stacktraces_enabler = []() { }(); template -static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, - const void *ucp, int *min_dropped_frames) { +static int UnwindImpl(void **result, uintptr_t *frames, int *sizes, + int max_depth, int skip_count, const void *ucp, + int *min_dropped_frames) { if (recursive || disable_stacktraces.load(std::memory_order_relaxed)) { return 0; } @@ -75,7 +78,8 @@ static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, int size; uintptr_t pc = emscripten_stack_snapshot(); - size = emscripten_stack_unwind_buffer(pc, stack, kStackLength); + size = + static_cast(emscripten_stack_unwind_buffer(pc, stack, kStackLength)); int result_count = size - skip_count; if (result_count < 0) result_count = 0; @@ -83,8 +87,13 @@ static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, for (int i = 0; i < result_count; i++) result[i] = stack[i + skip_count]; if (IS_STACK_FRAMES) { - // No implementation for finding out the stack frame sizes yet. - memset(sizes, 0, sizeof(*sizes) * result_count); + // No implementation for finding out the stack frames yet. + if (frames != nullptr) { + memset(frames, 0, sizeof(*frames) * static_cast(result_count)); + } + if (sizes != nullptr) { + memset(sizes, 0, sizeof(*sizes) * static_cast(result_count)); + } } if (min_dropped_frames != nullptr) { if (size - skip_count - max_depth > 0) { diff --git a/absl/debugging/internal/stacktrace_generic-inl.inc b/absl/debugging/internal/stacktrace_generic-inl.inc index 5fa169a..e7a11fc 100644 --- a/absl/debugging/internal/stacktrace_generic-inl.inc +++ b/absl/debugging/internal/stacktrace_generic-inl.inc @@ -56,8 +56,9 @@ ABSL_ATTRIBUTE_UNUSED static int stacktraces_enabler = []() { }(); template -static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, - const void *ucp, int *min_dropped_frames) { +static int UnwindImpl(void** result, uintptr_t* frames, int* sizes, + int max_depth, int skip_count, const void* ucp, + int* min_dropped_frames) { if (recursive || disable_stacktraces.load(std::memory_order_relaxed)) { return 0; } @@ -79,8 +80,13 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, result[i] = stack[i + skip_count]; if (IS_STACK_FRAMES) { - // No implementation for finding out the stack frame sizes yet. - memset(sizes, 0, sizeof(*sizes) * static_cast(result_count)); + // No implementation for finding out the stack frames yet. + if (frames != nullptr) { + memset(frames, 0, sizeof(*frames) * static_cast(result_count)); + } + if (sizes != nullptr) { + memset(sizes, 0, sizeof(*sizes) * static_cast(result_count)); + } } if (min_dropped_frames != nullptr) { if (size - skip_count - max_depth > 0) { diff --git a/absl/debugging/internal/stacktrace_powerpc-inl.inc b/absl/debugging/internal/stacktrace_powerpc-inl.inc index a49ed2f..f82ca8f 100644 --- a/absl/debugging/internal/stacktrace_powerpc-inl.inc +++ b/absl/debugging/internal/stacktrace_powerpc-inl.inc @@ -21,6 +21,7 @@ #ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_POWERPC_INL_H_ #define ABSL_DEBUGGING_INTERNAL_STACKTRACE_POWERPC_INL_H_ +#include "absl/debugging/internal/addresses.h" #if defined(__linux__) #include // for PT_NIP. #include // for ucontext_t @@ -40,22 +41,22 @@ // Given a stack pointer, return the saved link register value. // Note that this is the link register for a callee. -static inline void *StacktracePowerPCGetLR(void **sp) { +static inline void **StacktracePowerPCGetLRPtr(void **sp) { // PowerPC has 3 main ABIs, which say where in the stack the // Link Register is. For DARWIN and AIX (used by apple and // linux ppc64), it's in sp[2]. For SYSV (used by linux ppc), // it's in sp[1]. #if defined(_CALL_AIX) || defined(_CALL_DARWIN) - return *(sp+2); + return (sp + 2); #elif defined(_CALL_SYSV) - return *(sp+1); + return (sp + 1); #elif defined(__APPLE__) || defined(__FreeBSD__) || \ (defined(__linux__) && defined(__PPC64__)) // This check is in case the compiler doesn't define _CALL_AIX/etc. - return *(sp+2); + return (sp + 2); #elif defined(__linux) // This check is in case the compiler doesn't define _CALL_SYSV. - return *(sp+1); + return (sp + 1); #else #error Need to specify the PPC ABI for your architecture. #endif @@ -68,6 +69,7 @@ static inline void *StacktracePowerPCGetLR(void **sp) { template ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. static void **NextStackFrame(void **old_sp, const void *uc) { void **new_sp = (void **) *old_sp; enum { kStackAlignment = 16 }; @@ -125,9 +127,8 @@ static void **NextStackFrame(void **old_sp, const void *uc) { } } - if (new_sp != nullptr && - kernel_symbol_status == kAddressValid && - StacktracePowerPCGetLR(new_sp) == kernel_sigtramp_rt64_address) { + if (new_sp != nullptr && kernel_symbol_status == kAddressValid && + *StacktracePowerPCGetLRPtr(new_sp) == kernel_sigtramp_rt64_address) { const ucontext_t* signal_context = reinterpret_cast(uc); void **const sp_before_signal = @@ -164,8 +165,10 @@ ABSL_ATTRIBUTE_NOINLINE static void AbslStacktracePowerPCDummyFunction() { template ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. -static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, - const void *ucp, int *min_dropped_frames) { +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. +static int UnwindImpl(void **result, uintptr_t *frames, int *sizes, + int max_depth, int skip_count, const void *ucp, + int *min_dropped_frames) { void **sp; // Apple macOS uses an old version of gnu as -- both Darwin 7.9.0 (Panther) // and Darwin 8.8.1 (Tiger) use as 1.38. This means we have to use a @@ -211,13 +214,21 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, if (skip_count > 0) { skip_count--; } else { - result[n] = StacktracePowerPCGetLR(sp); + void **lr = StacktracePowerPCGetLRPtr(sp); + result[n] = *lr; if (IS_STACK_FRAMES) { - if (next_sp > sp) { - sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; - } else { - // A frame-size of 0 is used to indicate unknown frame size. - sizes[n] = 0; + if (frames != nullptr) { + frames[n] = absl::debugging_internal::StripPointerMetadata(lr) + + 1 * sizeof(void *) /* go past the return address */; + } + if (sizes != nullptr) { + if (next_sp > sp) { + sizes[n] = absl::debugging_internal::StripPointerMetadata(next_sp) - + absl::debugging_internal::StripPointerMetadata(sp); + } else { + // A frame-size of 0 is used to indicate unknown frame size. + sizes[n] = 0; + } } } n++; diff --git a/absl/debugging/internal/stacktrace_riscv-inl.inc b/absl/debugging/internal/stacktrace_riscv-inl.inc index 3f9e124..f9919c6 100644 --- a/absl/debugging/internal/stacktrace_riscv-inl.inc +++ b/absl/debugging/internal/stacktrace_riscv-inl.inc @@ -20,6 +20,7 @@ #include #include "absl/base/config.h" +#include "absl/debugging/internal/addresses.h" #if defined(__linux__) #include #include @@ -55,6 +56,7 @@ static inline ptrdiff_t ComputeStackFrameSize(const T *low, const T *high) { template ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. static void ** NextStackFrame(void **old_frame_pointer, const void *uc, const std::pair range) { // . @@ -117,8 +119,10 @@ static void ** NextStackFrame(void **old_frame_pointer, const void *uc, template ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. -static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, - const void *ucp, int *min_dropped_frames) { +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. +static int UnwindImpl(void **result, uintptr_t *frames, int *sizes, + int max_depth, int skip_count, const void *ucp, + int *min_dropped_frames) { // The `frame_pointer` that is computed here points to the top of the frame. // The two words preceding the address are the return address and the previous // frame pointer. @@ -153,8 +157,13 @@ static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, result[n] = return_address; if (IS_STACK_FRAMES) { // NextStackFrame() has already checked that frame size fits to int - sizes[n] = static_cast(ComputeStackFrameSize(frame_pointer, - next_frame_pointer)); + if (frames != nullptr) { + frames[n] = + absl::debugging_internal::StripPointerMetadata(frame_pointer); + } + if (sizes != nullptr) { + sizes[n] = ComputeStackFrameSize(frame_pointer, next_frame_pointer); + } } n++; } diff --git a/absl/debugging/internal/stacktrace_unimplemented-inl.inc b/absl/debugging/internal/stacktrace_unimplemented-inl.inc index 5b8fb19..ec63940 100644 --- a/absl/debugging/internal/stacktrace_unimplemented-inl.inc +++ b/absl/debugging/internal/stacktrace_unimplemented-inl.inc @@ -2,9 +2,10 @@ #define ABSL_DEBUGGING_INTERNAL_STACKTRACE_UNIMPLEMENTED_INL_H_ template -static int UnwindImpl(void** /* result */, int* /* sizes */, - int /* max_depth */, int /* skip_count */, - const void* /* ucp */, int *min_dropped_frames) { +static int UnwindImpl(void** /* result */, uintptr_t* /* frames */, + int* /* sizes */, int /* max_depth */, + int /* skip_count */, const void* /* ucp */, + int* min_dropped_frames) { if (min_dropped_frames != nullptr) { *min_dropped_frames = 0; } diff --git a/absl/debugging/internal/stacktrace_win32-inl.inc b/absl/debugging/internal/stacktrace_win32-inl.inc index ef2b973..f57c187 100644 --- a/absl/debugging/internal/stacktrace_win32-inl.inc +++ b/absl/debugging/internal/stacktrace_win32-inl.inc @@ -37,42 +37,29 @@ #ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_WIN32_INL_H_ #define ABSL_DEBUGGING_INTERNAL_STACKTRACE_WIN32_INL_H_ -#include // for GetProcAddress and GetModuleHandle -#include - -typedef USHORT NTAPI RtlCaptureStackBackTrace_Function( - IN ULONG frames_to_skip, - IN ULONG frames_to_capture, - OUT PVOID *backtrace, - OUT PULONG backtrace_hash); +#include // CaptureStackBackTrace -// It is not possible to load RtlCaptureStackBackTrace at static init time in -// UWP. CaptureStackBackTrace is the public version of RtlCaptureStackBackTrace -#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) && \ - !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) -static RtlCaptureStackBackTrace_Function* const RtlCaptureStackBackTrace_fn = - &::CaptureStackBackTrace; -#else -// Load the function we need at static init time, where we don't have -// to worry about someone else holding the loader's lock. -static RtlCaptureStackBackTrace_Function* const RtlCaptureStackBackTrace_fn = - (RtlCaptureStackBackTrace_Function*)GetProcAddress( - GetModuleHandleA("ntdll.dll"), "RtlCaptureStackBackTrace"); -#endif // WINAPI_PARTITION_APP && !WINAPI_PARTITION_DESKTOP +#include template -static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, - const void*, int* min_dropped_frames) { +static int UnwindImpl(void** result, uintptr_t* frames, int* sizes, + int max_depth, int skip_count, const void*, + int* min_dropped_frames) { USHORT n = 0; - if (!RtlCaptureStackBackTrace_fn || skip_count < 0 || max_depth < 0) { + if (skip_count < 0 || max_depth < 0) { // can't get a stacktrace with no function/invalid args } else { - n = RtlCaptureStackBackTrace_fn(static_cast(skip_count) + 2, - static_cast(max_depth), result, 0); + n = CaptureStackBackTrace(static_cast(skip_count) + 2, + static_cast(max_depth), result, 0); } if (IS_STACK_FRAMES) { - // No implementation for finding out the stack frame sizes yet. - memset(sizes, 0, sizeof(*sizes) * n); + // No implementation for finding out the stack frames yet. + if (frames != nullptr) { + memset(frames, 0, sizeof(*frames) * n); + } + if (sizes != nullptr) { + memset(sizes, 0, sizeof(*sizes) * n); + } } if (min_dropped_frames != nullptr) { // Not implemented. diff --git a/absl/debugging/internal/stacktrace_x86-inl.inc b/absl/debugging/internal/stacktrace_x86-inl.inc index 1975ba7..96b128e 100644 --- a/absl/debugging/internal/stacktrace_x86-inl.inc +++ b/absl/debugging/internal/stacktrace_x86-inl.inc @@ -33,6 +33,7 @@ #include "absl/base/macros.h" #include "absl/base/port.h" #include "absl/debugging/internal/address_is_readable.h" +#include "absl/debugging/internal/addresses.h" #include "absl/debugging/internal/vdso_support.h" // a no-op on non-elf or non-glibc systems #include "absl/debugging/stacktrace.h" @@ -163,6 +164,7 @@ static uintptr_t GetFP(const void *vuc) { template ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. static void **NextStackFrame(void **old_fp, const void *uc, size_t stack_low, size_t stack_high) { void **new_fp = (void **)*old_fp; @@ -326,9 +328,11 @@ static void **NextStackFrame(void **old_fp, const void *uc, template ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_THREAD // May read random elements from stack. ABSL_ATTRIBUTE_NOINLINE -static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, - const void *ucp, int *min_dropped_frames) { +static int UnwindImpl(void **result, uintptr_t *frames, int *sizes, + int max_depth, int skip_count, const void *ucp, + int *min_dropped_frames) { int n = 0; void **fp = reinterpret_cast(__builtin_frame_address(0)); @@ -349,13 +353,19 @@ static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, } else { result[n] = *(fp + 1); if (IS_STACK_FRAMES) { - if (next_fp > fp) { - sizes[n] = static_cast( - reinterpret_cast(next_fp) - - reinterpret_cast(fp)); - } else { - // A frame-size of 0 is used to indicate unknown frame size. - sizes[n] = 0; + if (frames) { + frames[n] = absl::debugging_internal::StripPointerMetadata(fp) + + 2 * sizeof(void *) /* go past the return address */; + } + if (sizes) { + if (next_fp > fp) { + sizes[n] = static_cast( + absl::debugging_internal::StripPointerMetadata(next_fp) - + absl::debugging_internal::StripPointerMetadata(fp)); + } else { + // A frame-size of 0 is used to indicate unknown frame size. + sizes[n] = 0; + } } } n++; diff --git a/absl/debugging/stacktrace.cc b/absl/debugging/stacktrace.cc index ff8069f..f71e80c 100644 --- a/absl/debugging/stacktrace.cc +++ b/absl/debugging/stacktrace.cc @@ -36,12 +36,42 @@ #include "absl/debugging/stacktrace.h" +#include +#include + +#include #include #include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" #include "absl/base/port.h" #include "absl/debugging/internal/stacktrace_config.h" +#ifdef ABSL_INTERNAL_HAVE_ALLOCA +#error ABSL_INTERNAL_HAVE_ALLOCA cannot be directly set +#endif + +#ifdef _WIN32 +#include +#define ABSL_INTERNAL_HAVE_ALLOCA 1 +#else +#ifdef __has_include +#if __has_include() +#include +#define ABSL_INTERNAL_HAVE_ALLOCA 1 +#elif !defined(alloca) +static void* alloca(size_t) noexcept { return nullptr; } +#endif +#endif +#endif + +#ifdef ABSL_INTERNAL_HAVE_ALLOCA +static constexpr bool kHaveAlloca = true; +#else +static constexpr bool kHaveAlloca = false; +#endif + #if defined(ABSL_STACKTRACE_INL_HEADER) #include ABSL_STACKTRACE_INL_HEADER #else @@ -66,59 +96,111 @@ typedef int (*Unwinder)(void**, int*, int, int, const void*, int*); std::atomic custom; template -ABSL_ATTRIBUTE_ALWAYS_INLINE inline int Unwind(void** result, int* sizes, - int max_depth, int skip_count, - const void* uc, +ABSL_ATTRIBUTE_ALWAYS_INLINE inline int Unwind(void** result, uintptr_t* frames, + int* sizes, int max_depth, + int skip_count, const void* uc, int* min_dropped_frames) { - Unwinder f = &UnwindImpl; Unwinder g = custom.load(std::memory_order_acquire); - if (g != nullptr) f = g; - + int size; // Add 1 to skip count for the unwinder function itself - int size = (*f)(result, sizes, max_depth, skip_count + 1, uc, - min_dropped_frames); - // To disable tail call to (*f)(...) + ++skip_count; + if (g != nullptr) { + size = (*g)(result, sizes, max_depth, skip_count, uc, min_dropped_frames); + // Frame pointers aren't returned by existing hooks, so clear them. + if (frames != nullptr) { + std::fill(frames, frames + size, uintptr_t()); + } + } else { + size = UnwindImpl( + result, frames, sizes, max_depth, skip_count, uc, min_dropped_frames); + } ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); return size; } } // anonymous namespace -ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int GetStackFrames( - void** result, int* sizes, int max_depth, int skip_count) { - return Unwind(result, sizes, max_depth, skip_count, nullptr, - nullptr); +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int +internal_stacktrace::GetStackFrames(void** result, uintptr_t* frames, + int* sizes, int max_depth, int skip_count) { + if (internal_stacktrace::ShouldFixUpStack()) { + size_t depth = static_cast(Unwind( + result, frames, sizes, max_depth, skip_count, nullptr, nullptr)); + internal_stacktrace::FixUpStack(result, frames, sizes, + static_cast(max_depth), depth); + return static_cast(depth); + } + + return Unwind(result, frames, sizes, max_depth, skip_count, + nullptr, nullptr); } ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int -GetStackFramesWithContext(void** result, int* sizes, int max_depth, - int skip_count, const void* uc, - int* min_dropped_frames) { - return Unwind(result, sizes, max_depth, skip_count, uc, +internal_stacktrace::GetStackFramesWithContext(void** result, uintptr_t* frames, + int* sizes, int max_depth, + int skip_count, const void* uc, + int* min_dropped_frames) { + if (internal_stacktrace::ShouldFixUpStack()) { + size_t depth = static_cast(Unwind( + result, frames, sizes, max_depth, skip_count, uc, min_dropped_frames)); + internal_stacktrace::FixUpStack(result, frames, sizes, + static_cast(max_depth), depth); + return static_cast(depth); + } + + return Unwind(result, frames, sizes, max_depth, skip_count, uc, min_dropped_frames); } ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int GetStackTrace( void** result, int max_depth, int skip_count) { - return Unwind(result, nullptr, max_depth, skip_count, nullptr, - nullptr); + if (internal_stacktrace::ShouldFixUpStack()) { + if constexpr (kHaveAlloca) { + const size_t nmax = static_cast(max_depth); + uintptr_t* frames = + static_cast(alloca(nmax * sizeof(*frames))); + int* sizes = static_cast(alloca(nmax * sizeof(*sizes))); + size_t depth = static_cast(Unwind( + result, frames, sizes, max_depth, skip_count, nullptr, nullptr)); + internal_stacktrace::FixUpStack(result, frames, sizes, nmax, depth); + return static_cast(depth); + } + } + + return Unwind(result, nullptr, nullptr, max_depth, skip_count, + nullptr, nullptr); } ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int GetStackTraceWithContext(void** result, int max_depth, int skip_count, const void* uc, int* min_dropped_frames) { - return Unwind(result, nullptr, max_depth, skip_count, uc, - min_dropped_frames); + if (internal_stacktrace::ShouldFixUpStack()) { + if constexpr (kHaveAlloca) { + const size_t nmax = static_cast(max_depth); + uintptr_t* frames = + static_cast(alloca(nmax * sizeof(*frames))); + int* sizes = static_cast(alloca(nmax * sizeof(*sizes))); + size_t depth = static_cast( + Unwind(result, frames, sizes, max_depth, skip_count, uc, + min_dropped_frames)); + internal_stacktrace::FixUpStack(result, frames, sizes, nmax, depth); + return static_cast(depth); + } + } + + return Unwind(result, nullptr, nullptr, max_depth, skip_count, + uc, min_dropped_frames); } void SetStackUnwinder(Unwinder w) { custom.store(w, std::memory_order_release); } -int DefaultStackUnwinder(void** pcs, int* sizes, int depth, int skip, - const void* uc, int* min_dropped_frames) { +ABSL_ATTRIBUTE_ALWAYS_INLINE static inline int DefaultStackUnwinderImpl( + void** pcs, uintptr_t* frames, int* sizes, int depth, int skip, + const void* uc, int* min_dropped_frames) { skip++; // For this function - Unwinder f = nullptr; + decltype(&UnwindImpl) f; if (sizes == nullptr) { if (uc == nullptr) { f = &UnwindImpl; @@ -132,11 +214,46 @@ int DefaultStackUnwinder(void** pcs, int* sizes, int depth, int skip, f = &UnwindImpl; } } - volatile int x = 0; - int n = (*f)(pcs, sizes, depth, skip, uc, min_dropped_frames); - x = 1; (void) x; // To disable tail call to (*f)(...) + return (*f)(pcs, frames, sizes, depth, skip, uc, min_dropped_frames); +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int +internal_stacktrace::DefaultStackUnwinder(void** pcs, uintptr_t* frames, + int* sizes, int depth, int skip, + const void* uc, + int* min_dropped_frames) { + int n = DefaultStackUnwinderImpl(pcs, frames, sizes, depth, skip, uc, + min_dropped_frames); + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); + return n; +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int DefaultStackUnwinder( + void** pcs, int* sizes, int depth, int skip, const void* uc, + int* min_dropped_frames) { + int n = DefaultStackUnwinderImpl(pcs, nullptr, sizes, depth, skip, uc, + min_dropped_frames); + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); return n; } +ABSL_ATTRIBUTE_WEAK bool internal_stacktrace::ShouldFixUpStack() { + return false; +} + +// Fixes up the stack trace of the current thread, in the first `depth` frames +// of each buffer. The buffers need to be larger than `depth`, to accommodate +// any newly inserted elements. `depth` is updated to reflect the new number of +// elements valid across all the buffers. (It is therefore recommended that all +// buffer sizes be equal.) +// +// The `frames` and `sizes` parameters denote the bounds of the stack frame +// corresponding to each instruction pointer in the `pcs`. +// Any elements inside these buffers may be zero or null, in which case that +// information is assumed to be absent/unavailable. +ABSL_ATTRIBUTE_WEAK void internal_stacktrace::FixUpStack(void**, uintptr_t*, + int*, size_t, + size_t&) {} + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/debugging/stacktrace.h b/absl/debugging/stacktrace.h index 0ec0ffd..8777172 100644 --- a/absl/debugging/stacktrace.h +++ b/absl/debugging/stacktrace.h @@ -31,11 +31,53 @@ #ifndef ABSL_DEBUGGING_STACKTRACE_H_ #define ABSL_DEBUGGING_STACKTRACE_H_ +#include +#include + +#include "absl/base/attributes.h" #include "absl/base/config.h" namespace absl { ABSL_NAMESPACE_BEGIN +namespace internal_stacktrace { + +// Same as `absl::GetStackFrames`, but with an optional `frames` parameter to +// allow callers to receive the raw stack frame addresses. +// This is internal for now; use `absl::GetStackFrames()` instead. +extern int GetStackFrames(void** result, uintptr_t* frames, int* sizes, + int max_depth, int skip_count); + +// Same as `absl::GetStackFramesWithContext`, but with an optional `frames` +// parameter to allow callers to receive a start address for each stack frame. +// The address may be zero in cases where it cannot be computed. +// +// DO NOT use this function without consulting the owners of absl/debuggging. +// There is NO GUARANTEE on the precise frame addresses returned on any given +// platform. It is only intended to provide sufficient non-overlapping bounds on +// the local variables of a stack frame when used in conjunction with the +// returned frame sizes. The actual pointers may be ABI-dependent, may vary at +// run time, and are subject to breakage without notice. +// +// Implementation note: +// Currently, we *attempt* to return the Canonical Frame Address (CFA) in DWARF +// on most platforms. This is the value of the stack pointer just before the +// 'call' instruction is executed in the caller. +// Not all platforms and toolchains support this exact address, so this should +// not be relied on for correctness. +extern int GetStackFramesWithContext(void** result, uintptr_t* frames, + int* sizes, int max_depth, int skip_count, + const void* uc, int* min_dropped_frames); + +// Same as `absl::DefaultStackUnwinder`, but with an optional `frames` parameter +// to allow callers to receive the raw stack frame addresses. +// This is internal for now; do not depend on this externally. +extern int DefaultStackUnwinder(void** pcs, uintptr_t* frames, int* sizes, + int max_depth, int skip_count, const void* uc, + int* min_dropped_frames); + +} // namespace internal_stacktrace + // GetStackFrames() // // Records program counter values for up to `max_depth` frames, skipping the @@ -78,8 +120,13 @@ ABSL_NAMESPACE_BEGIN // // This routine may return fewer stack frame entries than are // available. Also note that `result` and `sizes` must both be non-null. -extern int GetStackFrames(void** result, int* sizes, int max_depth, - int skip_count); +ABSL_ATTRIBUTE_ALWAYS_INLINE inline int GetStackFrames(void** result, + int* sizes, + int max_depth, + int skip_count) { + return internal_stacktrace::GetStackFrames(result, nullptr, sizes, max_depth, + skip_count); +} // GetStackFramesWithContext() // @@ -102,9 +149,12 @@ extern int GetStackFrames(void** result, int* sizes, int max_depth, // or other reasons. (This value will be set to `0` if no frames were dropped.) // The number of total stack frames is guaranteed to be >= skip_count + // max_depth + *min_dropped_frames. -extern int GetStackFramesWithContext(void** result, int* sizes, int max_depth, - int skip_count, const void* uc, - int* min_dropped_frames); +ABSL_ATTRIBUTE_ALWAYS_INLINE inline int GetStackFramesWithContext( + void** result, int* sizes, int max_depth, int skip_count, const void* uc, + int* min_dropped_frames) { + return internal_stacktrace::GetStackFramesWithContext( + result, nullptr, sizes, max_depth, skip_count, uc, min_dropped_frames); +} // GetStackTrace() // @@ -225,6 +275,24 @@ namespace debugging_internal { // working. extern bool StackTraceWorksForTest(); } // namespace debugging_internal + +namespace internal_stacktrace { +extern bool ShouldFixUpStack(); + +// Fixes up the stack trace of the current thread, in the first `depth` frames +// of each buffer. The buffers need to be larger than `depth`, to accommodate +// any newly inserted elements. `depth` is updated to reflect the new number of +// elements valid across all the buffers. (It is therefore recommended that all +// buffer sizes be equal.) +// +// The `frames` and `sizes` parameters denote the bounds of the stack frame +// corresponding to each instruction pointer in the `pcs`. +// Any elements inside these buffers may be zero or null, in which case that +// information is assumed to be absent/unavailable. +extern void FixUpStack(void** pcs, uintptr_t* frames, int* sizes, + size_t capacity, size_t& depth); +} // namespace internal_stacktrace + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/debugging/symbolize_elf.inc b/absl/debugging/symbolize_elf.inc index a98ca81..9836c93 100644 --- a/absl/debugging/symbolize_elf.inc +++ b/absl/debugging/symbolize_elf.inc @@ -125,12 +125,20 @@ namespace { // Some platforms use a special .opd section to store function pointers. const char kOpdSectionName[] = ".opd"; -#if (defined(__powerpc__) && !(_CALL_ELF > 1)) || defined(__ia64) +#if defined(__powerpc64__) && defined(_CALL_ELF) +#if _CALL_ELF <= 1 +#define ABSL_INTERNAL_HAVE_PPC64_ELFV1_ABI 1 +#endif +#endif +#if defined(ABSL_INTERNAL_HAVE_PPC64_ELFV1_ABI) || defined(__ia64) // Use opd section for function descriptors on these platforms, the function // address is the first word of the descriptor. -enum { kPlatformUsesOPDSections = 1 }; -#else // not PPC or IA64 -enum { kPlatformUsesOPDSections = 0 }; +// +// https://maskray.me/blog/2023-02-26-linker-notes-on-power-isa notes that +// opd sections are used on 64-bit PowerPC with the ELFv1 ABI. +inline constexpr bool kPlatformUsesOPDSections = true; +#else +inline constexpr bool kPlatformUsesOPDSections = false; #endif // This works for PowerPC & IA64 only. A function descriptor consist of two @@ -1451,11 +1459,11 @@ static bool MaybeInitializeObjFile(ObjFile *obj) { } phoff += phentsize; -#if defined(__powerpc__) && !(_CALL_ELF > 1) - // On the PowerPC ELF v1 ABI, function pointers actually point to function - // descriptors. These descriptors are stored in an .opd section, which is - // mapped read-only. We thus need to look at all readable segments, not - // just the executable ones. +#ifdef ABSL_INTERNAL_HAVE_PPC64_ELFV1_ABI + // On the PowerPC 64-bit ELFv1 ABI, function pointers actually point to + // function descriptors. These descriptors are stored in an .opd section, + // which is mapped read-only. We thus need to look at all readable + // segments, not just the executable ones. constexpr int interesting = PF_R; #else constexpr int interesting = PF_X | PF_R; @@ -1762,3 +1770,5 @@ extern "C" bool AbslInternalGetFileMappingHint(const void **start, return absl::debugging_internal::GetFileMappingHint(start, end, offset, filename); } + +#undef ABSL_INTERNAL_HAVE_PPC64_ELFV1_ABI diff --git a/absl/debugging/symbolize_emscripten.inc b/absl/debugging/symbolize_emscripten.inc index a0f344d..f6da0ac 100644 --- a/absl/debugging/symbolize_emscripten.inc +++ b/absl/debugging/symbolize_emscripten.inc @@ -58,12 +58,13 @@ bool Symbolize(const void* pc, char* out, int out_size) { return false; } - strncpy(out, func_name, out_size); + strncpy(out, func_name, static_cast(out_size)); if (out[out_size - 1] != '\0') { // strncpy() does not '\0' terminate when it truncates. static constexpr char kEllipsis[] = "..."; - int ellipsis_size = std::min(sizeof(kEllipsis) - 1, out_size - 1); + size_t ellipsis_size = + std::min(sizeof(kEllipsis) - 1, static_cast(out_size) - 1); memcpy(out + out_size - ellipsis_size - 1, kEllipsis, ellipsis_size); out[out_size - 1] = '\0'; } diff --git a/absl/debugging/symbolize_win32.inc b/absl/debugging/symbolize_win32.inc index 53a099a..589890f 100644 --- a/absl/debugging/symbolize_win32.inc +++ b/absl/debugging/symbolize_win32.inc @@ -15,7 +15,9 @@ // See "Retrieving Symbol Information by Address": // https://msdn.microsoft.com/en-us/library/windows/desktop/ms680578(v=vs.85).aspx +#include #include +#include // MSVC header dbghelp.h has a warning for an ignored typedef. #pragma warning(push) @@ -45,13 +47,30 @@ void InitializeSymbolizer(const char*) { // symbols be loaded. This is the fastest, most efficient way to use // the symbol handler. SymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_UNDNAME); - if (!SymInitialize(process, nullptr, true)) { - // GetLastError() returns a Win32 DWORD, but we assign to - // unsigned long long to simplify the ABSL_RAW_LOG case below. The uniform - // initialization guarantees this is not a narrowing conversion. - const unsigned long long error{GetLastError()}; // NOLINT(runtime/int) - ABSL_RAW_LOG(FATAL, "SymInitialize() failed: %llu", error); + DWORD syminitialize_error; + constexpr int kSymInitializeRetries = 5; + for (int i = 0; i < kSymInitializeRetries; ++i) { + if (SymInitialize(process, nullptr, true)) { + return; + } + + // SymInitialize can fail sometimes with a STATUS_INFO_LENGTH_MISMATCH + // NTSTATUS (0xC0000004), which can happen when a module load occurs during + // the SymInitialize call. In this case, we should try calling SymInitialize + // again. + syminitialize_error = GetLastError(); + // Both NTSTATUS and DWORD are 32-bit numbers, which makes the cast safe. + if (syminitialize_error != + static_cast(STATUS_INFO_LENGTH_MISMATCH)) { + break; + } } + + // GetLastError() returns a Win32 DWORD, but we assign to + // unsigned long long to simplify the ABSL_RAW_LOG case below. The uniform + // initialization guarantees this is not a narrowing conversion. + const unsigned long long error{syminitialize_error}; + ABSL_RAW_LOG(FATAL, "SymInitialize() failed: %llu", error); } bool Symbolize(const void* pc, char* out, int out_size) { diff --git a/absl/flags/commandlineflag.h b/absl/flags/commandlineflag.h index a9ffd02..9098b4c 100644 --- a/absl/flags/commandlineflag.h +++ b/absl/flags/commandlineflag.h @@ -30,7 +30,7 @@ #include #include "absl/base/config.h" -#include "absl/base/internal/fast_type_id.h" +#include "absl/base/fast_type_id.h" #include "absl/flags/internal/commandlineflag.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -80,7 +80,7 @@ class CommandLineFlag { // Return true iff flag has type T. template inline bool IsOfType() const { - return TypeId() == base_internal::FastTypeId(); + return TypeId() == FastTypeId(); } // absl::CommandLineFlag::TryGet() diff --git a/absl/flags/flag.h b/absl/flags/flag.h index 19d0ef9..e052d5f 100644 --- a/absl/flags/flag.h +++ b/absl/flags/flag.h @@ -35,6 +35,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/flags/commandlineflag.h" #include "absl/flags/config.h" @@ -94,7 +95,7 @@ using Flag = flags_internal::Flag; // // FLAGS_firstname is a Flag of type `std::string` // std::string first_name = absl::GetFlag(FLAGS_firstname); template -ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag& flag) { +[[nodiscard]] T GetFlag(const absl::Flag& flag) { return flags_internal::FlagImplPeer::InvokeGet(flag); } @@ -106,7 +107,7 @@ ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag& flag) { // thread-safe, but is potentially expensive. Avoid setting flags in general, // but especially within performance-critical code. template -void SetFlag(absl::Flag* flag, const T& v) { +void SetFlag(absl::Flag* absl_nonnull flag, const T& v) { flags_internal::FlagImplPeer::InvokeSet(*flag, v); } @@ -114,7 +115,7 @@ void SetFlag(absl::Flag* flag, const T& v) { // convertible to `T`. E.g., use this overload to pass a "const char*" when `T` // is `std::string`. template -void SetFlag(absl::Flag* flag, const V& v) { +void SetFlag(absl::Flag* absl_nonnull flag, const V& v) { T value(v); flags_internal::FlagImplPeer::InvokeSet(*flag, value); } diff --git a/absl/flags/internal/commandlineflag.h b/absl/flags/internal/commandlineflag.h index ebfe81b..daef4e3 100644 --- a/absl/flags/internal/commandlineflag.h +++ b/absl/flags/internal/commandlineflag.h @@ -17,7 +17,7 @@ #define ABSL_FLAGS_INTERNAL_COMMANDLINEFLAG_H_ #include "absl/base/config.h" -#include "absl/base/internal/fast_type_id.h" +#include "absl/base/fast_type_id.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -28,7 +28,7 @@ namespace flags_internal { // cases this id is enough to uniquely identify the flag's value type. In a few // cases we'll have to resort to using actual RTTI implementation if it is // available. -using FlagFastTypeId = absl::base_internal::FastTypeIdType; +using FlagFastTypeId = absl::FastTypeIdType; // Options that control SetCommandLineOptionWithMode. enum FlagSettingMode { diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index ccd2667..37f6ef1 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc @@ -34,6 +34,7 @@ #include "absl/base/config.h" #include "absl/base/const_init.h" #include "absl/base/dynamic_annotations.h" +#include "absl/base/fast_type_id.h" #include "absl/base/no_destructor.h" #include "absl/base/optimization.h" #include "absl/base/thread_annotations.h" @@ -59,7 +60,7 @@ namespace { // Currently we only validate flag values for user-defined flag types. bool ShouldValidateFlagValue(FlagFastTypeId flag_type_id) { #define DONT_VALIDATE(T, _) \ - if (flag_type_id == base_internal::FastTypeId()) return false; + if (flag_type_id == absl::FastTypeId()) return false; ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(DONT_VALIDATE) #undef DONT_VALIDATE diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index a6e7986..b61a247 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h @@ -57,7 +57,7 @@ template using Flag = flags_internal::Flag; template -ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag& flag); +[[nodiscard]] T GetFlag(const absl::Flag& flag); template void SetFlag(absl::Flag* flag, const T& v); @@ -783,7 +783,7 @@ class FlagImpl final : public CommandLineFlag { // heap allocation during initialization, which is both slows program startup // and can fail. Using reserved space + placement new allows us to avoid both // problems. - alignas(absl::Mutex) mutable char data_guard_[sizeof(absl::Mutex)]; + alignas(absl::Mutex) mutable unsigned char data_guard_[sizeof(absl::Mutex)]; }; #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic pop @@ -828,7 +828,7 @@ class Flag { U u; #if !defined(NDEBUG) - impl_.AssertValidType(base_internal::FastTypeId(), &GenRuntimeTypeId); + impl_.AssertValidType(absl::FastTypeId(), &GenRuntimeTypeId); #endif if (ABSL_PREDICT_FALSE(!value_.Get(impl_.seq_lock_, u.value))) { @@ -837,7 +837,7 @@ class Flag { return std::move(u.value); } void Set(const T& v) { - impl_.AssertValidType(base_internal::FastTypeId(), &GenRuntimeTypeId); + impl_.AssertValidType(absl::FastTypeId(), &GenRuntimeTypeId); impl_.Write(&v); } @@ -876,7 +876,8 @@ class FlagImplPeer { template void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) { struct AlignedSpace { - alignas(MaskedPointer::RequiredAlignment()) alignas(T) char buf[sizeof(T)]; + alignas(MaskedPointer::RequiredAlignment()) alignas( + T) unsigned char buf[sizeof(T)]; }; using Allocator = std::allocator; switch (op) { @@ -901,7 +902,7 @@ void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) { case FlagOp::kSizeof: return reinterpret_cast(static_cast(sizeof(T))); case FlagOp::kFastTypeId: - return const_cast(base_internal::FastTypeId()); + return const_cast(absl::FastTypeId()); case FlagOp::kRuntimeTypeId: return const_cast(GenRuntimeTypeId()); case FlagOp::kParse: { diff --git a/absl/flags/internal/registry.h b/absl/flags/internal/registry.h index 4b68c85..be9aacc 100644 --- a/absl/flags/internal/registry.h +++ b/absl/flags/internal/registry.h @@ -19,6 +19,7 @@ #include #include "absl/base/config.h" +#include "absl/base/fast_type_id.h" #include "absl/flags/commandlineflag.h" #include "absl/flags/internal/commandlineflag.h" #include "absl/strings/string_view.h" @@ -73,7 +74,7 @@ void FinalizeRegistry(); // // Retire flag with name "name" and type indicated by ops. -void Retire(const char* name, FlagFastTypeId type_id, char* buf); +void Retire(const char* name, FlagFastTypeId type_id, unsigned char* buf); constexpr size_t kRetiredFlagObjSize = 3 * sizeof(void*); constexpr size_t kRetiredFlagObjAlignment = alignof(void*); @@ -83,11 +84,11 @@ template class RetiredFlag { public: void Retire(const char* flag_name) { - flags_internal::Retire(flag_name, base_internal::FastTypeId(), buf_); + flags_internal::Retire(flag_name, absl::FastTypeId(), buf_); } private: - alignas(kRetiredFlagObjAlignment) char buf_[kRetiredFlagObjSize]; + alignas(kRetiredFlagObjAlignment) unsigned char buf_[kRetiredFlagObjSize]; }; } // namespace flags_internal diff --git a/absl/flags/parse.cc b/absl/flags/parse.cc index 8be2016..c87cacd 100644 --- a/absl/flags/parse.cc +++ b/absl/flags/parse.cc @@ -76,9 +76,13 @@ ABSL_CONST_INIT bool fromenv_needs_processing ABSL_CONST_INIT bool tryfromenv_needs_processing ABSL_GUARDED_BY(ProcessingChecksMutex()) = false; -ABSL_CONST_INIT absl::Mutex specified_flags_guard(absl::kConstInit); +absl::Mutex* SpecifiedFlagsMutex() { + static absl::NoDestructor mutex; + return mutex.get(); +} + ABSL_CONST_INIT std::vector* specified_flags - ABSL_GUARDED_BY(specified_flags_guard) = nullptr; + ABSL_GUARDED_BY(SpecifiedFlagsMutex()) = nullptr; // Suggesting at most kMaxHints flags in case of misspellings. ABSL_CONST_INIT const size_t kMaxHints = 100; @@ -640,7 +644,7 @@ void ReportUnrecognizedFlags( // -------------------------------------------------------------------- bool WasPresentOnCommandLine(absl::string_view flag_name) { - absl::ReaderMutexLock l(&specified_flags_guard); + absl::ReaderMutexLock l(SpecifiedFlagsMutex()); ABSL_INTERNAL_CHECK(specified_flags != nullptr, "ParseCommandLine is not invoked yet"); @@ -767,7 +771,7 @@ HelpMode ParseAbseilFlagsOnlyImpl( } positional_args.push_back(argv[0]); - absl::MutexLock l(&flags_internal::specified_flags_guard); + absl::MutexLock l(flags_internal::SpecifiedFlagsMutex()); if (specified_flags == nullptr) { specified_flags = new std::vector; } else { diff --git a/absl/flags/reflection.cc b/absl/flags/reflection.cc index ea856ff..b8b4a2e 100644 --- a/absl/flags/reflection.cc +++ b/absl/flags/reflection.cc @@ -289,11 +289,10 @@ class RetiredFlagObj final : public CommandLineFlag { } // namespace -void Retire(const char* name, FlagFastTypeId type_id, char* buf) { +void Retire(const char* name, FlagFastTypeId type_id, unsigned char* buf) { static_assert(sizeof(RetiredFlagObj) == kRetiredFlagObjSize, ""); static_assert(alignof(RetiredFlagObj) == kRetiredFlagObjAlignment, ""); - auto* flag = ::new (static_cast(buf)) - flags_internal::RetiredFlagObj(name, type_id); + auto* flag = ::new (buf) flags_internal::RetiredFlagObj(name, type_id); FlagRegistry::GlobalRegistry().RegisterFlag(*flag, nullptr); } diff --git a/absl/functional/any_invocable.h b/absl/functional/any_invocable.h index 3acb9fd..43ea9af 100644 --- a/absl/functional/any_invocable.h +++ b/absl/functional/any_invocable.h @@ -25,7 +25,7 @@ // // NOTE: `absl::AnyInvocable` is similar to the C++23 `std::move_only_function` // abstraction, but has a slightly different API and is not designed to be a -// drop-in replacement or C++11-compatible backfill of that type. +// drop-in replacement or backfill of that type. // // Credits to Matt Calabrese (https://github.com/mattcalabrese) for the original // implementation. @@ -97,11 +97,10 @@ ABSL_NAMESPACE_BEGIN // my_func(std::move(func6)); // // `AnyInvocable` also properly respects `const` qualifiers, reference -// qualifiers, and the `noexcept` specification (only in C++ 17 and beyond) as -// part of the user-specified function type (e.g. -// `AnyInvocable`). These qualifiers will be applied -// to the `AnyInvocable` object's `operator()`, and the underlying invocable -// must be compatible with those qualifiers. +// qualifiers, and the `noexcept` specification as part of the user-specified +// function type (e.g. `AnyInvocable`). These +// qualifiers will be applied to the `AnyInvocable` object's `operator()`, and +// the underlying invocable must be compatible with those qualifiers. // // Comparison of const and non-const function types: // @@ -280,11 +279,10 @@ class AnyInvocable : private internal_any_invocable::Impl { // // WARNING: An `AnyInvocable` that wraps an empty `std::function` is not // itself empty. This behavior is consistent with the standard equivalent - // `std::move_only_function`. - // - // In other words: + // `std::move_only_function`. In the following example, `a()` will actually + // invoke `f()`, leading to an `std::bad_function_call` exception: // std::function f; // empty - // absl::AnyInvocable a = std::move(f); // not empty + // absl::AnyInvocable a = f; // not empty // // Invoking an empty `AnyInvocable` results in undefined behavior. explicit operator bool() const noexcept { return this->HasValue(); } diff --git a/absl/functional/function_ref.h b/absl/functional/function_ref.h index 96cece5..f1d087a 100644 --- a/absl/functional/function_ref.h +++ b/absl/functional/function_ref.h @@ -82,17 +82,12 @@ class FunctionRef; // // replaced by an `absl::FunctionRef`: // bool Visitor(absl::FunctionRef // callback); -// -// Note: the assignment operator within an `absl::FunctionRef` is intentionally -// deleted to prevent misuse; because the `absl::FunctionRef` does not own the -// underlying type, assignment likely indicates misuse. template class FunctionRef { private: // Used to disable constructors for objects that are not compatible with the // signature of this FunctionRef. - template > + template > using EnableIfCompatible = typename std::enable_if::value || std::is_convertible::value>::type; @@ -122,9 +117,7 @@ class FunctionRef { ptr_.fun = reinterpret_cast(f); } - // To help prevent subtle lifetime bugs, FunctionRef is not assignable. - // Typically, it should only be used as an argument type. - FunctionRef& operator=(const FunctionRef& rhs) = delete; + FunctionRef& operator=(const FunctionRef& rhs) = default; FunctionRef(const FunctionRef& rhs) = default; // Call the underlying object. diff --git a/absl/functional/internal/any_invocable.h b/absl/functional/internal/any_invocable.h index c2d8cd4..167d947 100644 --- a/absl/functional/internal/any_invocable.h +++ b/absl/functional/internal/any_invocable.h @@ -65,7 +65,6 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" -#include "absl/base/internal/invoke.h" #include "absl/base/macros.h" #include "absl/base/optimization.h" #include "absl/meta/type_traits.h" @@ -74,15 +73,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN -// Helper macro used to prevent spelling `noexcept` in language versions older -// than C++17, where it is not part of the type system, in order to avoid -// compilation failures and internal compiler errors. -#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L -#define ABSL_INTERNAL_NOEXCEPT_SPEC(noex) noexcept(noex) -#else -#define ABSL_INTERNAL_NOEXCEPT_SPEC(noex) -#endif - // Defined in functional/any_invocable.h template class AnyInvocable; @@ -107,44 +97,30 @@ struct IsAnyInvocable> : std::true_type {}; // //////////////////////////////////////////////////////////////////////////////// -// A type trait that tells us whether or not a target function type should be +// A metafunction that tells us whether or not a target function type should be // stored locally in the small object optimization storage template -using IsStoredLocally = std::integral_constant< - bool, sizeof(T) <= kStorageSize && alignof(T) <= kAlignment && - kAlignment % alignof(T) == 0 && - std::is_nothrow_move_constructible::value>; +constexpr bool IsStoredLocally() { + if constexpr (sizeof(T) <= kStorageSize && alignof(T) <= kAlignment && + kAlignment % alignof(T) == 0) { + return std::is_nothrow_move_constructible::value; + } + return false; +} // An implementation of std::remove_cvref_t of C++20. template using RemoveCVRef = typename std::remove_cv::type>::type; -//////////////////////////////////////////////////////////////////////////////// -// -// An implementation of the C++ standard INVOKE pseudo-macro, operation is -// equivalent to std::invoke except that it forces an implicit conversion to the -// specified return type. If "R" is void, the function is executed and the -// return value is simply ignored. -template ::value>> -void InvokeR(F&& f, P&&... args) { - absl::base_internal::invoke(std::forward(f), std::forward

(args)...); -} - -template ::value, int> = 0> +// An implementation of std::invoke_r of C++23. +template ReturnType InvokeR(F&& f, P&&... args) { - // GCC 12 has a false-positive -Wmaybe-uninitialized warning here. -#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - return absl::base_internal::invoke(std::forward(f), - std::forward

(args)...); -#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) -#pragma GCC diagnostic pop -#endif + if constexpr (std::is_void_v) { + std::invoke(std::forward(f), std::forward

(args)...); + } else { + return std::invoke(std::forward(f), std::forward

(args)...); + } } // @@ -198,32 +174,14 @@ union TypeErasedState { } remote; // Local-storage for the type-erased object when small and trivial enough - alignas(kAlignment) char storage[kStorageSize]; + alignas(kAlignment) unsigned char storage[kStorageSize]; }; // A typed accessor for the object in `TypeErasedState` storage template T& ObjectInLocalStorage(TypeErasedState* const state) { // We launder here because the storage may be reused with the same type. -#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606L return *std::launder(reinterpret_cast(&state->storage)); -#elif ABSL_HAVE_BUILTIN(__builtin_launder) - return *__builtin_launder(reinterpret_cast(&state->storage)); -#else - - // When `std::launder` or equivalent are not available, we rely on undefined - // behavior, which works as intended on Abseil's officially supported - // platforms as of Q2 2022. -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif - return *reinterpret_cast(&state->storage); -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - -#endif } // The type for functions issuing lifetime-related operations: move and dispose @@ -231,14 +189,14 @@ T& ObjectInLocalStorage(TypeErasedState* const state) { // NOTE: When specifying `FunctionToCall::`dispose, the same state must be // passed as both "from" and "to". using ManagerType = void(FunctionToCall /*operation*/, - TypeErasedState* /*from*/, TypeErasedState* /*to*/) - ABSL_INTERNAL_NOEXCEPT_SPEC(true); + TypeErasedState* /*from*/, + TypeErasedState* /*to*/) noexcept(true); // The type for functions issuing the actual invocation of the object // A pointer to such a function is contained in each AnyInvocable instance. template -using InvokerType = ReturnType(TypeErasedState*, ForwardedParameterType

...) - ABSL_INTERNAL_NOEXCEPT_SPEC(SigIsNoexcept); +using InvokerType = ReturnType( + TypeErasedState*, ForwardedParameterType

...) noexcept(SigIsNoexcept); // The manager that is used when AnyInvocable is empty inline void EmptyManager(FunctionToCall /*operation*/, @@ -275,7 +233,7 @@ template void LocalManagerNontrivial(FunctionToCall operation, TypeErasedState* const from, TypeErasedState* const to) noexcept { - static_assert(IsStoredLocally::value, + static_assert(IsStoredLocally(), "Local storage must only be used for supported types."); static_assert(!std::is_trivially_copyable::value, "Locally stored types must be trivially copyable."); @@ -303,7 +261,7 @@ ReturnType LocalInvoker( ForwardedParameterType

... args) noexcept(SigIsNoexcept) { using RawT = RemoveCVRef; static_assert( - IsStoredLocally::value, + IsStoredLocally(), "Target object must be in local storage in order to be invoked from it."); auto& f = (ObjectInLocalStorage)(state); @@ -338,7 +296,7 @@ template void RemoteManagerNontrivial(FunctionToCall operation, TypeErasedState* const from, TypeErasedState* const to) noexcept { - static_assert(!IsStoredLocally::value, + static_assert(!IsStoredLocally(), "Remote storage must only be used for types that do not " "qualify for local storage."); @@ -360,7 +318,7 @@ ReturnType RemoteInvoker( TypeErasedState* const state, ForwardedParameterType

... args) noexcept(SigIsNoexcept) { using RawT = RemoveCVRef; - static_assert(!IsStoredLocally::value, + static_assert(!IsStoredLocally(), "Target object must be in remote storage in order to be " "invoked from it."); @@ -440,13 +398,6 @@ class CoreImpl { CoreImpl() noexcept : manager_(EmptyManager), invoker_(nullptr) {} - enum class TargetType { - kPointer, - kCompatibleAnyInvocable, - kIncompatibleAnyInvocable, - kOther, - }; - // Note: QualDecayedTRef here includes the cv-ref qualifiers associated with // the invocation of the Invocable. The unqualified type is the target object // type to be stored. @@ -454,19 +405,47 @@ class CoreImpl { explicit CoreImpl(TypedConversionConstruct, F&& f) { using DecayedT = RemoveCVRef; - constexpr TargetType kTargetType = - (std::is_pointer::value || - std::is_member_pointer::value) - ? TargetType::kPointer - : IsCompatibleAnyInvocable::value - ? TargetType::kCompatibleAnyInvocable - : IsAnyInvocable::value - ? TargetType::kIncompatibleAnyInvocable - : TargetType::kOther; - // NOTE: We only use integers instead of enums as template parameters in - // order to work around a bug on C++14 under MSVC 2017. - // See b/236131881. - Initialize(std::forward(f)); + if constexpr (std::is_pointer::value || + std::is_member_pointer::value) { + // This condition handles types that decay into pointers. This includes + // function references, which cannot be null. GCC warns against comparing + // their decayed form with nullptr (https://godbolt.org/z/9r9TMTcPK). + // We could work around this warning with constexpr programming, using + // std::is_function_v>, but we choose to ignore + // it instead of writing more code. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpragmas" +#pragma GCC diagnostic ignored "-Waddress" +#pragma GCC diagnostic ignored "-Wnonnull-compare" +#endif + if (static_cast(f) == nullptr) { +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + manager_ = EmptyManager; + invoker_ = nullptr; + } else { + InitializeStorage(std::forward(f)); + } + } else if constexpr (IsCompatibleAnyInvocable::value) { + // In this case we can "steal the guts" of the other AnyInvocable. + f.manager_(FunctionToCall::relocate_from_to, &f.state_, &state_); + manager_ = f.manager_; + invoker_ = f.invoker_; + + f.manager_ = EmptyManager; + f.invoker_ = nullptr; + } else if constexpr (IsAnyInvocable::value) { + if (f.HasValue()) { + InitializeStorage(std::forward(f)); + } else { + manager_ = EmptyManager; + invoker_ = nullptr; + } + } else { + InitializeStorage(std::forward(f)); + } } // Note: QualTRef here includes the cv-ref qualifiers associated with the @@ -517,122 +496,43 @@ class CoreImpl { invoker_ = nullptr; } - template = 0> - void Initialize(F&& f) { -// This condition handles types that decay into pointers, which includes -// function references. Since function references cannot be null, GCC warns -// against comparing their decayed form with nullptr. -// Since this is template-heavy code, we prefer to disable these warnings -// locally instead of adding yet another overload of this function. -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpragmas" -#pragma GCC diagnostic ignored "-Waddress" -#pragma GCC diagnostic ignored "-Wnonnull-compare" -#endif - if (static_cast>(f) == nullptr) { -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - manager_ = EmptyManager; - invoker_ = nullptr; - return; - } - InitializeStorage(std::forward(f)); - } - - template = 0> - void Initialize(F&& f) { - // In this case we can "steal the guts" of the other AnyInvocable. - f.manager_(FunctionToCall::relocate_from_to, &f.state_, &state_); - manager_ = f.manager_; - invoker_ = f.invoker_; - - f.manager_ = EmptyManager; - f.invoker_ = nullptr; - } - - template = 0> - void Initialize(F&& f) { - if (f.HasValue()) { - InitializeStorage(std::forward(f)); - } else { - manager_ = EmptyManager; - invoker_ = nullptr; - } - } - - template > - void Initialize(F&& f) { - InitializeStorage(std::forward(f)); - } - // Use local (inline) storage for applicable target object types. - template >::value>> + template void InitializeStorage(Args&&... args) { using RawT = RemoveCVRef; - ::new (static_cast(&state_.storage)) - RawT(std::forward(args)...); - - invoker_ = LocalInvoker; - // We can simplify our manager if we know the type is trivially copyable. - InitializeLocalManager(); - } - - // Use remote storage for target objects that cannot be stored locally. - template >::value, - int> = 0> - void InitializeStorage(Args&&... args) { - InitializeRemoteManager>(std::forward(args)...); - // This is set after everything else in case an exception is thrown in an - // earlier step of the initialization. - invoker_ = RemoteInvoker; - } - - template ::value>> - void InitializeLocalManager() { - manager_ = LocalManagerTrivial; - } - - template ::value, int> = 0> - void InitializeLocalManager() { - manager_ = LocalManagerNontrivial; - } - - template - using HasTrivialRemoteStorage = - std::integral_constant::value && - alignof(T) <= - ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT>; - - template ::value>> - void InitializeRemoteManager(Args&&... args) { - // unique_ptr is used for exception-safety in case construction throws. - std::unique_ptr uninitialized_target( - ::operator new(sizeof(T)), TrivialDeleter(sizeof(T))); - ::new (uninitialized_target.get()) T(std::forward(args)...); - state_.remote.target = uninitialized_target.release(); - state_.remote.size = sizeof(T); - manager_ = RemoteManagerTrivial; + if constexpr (IsStoredLocally()) { + ::new (static_cast(&state_.storage)) + RawT(std::forward(args)...); + invoker_ = LocalInvoker; + // We can simplify our manager if we know the type is trivially copyable. + if constexpr (std::is_trivially_copyable_v) { + manager_ = LocalManagerTrivial; + } else { + manager_ = LocalManagerNontrivial; + } + } else { + InitializeRemoteManager(std::forward(args)...); + // This is set after everything else in case an exception is thrown in an + // earlier step of the initialization. + invoker_ = RemoteInvoker; + } } - template ::value, int> = 0> + template void InitializeRemoteManager(Args&&... args) { - state_.remote.target = ::new T(std::forward(args)...); - manager_ = RemoteManagerNontrivial; + if constexpr (std::is_trivially_destructible_v && + alignof(T) <= ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT) { + // unique_ptr is used for exception-safety in case construction throws. + std::unique_ptr uninitialized_target( + ::operator new(sizeof(T)), TrivialDeleter(sizeof(T))); + ::new (uninitialized_target.get()) T(std::forward(args)...); + state_.remote.target = uninitialized_target.release(); + state_.remote.size = sizeof(T); + manager_ = RemoteManagerTrivial; + } else { + state_.remote.target = ::new T(std::forward(args)...); + manager_ = RemoteManagerNontrivial; + } } ////////////////////////////////////////////////////////////////////////////// @@ -734,17 +634,12 @@ using CanAssignReferenceWrapper = TrueAlias< absl::enable_if_t::template CallIsNoexceptIfSigIsNoexcept< std::reference_wrapper>::value>>; -//////////////////////////////////////////////////////////////////////////////// -// // The constraint for checking whether or not a call meets the noexcept -// callability requirements. This is a preprocessor macro because specifying it +// callability requirements. We use a preprocessor macro because specifying it // this way as opposed to a disjunction/branch can improve the user-side error // messages and avoids an instantiation of std::is_nothrow_invocable_r in the // cases where the user did not specify a noexcept function type. // -#define ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT(inv_quals, noex) \ - ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_##noex(inv_quals) - // The disjunction below is because we can't rely on std::is_nothrow_invocable_r // to give the right result when ReturnType is non-moveable in toolchains that // don't treat non-moveable result types correctly. For example this was the @@ -759,7 +654,7 @@ using CanAssignReferenceWrapper = TrueAlias< UnwrapStdReferenceWrapper> inv_quals, P...>, \ std::is_same< \ ReturnType, \ - absl::base_internal::invoke_result_t< \ + std::invoke_result_t< \ UnwrapStdReferenceWrapper> inv_quals, \ P...>>>>::value> @@ -775,13 +670,13 @@ using CanAssignReferenceWrapper = TrueAlias< // noex is "true" if the function type is noexcept, or false if it is not. // // The CallIsValid condition is more complicated than simply using -// absl::base_internal::is_invocable_r because we can't rely on it to give the -// right result when ReturnType is non-moveable in toolchains that don't treat -// non-moveable result types correctly. For example this was the case in libc++ -// before commit c3a24882 (2022-05). +// std::is_invocable_r because we can't rely on it to give the right result +// when ReturnType is non-moveable in toolchains that don't treat non-moveable +// result types correctly. For example this was the case in libc++ before commit +// c3a24882 (2022-05). #define ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, noex) \ template \ - class Impl \ + class Impl \ : public CoreImpl { \ public: \ /*The base class, which contains the datamembers and core operations*/ \ @@ -790,17 +685,16 @@ using CanAssignReferenceWrapper = TrueAlias< /*SFINAE constraint to check if F is invocable with the proper signature*/ \ template \ using CallIsValid = TrueAlias inv_quals, P...>, \ - std::is_same inv_quals, P...>>>::value>>; \ + std::is_invocable_r inv_quals, P...>, \ + std::is_same< \ + ReturnType, \ + std::invoke_result_t inv_quals, P...>>>::value>>; \ \ /*SFINAE constraint to check if F is nothrow-invocable when necessary*/ \ template \ using CallIsNoexceptIfSigIsNoexcept = \ - TrueAlias; \ + TrueAlias; \ \ /*Put the AnyInvocable into an empty state.*/ \ Impl() = default; \ @@ -822,8 +716,7 @@ using CanAssignReferenceWrapper = TrueAlias< \ /*Raises a fatal error when the AnyInvocable is invoked after a move*/ \ static ReturnType InvokedAfterMove( \ - TypeErasedState*, \ - ForwardedParameterType

...) noexcept(noex) { \ + TypeErasedState*, ForwardedParameterType

...) noexcept(noex) { \ ABSL_HARDENING_ASSERT(false && "AnyInvocable use-after-move"); \ std::terminate(); \ } \ @@ -851,18 +744,11 @@ using CanAssignReferenceWrapper = TrueAlias< } \ } -// Define the `noexcept(true)` specialization only for C++17 and beyond, when -// `noexcept` is part of the type system. -#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L // A convenience macro that defines specializations for the noexcept(true) and // noexcept(false) forms, given the other properties. #define ABSL_INTERNAL_ANY_INVOCABLE_IMPL(cv, ref, inv_quals) \ ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, false); \ ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, true) -#else -#define ABSL_INTERNAL_ANY_INVOCABLE_IMPL(cv, ref, inv_quals) \ - ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, false) -#endif // Non-ref-qualified partial specializations ABSL_INTERNAL_ANY_INVOCABLE_IMPL(, , &); @@ -881,8 +767,6 @@ ABSL_INTERNAL_ANY_INVOCABLE_IMPL(const, &&, const&&); #undef ABSL_INTERNAL_ANY_INVOCABLE_IMPL_ #undef ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_false #undef ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_true -#undef ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT -#undef ABSL_INTERNAL_NOEXCEPT_SPEC } // namespace internal_any_invocable ABSL_NAMESPACE_END diff --git a/absl/functional/internal/front_binder.h b/absl/functional/internal/front_binder.h index 44a5492..62f373f 100644 --- a/absl/functional/internal/front_binder.h +++ b/absl/functional/internal/front_binder.h @@ -21,7 +21,6 @@ #include #include -#include "absl/base/internal/invoke.h" #include "absl/container/internal/compressed_tuple.h" #include "absl/meta/type_traits.h" #include "absl/utility/utility.h" @@ -33,9 +32,8 @@ namespace functional_internal { // Invoke the method, expanding the tuple of bound arguments. template R Apply(Tuple&& bound, absl::index_sequence, Args&&... free) { - return base_internal::invoke( - std::forward(bound).template get()..., - std::forward(free)...); + return std::invoke(std::forward(bound).template get()..., + std::forward(free)...); } template @@ -50,23 +48,23 @@ class FrontBinder { constexpr explicit FrontBinder(absl::in_place_t, Ts&&... ts) : bound_args_(std::forward(ts)...) {} - template > + template > R operator()(FreeArgs&&... free_args) & { return functional_internal::Apply(bound_args_, Idx(), std::forward(free_args)...); } template > + class R = std::invoke_result_t> R operator()(FreeArgs&&... free_args) const& { return functional_internal::Apply(bound_args_, Idx(), std::forward(free_args)...); } - template > + template > R operator()(FreeArgs&&... free_args) && { // This overload is called when *this is an rvalue. If some of the bound // arguments are stored by value or rvalue reference, we move them. @@ -75,8 +73,8 @@ class FrontBinder { } template > + class R = std::invoke_result_t> R operator()(FreeArgs&&... free_args) const&& { // This overload is called when *this is an rvalue. If some of the bound // arguments are stored by value or rvalue reference, we move them. diff --git a/absl/functional/internal/function_ref.h b/absl/functional/internal/function_ref.h index 1cd34a3..27d45b8 100644 --- a/absl/functional/internal/function_ref.h +++ b/absl/functional/internal/function_ref.h @@ -19,7 +19,6 @@ #include #include -#include "absl/base/internal/invoke.h" #include "absl/functional/any_invocable.h" #include "absl/meta/type_traits.h" @@ -74,15 +73,13 @@ using Invoker = R (*)(VoidPtr, typename ForwardT::type...); template R InvokeObject(VoidPtr ptr, typename ForwardT::type... args) { auto o = static_cast(ptr.obj); - return static_cast( - absl::base_internal::invoke(*o, std::forward(args)...)); + return static_cast(std::invoke(*o, std::forward(args)...)); } template R InvokeFunction(VoidPtr ptr, typename ForwardT::type... args) { auto f = reinterpret_cast(ptr.fun); - return static_cast( - absl::base_internal::invoke(f, std::forward(args)...)); + return static_cast(std::invoke(f, std::forward(args)...)); } template diff --git a/absl/functional/overload.h b/absl/functional/overload.h index 7e19e70..35eec96 100644 --- a/absl/functional/overload.h +++ b/absl/functional/overload.h @@ -23,8 +23,6 @@ // Before using this function, consider whether named function overloads would // be a better design. // -// Note: absl::Overload requires C++17. -// // Example: // // std::variant v(int32_t{1}); @@ -46,9 +44,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN -#if defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ - ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L - template struct Overload final : T... { using T::operator()...; @@ -71,21 +66,6 @@ struct Overload final : T... { template Overload(T...) -> Overload; -#else - -namespace functional_internal { -template -constexpr bool kDependentFalse = false; -} - -template -auto Overload(T&&...) { - static_assert(functional_internal::kDependentFalse, - "Overload is only usable with C++17 or above."); -} - -#endif - ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/hash/hash.h b/absl/hash/hash.h index 479b17b..23f4e9d 100644 --- a/absl/hash/hash.h +++ b/absl/hash/hash.h @@ -87,6 +87,7 @@ #include "absl/base/config.h" #include "absl/functional/function_ref.h" #include "absl/hash/internal/hash.h" +#include "absl/hash/internal/weakly_mixed_integer.h" #include "absl/meta/type_traits.h" namespace absl { @@ -356,6 +357,12 @@ class HashState : public hash_internal::HashStateBase { hash_state.combine_contiguous_(hash_state.state_, first, size); return hash_state; } + + static HashState combine_weakly_mixed_integer( + HashState hash_state, hash_internal::WeaklyMixedInteger value) { + hash_state.combine_weakly_mixed_integer_(hash_state.state_, value); + return hash_state; + } using HashState::HashStateBase::combine_contiguous; private: @@ -371,6 +378,13 @@ class HashState : public hash_internal::HashStateBase { state = T::combine_contiguous(std::move(state), first, size); } + template + static void CombineWeaklyMixedIntegerImpl( + void* p, hash_internal::WeaklyMixedInteger value) { + T& state = *static_cast(p); + state = T::combine_weakly_mixed_integer(std::move(state), value); + } + static HashState combine_raw(HashState hash_state, uint64_t value) { hash_state.combine_raw_(hash_state.state_, value); return hash_state; @@ -385,6 +399,7 @@ class HashState : public hash_internal::HashStateBase { template void Init(T* state) { state_ = state; + combine_weakly_mixed_integer_ = &CombineWeaklyMixedIntegerImpl; combine_contiguous_ = &CombineContiguousImpl; combine_raw_ = &CombineRawImpl; run_combine_unordered_ = &RunCombineUnorderedImpl; @@ -424,6 +439,7 @@ class HashState : public hash_internal::HashStateBase { // Do not erase an already erased state. void Init(HashState* state) { state_ = state->state_; + combine_weakly_mixed_integer_ = state->combine_weakly_mixed_integer_; combine_contiguous_ = state->combine_contiguous_; combine_raw_ = state->combine_raw_; run_combine_unordered_ = state->run_combine_unordered_; @@ -435,6 +451,8 @@ class HashState : public hash_internal::HashStateBase { } void* state_; + void (*combine_weakly_mixed_integer_)( + void*, absl::hash_internal::WeaklyMixedInteger); void (*combine_contiguous_)(void*, const unsigned char*, size_t); void (*combine_raw_)(void*, uint64_t); HashState (*run_combine_unordered_)( diff --git a/absl/hash/internal/hash.cc b/absl/hash/internal/hash.cc index e0a8ea9..9abace5 100644 --- a/absl/hash/internal/hash.cc +++ b/absl/hash/internal/hash.cc @@ -55,13 +55,9 @@ uint64_t MixingHashState::CombineLargeContiguousImpl64( ABSL_CONST_INIT const void* const MixingHashState::kSeed = &kSeed; -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr uint64_t MixingHashState::kStaticRandomData[]; -#endif - uint64_t MixingHashState::LowLevelHashImpl(const unsigned char* data, size_t len) { - return LowLevelHashLenGt16(data, len, Seed(), &kStaticRandomData[0]); + return LowLevelHashLenGt32(data, len, Seed(), &kStaticRandomData[0]); } } // namespace hash_internal diff --git a/absl/hash/internal/hash.h b/absl/hash/internal/hash.h index f4a0d78..eb53823 100644 --- a/absl/hash/internal/hash.h +++ b/absl/hash/internal/hash.h @@ -24,15 +24,29 @@ #include #endif +// We include config.h here to make sure that ABSL_INTERNAL_CPLUSPLUS_LANG is +// defined. #include "absl/base/config.h" +// GCC15 warns that is deprecated in C++17 and suggests using +// instead, even though is not available in C++17 mode prior +// to GCC9. +#if defined(__has_include) +#if __has_include() +#define ABSL_INTERNAL_VERSION_HEADER_AVAILABLE 1 +#endif +#endif + // For feature testing and determining which headers can be included. -#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L || \ + defined(ABSL_INTERNAL_VERSION_HEADER_AVAILABLE) #include #else #include #endif +#undef ABSL_INTERNAL_VERSION_HEADER_AVAILABLE + #include #include #include @@ -51,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -65,7 +80,7 @@ #include "absl/base/port.h" #include "absl/container/fixed_array.h" #include "absl/hash/internal/city.h" -#include "absl/hash/internal/low_level_hash.h" +#include "absl/hash/internal/weakly_mixed_integer.h" #include "absl/meta/type_traits.h" #include "absl/numeric/bits.h" #include "absl/numeric/int128.h" @@ -78,14 +93,6 @@ #include // NOLINT #endif -#ifdef ABSL_HAVE_STD_STRING_VIEW -#include -#endif - -#ifdef __ARM_ACLE -#include -#endif - namespace absl { ABSL_NAMESPACE_BEGIN @@ -375,14 +382,14 @@ template (&value); uint64_t v; - if (sizeof(T) == 1) { + if constexpr (sizeof(T) == 1) { v = *start; - } else if (sizeof(T) == 2) { + } else if constexpr (sizeof(T) == 2) { v = absl::base_internal::UnalignedLoad16(start); - } else if (sizeof(T) == 4) { + } else if constexpr (sizeof(T) == 4) { v = absl::base_internal::UnalignedLoad32(start); } else { - assert(sizeof(T) == 8); + static_assert(sizeof(T) == 8); v = absl::base_internal::UnalignedLoad64(start); } return CombineRaw()(std::move(hash_state), v); @@ -394,6 +401,11 @@ H hash_bytes(H hash_state, const T& value) { return H::combine_contiguous(std::move(hash_state), start, sizeof(value)); } +template +H hash_weakly_mixed_integer(H hash_state, WeaklyMixedInteger value) { + return H::combine_weakly_mixed_integer(std::move(hash_state), value); +} + // ----------------------------------------------------------------------------- // AbslHashValue for Basic Types // ----------------------------------------------------------------------------- @@ -512,7 +524,7 @@ H AbslHashValue(H hash_state, T C::*ptr) { // padding (namely when they have 1 or 3 ints). The value below is a lower // bound on the number of salient, non-padding bytes that we use for // hashing. - if (alignof(T C::*) == alignof(int)) { + if constexpr (alignof(T C::*) == alignof(int)) { // No padding when all subobjects have the same size as the total // alignment. This happens in 32-bit mode. return n; @@ -609,7 +621,7 @@ template H AbslHashValue(H hash_state, absl::string_view str) { return H::combine( H::combine_contiguous(std::move(hash_state), str.data(), str.size()), - str.size()); + WeaklyMixedInteger{str.size()}); } // Support std::wstring, std::u16string and std::u32string. @@ -622,11 +634,9 @@ H AbslHashValue( const std::basic_string, Alloc>& str) { return H::combine( H::combine_contiguous(std::move(hash_state), str.data(), str.size()), - str.size()); + WeaklyMixedInteger{str.size()}); } -#ifdef ABSL_HAVE_STD_STRING_VIEW - // Support std::wstring_view, std::u16string_view and std::u32string_view. template ::value || @@ -635,11 +645,9 @@ template str) { return H::combine( H::combine_contiguous(std::move(hash_state), str.data(), str.size()), - str.size()); + WeaklyMixedInteger{str.size()}); } -#endif // ABSL_HAVE_STD_STRING_VIEW - #if defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && \ (!defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) || \ __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ >= 130000) && \ @@ -685,7 +693,7 @@ typename std::enable_if::value, H>::type AbslHashValue( for (const auto& t : deque) { hash_state = H::combine(std::move(hash_state), t); } - return H::combine(std::move(hash_state), deque.size()); + return H::combine(std::move(hash_state), WeaklyMixedInteger{deque.size()}); } // AbslHashValue for hashing std::forward_list @@ -697,7 +705,7 @@ typename std::enable_if::value, H>::type AbslHashValue( hash_state = H::combine(std::move(hash_state), t); ++size; } - return H::combine(std::move(hash_state), size); + return H::combine(std::move(hash_state), WeaklyMixedInteger{size}); } // AbslHashValue for hashing std::list @@ -707,7 +715,7 @@ typename std::enable_if::value, H>::type AbslHashValue( for (const auto& t : list) { hash_state = H::combine(std::move(hash_state), t); } - return H::combine(std::move(hash_state), list.size()); + return H::combine(std::move(hash_state), WeaklyMixedInteger{list.size()}); } // AbslHashValue for hashing std::vector @@ -721,7 +729,7 @@ typename std::enable_if::value && !std::is_same::value, AbslHashValue(H hash_state, const std::vector& vector) { return H::combine(H::combine_contiguous(std::move(hash_state), vector.data(), vector.size()), - vector.size()); + WeaklyMixedInteger{vector.size()}); } // AbslHashValue special cases for hashing std::vector @@ -742,7 +750,8 @@ AbslHashValue(H hash_state, const std::vector& vector) { unsigned char c = static_cast(i); hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); } - return H::combine(combiner.finalize(std::move(hash_state)), vector.size()); + return H::combine(combiner.finalize(std::move(hash_state)), + WeaklyMixedInteger{vector.size()}); } #else // When not working around the libstdc++ bug above, we still have to contend @@ -758,7 +767,7 @@ typename std::enable_if::value && std::is_same::value, AbslHashValue(H hash_state, const std::vector& vector) { return H::combine(std::move(hash_state), std::hash>{}(vector), - vector.size()); + WeaklyMixedInteger{vector.size()}); } #endif @@ -775,7 +784,7 @@ AbslHashValue(H hash_state, const std::map& map) { for (const auto& t : map) { hash_state = H::combine(std::move(hash_state), t); } - return H::combine(std::move(hash_state), map.size()); + return H::combine(std::move(hash_state), WeaklyMixedInteger{map.size()}); } // AbslHashValue for hashing std::multimap @@ -788,7 +797,7 @@ AbslHashValue(H hash_state, for (const auto& t : map) { hash_state = H::combine(std::move(hash_state), t); } - return H::combine(std::move(hash_state), map.size()); + return H::combine(std::move(hash_state), WeaklyMixedInteger{map.size()}); } // AbslHashValue for hashing std::set @@ -798,7 +807,7 @@ typename std::enable_if::value, H>::type AbslHashValue( for (const auto& t : set) { hash_state = H::combine(std::move(hash_state), t); } - return H::combine(std::move(hash_state), set.size()); + return H::combine(std::move(hash_state), WeaklyMixedInteger{set.size()}); } // AbslHashValue for hashing std::multiset @@ -808,7 +817,7 @@ typename std::enable_if::value, H>::type AbslHashValue( for (const auto& t : set) { hash_state = H::combine(std::move(hash_state), t); } - return H::combine(std::move(hash_state), set.size()); + return H::combine(std::move(hash_state), WeaklyMixedInteger{set.size()}); } // ----------------------------------------------------------------------------- @@ -822,7 +831,7 @@ typename std::enable_if::value, H>::type AbslHashValue( H hash_state, const std::unordered_set& s) { return H::combine( H::combine_unordered(std::move(hash_state), s.begin(), s.end()), - s.size()); + WeaklyMixedInteger{s.size()}); } // AbslHashValue for hashing std::unordered_multiset @@ -833,7 +842,7 @@ typename std::enable_if::value, H>::type AbslHashValue( const std::unordered_multiset& s) { return H::combine( H::combine_unordered(std::move(hash_state), s.begin(), s.end()), - s.size()); + WeaklyMixedInteger{s.size()}); } // AbslHashValue for hashing std::unordered_set @@ -845,7 +854,7 @@ AbslHashValue(H hash_state, const std::unordered_map& s) { return H::combine( H::combine_unordered(std::move(hash_state), s.begin(), s.end()), - s.size()); + WeaklyMixedInteger{s.size()}); } // AbslHashValue for hashing std::unordered_multiset @@ -857,7 +866,7 @@ AbslHashValue(H hash_state, const std::unordered_multimap& s) { return H::combine( H::combine_unordered(std::move(hash_state), s.begin(), s.end()), - s.size()); + WeaklyMixedInteger{s.size()}); } // ----------------------------------------------------------------------------- @@ -968,11 +977,20 @@ hash_range_or_bytes(H hash_state, const T* data, size_t size) { // `false`. struct HashSelect { private: + struct WeaklyMixedIntegerProbe { + template + static H Invoke(H state, WeaklyMixedInteger value) { + return hash_internal::hash_weakly_mixed_integer(std::move(state), value); + } + }; + struct State : HashStateBase { static State combine_contiguous(State hash_state, const unsigned char*, size_t); using State::HashStateBase::combine_contiguous; static State combine_raw(State state, uint64_t value); + static State combine_weakly_mixed_integer(State hash_state, + WeaklyMixedInteger value); }; struct UniquelyRepresentedProbe { @@ -1034,6 +1052,7 @@ struct HashSelect { // disjunction provides short circuiting wrt instantiation. template using Apply = absl::disjunction< // + Probe, // Probe, // Probe, // Probe, // @@ -1063,8 +1082,7 @@ class ABSL_DLL MixingHashState : public HashStateBase { }; static constexpr uint64_t kMul = - sizeof(size_t) == 4 ? uint64_t{0xcc9e2d51} - : uint64_t{0xdcb22ca68cb134ed}; + uint64_t{0xdcb22ca68cb134ed}; template using IntegralFastPath = @@ -1099,7 +1117,7 @@ class ABSL_DLL MixingHashState : public HashStateBase { template ::value, int> = 0> static size_t hash(T value) { return static_cast( - WeakMix(Seed() ^ static_cast>(value))); + WeakMix(Seed(), static_cast>(value))); } // Overload of MixingHashState::hash() @@ -1114,6 +1132,18 @@ class ABSL_DLL MixingHashState : public HashStateBase { MixingHashState() : state_(Seed()) {} friend class MixingHashState::HashStateBase; + template + friend H absl::hash_internal::hash_weakly_mixed_integer(H, + WeaklyMixedInteger); + + static MixingHashState combine_weakly_mixed_integer( + MixingHashState hash_state, WeaklyMixedInteger value) { + // Some transformation for the value is needed to make an empty + // string/container change the mixing hash state. + // We use constant smaller than 8 bits to make compiler use + // `add` with an immediate operand with 1 byte value. + return MixingHashState{hash_state.state_ + (0x57 + value.value)}; + } template static MixingHashState RunCombineUnordered(MixingHashState state, @@ -1152,7 +1182,7 @@ class ABSL_DLL MixingHashState : public HashStateBase { // optimize Read1To3 and Read4To8 differently for the string case. static MixingHashState combine_raw(MixingHashState hash_state, uint64_t value) { - return MixingHashState(WeakMix(hash_state.state_ ^ value)); + return MixingHashState(WeakMix(hash_state.state_, value)); } // Implementation of the base case for combine_contiguous where we actually @@ -1180,7 +1210,7 @@ class ABSL_DLL MixingHashState : public HashStateBase { // Empty ranges have no effect. return state; } - return WeakMix(state ^ v); + return WeakMix(state, v); } ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t CombineContiguousImpl9to16( @@ -1222,8 +1252,8 @@ class ABSL_DLL MixingHashState : public HashStateBase { size_t len); // Reads 9 to 16 bytes from p. - // The least significant 8 bytes are in .first, the rest (zero padded) bytes - // are in .second. + // The least significant 8 bytes are in .first, and the rest of the bytes are + // in .second along with duplicated bytes from .first if len<16. static std::pair Read9To16(const unsigned char* p, size_t len) { uint64_t low_mem = Read8(p); @@ -1251,11 +1281,7 @@ class ABSL_DLL MixingHashState : public HashStateBase { #endif } - // Reads 4 to 8 bytes from p. Zero pads to fill uint64_t. - // TODO(b/384509507): consider optimizing this by not requiring the output to - // be equivalent to an integer load for 4/8 bytes. Currently, we rely on this - // behavior for the HashConsistentAcrossIntTypes test case. Ditto for - // Read1To3. + // Reads 4 to 8 bytes from p. Some input bytes may be duplicated in output. static uint64_t Read4To8(const unsigned char* p, size_t len) { // If `len < 8`, we duplicate bytes in the middle. // E.g.: @@ -1274,7 +1300,7 @@ class ABSL_DLL MixingHashState : public HashStateBase { return most_significant | least_significant; } - // Reads 1 to 3 bytes from p. Zero pads to fill uint32_t. + // Reads 1 to 3 bytes from p. Some input bytes may be duplicated in output. static uint32_t Read1To3(const unsigned char* p, size_t len) { // The trick used by this implementation is to avoid branches. // We always read three bytes by duplicating. @@ -1290,27 +1316,26 @@ class ABSL_DLL MixingHashState : public HashStateBase { } ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Mix(uint64_t lhs, uint64_t rhs) { + // For 32 bit platforms we are trying to use all 64 lower bits. + if constexpr (sizeof(size_t) < 8) { + uint64_t m = lhs * rhs; + return m ^ (m >> 32); + } // Though the 128-bit product on AArch64 needs two instructions, it is // still a good balance between speed and hash quality. - using MultType = - absl::conditional_t; - MultType m = lhs; + uint128 m = lhs; m *= rhs; - return static_cast(m ^ (m >> (sizeof(m) * 8 / 2))); + return Uint128High64(m) ^ Uint128Low64(m); } // Slightly lower latency than Mix, but with lower quality. The byte swap // helps ensure that low bits still have high quality. - ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t WeakMix(uint64_t n) { + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t WeakMix(uint64_t lhs, + uint64_t rhs) { + const uint64_t n = lhs ^ rhs; // WeakMix doesn't work well on 32-bit platforms so just use Mix. - if (sizeof(size_t) < 8) return Mix(n, kMul); -#ifdef __ARM_ACLE - // gbswap_64 compiles to `rev` on ARM, but `rbit` is better because it - // reverses bits rather than reversing bytes. - return __rbitll(n * kMul); -#else + if constexpr (sizeof(size_t) < 8) return Mix(n, kMul); return absl::gbswap_64(n * kMul); -#endif } // An extern to avoid bloat on a direct call to LowLevelHash() with fixed diff --git a/absl/hash/internal/low_level_hash.cc b/absl/hash/internal/low_level_hash.cc index ec02d7e..1a107ec 100644 --- a/absl/hash/internal/low_level_hash.cc +++ b/absl/hash/internal/low_level_hash.cc @@ -14,29 +14,44 @@ #include "absl/hash/internal/low_level_hash.h" +#include #include #include +#include "absl/base/config.h" #include "absl/base/internal/unaligned_access.h" +#include "absl/base/optimization.h" #include "absl/base/prefetch.h" #include "absl/numeric/int128.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace hash_internal { - -static uint64_t Mix(uint64_t v0, uint64_t v1) { +namespace { +uint64_t Mix(uint64_t v0, uint64_t v1) { absl::uint128 p = v0; p *= v1; return absl::Uint128Low64(p) ^ absl::Uint128High64(p); } +uint64_t Mix32Bytes(const uint8_t* ptr, uint64_t current_state, + const uint64_t salt[5]) { + uint64_t a = absl::base_internal::UnalignedLoad64(ptr); + uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8); + uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16); + uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24); + + uint64_t cs0 = Mix(a ^ salt[1], b ^ current_state); + uint64_t cs1 = Mix(c ^ salt[2], d ^ current_state); + return cs0 ^ cs1; +} +} // namespace -uint64_t LowLevelHashLenGt16(const void* data, size_t len, uint64_t seed, +uint64_t LowLevelHashLenGt32(const void* data, size_t len, uint64_t seed, const uint64_t salt[5]) { + assert(len > 32); const uint8_t* ptr = static_cast(data); - uint64_t starting_length = static_cast(len); - const uint8_t* last_16_ptr = ptr + starting_length - 16; - uint64_t current_state = seed ^ salt[0]; + uint64_t current_state = seed ^ salt[0] ^ len; + const uint8_t* last_32_ptr = ptr + len - 32; if (len > 64) { // If we have more than 64 bytes, we're going to handle chunks of 64 @@ -76,71 +91,13 @@ uint64_t LowLevelHashLenGt16(const void* data, size_t len, uint64_t seed, // We now have a data `ptr` with at most 64 bytes and the current state // of the hashing state machine stored in current_state. if (len > 32) { - uint64_t a = absl::base_internal::UnalignedLoad64(ptr); - uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8); - uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16); - uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24); - - uint64_t cs0 = Mix(a ^ salt[1], b ^ current_state); - uint64_t cs1 = Mix(c ^ salt[2], d ^ current_state); - current_state = cs0 ^ cs1; - - ptr += 32; - len -= 32; + current_state = Mix32Bytes(ptr, current_state, salt); } // We now have a data `ptr` with at most 32 bytes and the current state - // of the hashing state machine stored in current_state. - if (len > 16) { - uint64_t a = absl::base_internal::UnalignedLoad64(ptr); - uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8); - - current_state = Mix(a ^ salt[1], b ^ current_state); - } - - // We now have a data `ptr` with at least 1 and at most 16 bytes. But we can - // safely read from `ptr + len - 16`. - uint64_t a = absl::base_internal::UnalignedLoad64(last_16_ptr); - uint64_t b = absl::base_internal::UnalignedLoad64(last_16_ptr + 8); - - return Mix(a ^ salt[1] ^ starting_length, b ^ current_state); -} - -uint64_t LowLevelHash(const void* data, size_t len, uint64_t seed, - const uint64_t salt[5]) { - if (len > 16) return LowLevelHashLenGt16(data, len, seed, salt); - - // Prefetch the cacheline that data resides in. - PrefetchToLocalCache(data); - const uint8_t* ptr = static_cast(data); - uint64_t starting_length = static_cast(len); - uint64_t current_state = seed ^ salt[0]; - if (len == 0) return current_state; - - uint64_t a = 0; - uint64_t b = 0; - - // We now have a data `ptr` with at least 1 and at most 16 bytes. - if (len > 8) { - // When we have at least 9 and at most 16 bytes, set A to the first 64 - // bits of the input and B to the last 64 bits of the input. Yes, they - // will overlap in the middle if we are working with less than the full 16 - // bytes. - a = absl::base_internal::UnalignedLoad64(ptr); - b = absl::base_internal::UnalignedLoad64(ptr + len - 8); - } else if (len > 3) { - // If we have at least 4 and at most 8 bytes, set A to the first 32 - // bits and B to the last 32 bits. - a = absl::base_internal::UnalignedLoad32(ptr); - b = absl::base_internal::UnalignedLoad32(ptr + len - 4); - } else { - // If we have at least 1 and at most 3 bytes, read 2 bytes into A and the - // other byte into B, with some adjustments. - a = static_cast((ptr[0] << 8) | ptr[len - 1]); - b = static_cast(ptr[len >> 1]); - } - - return Mix(a ^ salt[1] ^ starting_length, b ^ current_state); + // of the hashing state machine stored in current_state. But we can + // safely read from `ptr + len - 32`. + return Mix32Bytes(last_32_ptr, current_state, salt); } } // namespace hash_internal diff --git a/absl/hash/internal/low_level_hash.h b/absl/hash/internal/low_level_hash.h index d460e35..49e9ec4 100644 --- a/absl/hash/internal/low_level_hash.h +++ b/absl/hash/internal/low_level_hash.h @@ -35,16 +35,12 @@ ABSL_NAMESPACE_BEGIN namespace hash_internal { // Hash function for a byte array. A 64-bit seed and a set of five 64-bit -// integers are hashed into the result. +// integers are hashed into the result. The length must be greater than 32. // // To allow all hashable types (including string_view and Span) to depend on // this algorithm, we keep the API low-level, with as few dependencies as // possible. -uint64_t LowLevelHash(const void* data, size_t len, uint64_t seed, - const uint64_t salt[5]); - -// Same as above except the length must be greater than 16. -uint64_t LowLevelHashLenGt16(const void* data, size_t len, uint64_t seed, +uint64_t LowLevelHashLenGt32(const void* data, size_t len, uint64_t seed, const uint64_t salt[5]); } // namespace hash_internal diff --git a/absl/hash/internal/spy_hash_state.h b/absl/hash/internal/spy_hash_state.h index 92490b1..e403113 100644 --- a/absl/hash/internal/spy_hash_state.h +++ b/absl/hash/internal/spy_hash_state.h @@ -16,12 +16,14 @@ #define ABSL_HASH_INTERNAL_SPY_HASH_STATE_H_ #include +#include #include #include #include #include #include "absl/hash/hash.h" +#include "absl/hash/internal/weakly_mixed_integer.h" #include "absl/strings/match.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" @@ -167,6 +169,11 @@ class SpyHashStateImpl : public HashStateBase> { return hash_state; } + static SpyHashStateImpl combine_weakly_mixed_integer( + SpyHashStateImpl hash_state, WeaklyMixedInteger value) { + return combine(std::move(hash_state), value.value); + } + using SpyHashStateImpl::HashStateBase::combine_contiguous; template diff --git a/absl/hash/internal/weakly_mixed_integer.h b/absl/hash/internal/weakly_mixed_integer.h new file mode 100644 index 0000000..5575436 --- /dev/null +++ b/absl/hash/internal/weakly_mixed_integer.h @@ -0,0 +1,38 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_HASH_INTERNAL_WEAKLY_MIXED_INTEGER_H_ +#define ABSL_HASH_INTERNAL_WEAKLY_MIXED_INTEGER_H_ + +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { + +// Contains an integer that will be mixed into a hash state more weakly than +// regular integers. It is useful for cases in which an integer is a part of a +// larger object and needs to be mixed as a supplement. E.g., absl::string_view +// and absl::Span are mixing their size wrapped with WeaklyMixedInteger. +struct WeaklyMixedInteger { + size_t value; +}; + +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HASH_INTERNAL_WEAKLY_MIXED_INTEGER_H_ diff --git a/absl/log/check.h b/absl/log/check.h index 50f633d..9e2219b 100644 --- a/absl/log/check.h +++ b/absl/log/check.h @@ -42,7 +42,8 @@ // CHECK() // -// `CHECK` terminates the program with a fatal error if `condition` is not true. +// `CHECK` enforces that the `condition` is true. If the condition is false, +// the program is terminated with a fatal error. // // The message may include additional information such as stack traces, when // available. diff --git a/absl/log/die_if_null.h b/absl/log/die_if_null.h index f773aa8..8597976 100644 --- a/absl/log/die_if_null.h +++ b/absl/log/die_if_null.h @@ -60,8 +60,8 @@ namespace log_internal { // Helper for `ABSL_DIE_IF_NULL`. template -ABSL_MUST_USE_RESULT T DieIfNull(const char* file, int line, - const char* exprtext, T&& t) { +[[nodiscard]] T DieIfNull(const char* file, int line, const char* exprtext, + T&& t) { if (ABSL_PREDICT_FALSE(t == nullptr)) { // Call a non-inline helper function for a small code size improvement. DieBecauseNull(file, line, exprtext); diff --git a/absl/log/globals.h b/absl/log/globals.h index 4feec40..9718967 100644 --- a/absl/log/globals.h +++ b/absl/log/globals.h @@ -43,7 +43,7 @@ ABSL_NAMESPACE_BEGIN // // Returns the value of the Minimum Log Level parameter. // This function is async-signal-safe. -ABSL_MUST_USE_RESULT absl::LogSeverityAtLeast MinLogLevel(); +[[nodiscard]] absl::LogSeverityAtLeast MinLogLevel(); // SetMinLogLevel() // @@ -82,7 +82,7 @@ class ScopedMinLogLevel final { // // Returns the value of the Stderr Threshold parameter. // This function is async-signal-safe. -ABSL_MUST_USE_RESULT absl::LogSeverityAtLeast StderrThreshold(); +[[nodiscard]] absl::LogSeverityAtLeast StderrThreshold(); // SetStderrThreshold() // @@ -118,8 +118,7 @@ class ScopedStderrThreshold final { // // Returns true if we should log a backtrace at the specified location. namespace log_internal { -ABSL_MUST_USE_RESULT bool ShouldLogBacktraceAt(absl::string_view file, - int line); +[[nodiscard]] bool ShouldLogBacktraceAt(absl::string_view file, int line); } // namespace log_internal // SetLogBacktraceLocation() @@ -145,7 +144,7 @@ void ClearLogBacktraceLocation(); // // Returns the value of the Prepend Log Prefix option. // This function is async-signal-safe. -ABSL_MUST_USE_RESULT bool ShouldPrependLogPrefix(); +[[nodiscard]] bool ShouldPrependLogPrefix(); // EnableLogPrefix() // diff --git a/absl/log/internal/append_truncated.h b/absl/log/internal/append_truncated.h index f0e7912..d420a8b 100644 --- a/absl/log/internal/append_truncated.h +++ b/absl/log/internal/append_truncated.h @@ -17,8 +17,10 @@ #include #include +#include #include "absl/base/config.h" +#include "absl/strings/internal/utf8.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" @@ -33,6 +35,32 @@ inline size_t AppendTruncated(absl::string_view src, absl::Span &dst) { dst.remove_prefix(src.size()); return src.size(); } +// Likewise, but it also takes a wide character string and transforms it into a +// UTF-8 encoded byte string regardless of the current locale. +// - On platforms where `wchar_t` is 2 bytes (e.g., Windows), the input is +// treated as UTF-16. +// - On platforms where `wchar_t` is 4 bytes (e.g., Linux, macOS), the input +// is treated as UTF-32. +inline size_t AppendTruncated(std::wstring_view src, absl::Span &dst) { + absl::strings_internal::ShiftState state; + size_t total_bytes_written = 0; + for (const wchar_t wc : src) { + // If the destination buffer might not be large enough to write the next + // character, stop. + if (dst.size() < absl::strings_internal::kMaxEncodedUTF8Size) break; + size_t bytes_written = + absl::strings_internal::WideToUtf8(wc, dst.data(), state); + if (bytes_written == static_cast(-1)) { + // Invalid character. Encode REPLACEMENT CHARACTER (U+FFFD) instead. + constexpr wchar_t kReplacementCharacter = L'\uFFFD'; + bytes_written = absl::strings_internal::WideToUtf8(kReplacementCharacter, + dst.data(), state); + } + dst.remove_prefix(bytes_written); + total_bytes_written += bytes_written; + } + return total_bytes_written; +} // Likewise, but `n` copies of `c`. inline size_t AppendTruncated(char c, size_t n, absl::Span &dst) { if (n > dst.size()) n = dst.size(); diff --git a/absl/log/internal/check_op.cc b/absl/log/internal/check_op.cc index cec9421..23db63b 100644 --- a/absl/log/internal/check_op.cc +++ b/absl/log/internal/check_op.cc @@ -35,26 +35,26 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace log_internal { -#define ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(x) \ - template absl::Nonnull MakeCheckOpString( \ - x, x, absl::Nonnull) -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(bool); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(int64_t); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(uint64_t); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(float); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(double); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(char); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(unsigned char); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const std::string&); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const absl::string_view&); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const char*); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const signed char*); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const unsigned char*); -ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const void*); -#undef ABSL_LOGGING_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING +#define ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(x) \ + template const char* absl_nonnull MakeCheckOpString( \ + x, x, const char* absl_nonnull) +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(bool); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(int64_t); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(uint64_t); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(float); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(double); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(char); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(unsigned char); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const std::string&); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const absl::string_view&); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const char*); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const signed char*); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const unsigned char*); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING(const void*); +#undef ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING CheckOpMessageBuilder::CheckOpMessageBuilder( - absl::Nonnull exprtext) { + const char* absl_nonnull exprtext) { stream_ << exprtext << " ("; } @@ -63,7 +63,7 @@ std::ostream& CheckOpMessageBuilder::ForVar2() { return stream_; } -absl::Nonnull CheckOpMessageBuilder::NewString() { +const char* absl_nonnull CheckOpMessageBuilder::NewString() { stream_ << ")"; // There's no need to free this string since the process is crashing. return absl::IgnoreLeak(new std::string(std::move(stream_).str()))->c_str(); @@ -103,9 +103,9 @@ void MakeCheckOpValueString(std::ostream& os, const void* p) { // Helper functions for string comparisons. #define DEFINE_CHECK_STROP_IMPL(name, func, expected) \ - absl::Nullable Check##func##expected##Impl( \ - absl::Nullable s1, absl::Nullable s2, \ - absl::Nonnull exprtext) { \ + const char* absl_nullable Check##func##expected##Impl( \ + const char* absl_nullable s1, const char* absl_nullable s2, \ + const char* absl_nonnull exprtext) { \ bool equal = s1 == s2 || (s1 && s2 && !func(s1, s2)); \ if (equal == expected) { \ return nullptr; \ diff --git a/absl/log/internal/check_op.h b/absl/log/internal/check_op.h index d56aa31..7253402 100644 --- a/absl/log/internal/check_op.h +++ b/absl/log/internal/check_op.h @@ -64,49 +64,48 @@ #endif #define ABSL_LOG_INTERNAL_CHECK_OP(name, op, val1, val1_text, val2, val2_text) \ - while (absl::Nullable absl_log_internal_check_op_result \ - ABSL_LOG_INTERNAL_ATTRIBUTE_UNUSED_IF_STRIP_LOG = \ - ::absl::log_internal::name##Impl( \ - ::absl::log_internal::GetReferenceableValue(val1), \ - ::absl::log_internal::GetReferenceableValue(val2), \ - ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL( \ - val1_text " " #op " " val2_text))) \ + while (const char* absl_nullable absl_log_internal_check_op_result \ + [[maybe_unused]] = ::absl::log_internal::name##Impl( \ + ::absl::log_internal::GetReferenceableValue(val1), \ + ::absl::log_internal::GetReferenceableValue(val2), \ + ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(val1_text " " #op \ + " " val2_text))) \ ABSL_LOG_INTERNAL_CONDITION_FATAL(STATELESS, true) \ - ABSL_LOG_INTERNAL_CHECK(absl::implicit_cast>( \ + ABSL_LOG_INTERNAL_CHECK(::absl::implicit_cast( \ absl_log_internal_check_op_result)) \ .InternalStream() #define ABSL_LOG_INTERNAL_QCHECK_OP(name, op, val1, val1_text, val2, \ val2_text) \ - while (absl::Nullable absl_log_internal_qcheck_op_result = \ + while (const char* absl_nullable absl_log_internal_qcheck_op_result = \ ::absl::log_internal::name##Impl( \ ::absl::log_internal::GetReferenceableValue(val1), \ ::absl::log_internal::GetReferenceableValue(val2), \ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL( \ val1_text " " #op " " val2_text))) \ ABSL_LOG_INTERNAL_CONDITION_QFATAL(STATELESS, true) \ - ABSL_LOG_INTERNAL_QCHECK(absl::implicit_cast>( \ + ABSL_LOG_INTERNAL_QCHECK(::absl::implicit_cast( \ absl_log_internal_qcheck_op_result)) \ .InternalStream() #define ABSL_LOG_INTERNAL_CHECK_STROP(func, op, expected, s1, s1_text, s2, \ s2_text) \ - while (absl::Nullable absl_log_internal_check_strop_result = \ + while (const char* absl_nullable absl_log_internal_check_strop_result = \ ::absl::log_internal::Check##func##expected##Impl( \ (s1), (s2), \ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(s1_text " " #op \ " " s2_text))) \ ABSL_LOG_INTERNAL_CONDITION_FATAL(STATELESS, true) \ - ABSL_LOG_INTERNAL_CHECK(absl::implicit_cast>( \ + ABSL_LOG_INTERNAL_CHECK(::absl::implicit_cast( \ absl_log_internal_check_strop_result)) \ .InternalStream() #define ABSL_LOG_INTERNAL_QCHECK_STROP(func, op, expected, s1, s1_text, s2, \ s2_text) \ - while (absl::Nullable absl_log_internal_qcheck_strop_result = \ + while (const char* absl_nullable absl_log_internal_qcheck_strop_result = \ ::absl::log_internal::Check##func##expected##Impl( \ (s1), (s2), \ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(s1_text " " #op \ " " s2_text))) \ ABSL_LOG_INTERNAL_CONDITION_QFATAL(STATELESS, true) \ - ABSL_LOG_INTERNAL_QCHECK(absl::implicit_cast>( \ + ABSL_LOG_INTERNAL_QCHECK(::absl::implicit_cast( \ absl_log_internal_qcheck_strop_result)) \ .InternalStream() @@ -135,8 +134,8 @@ // strip the call to stringify the non-ok `Status` as long as we don't log it; // dropping the `Status`'s message text is out of scope. #define ABSL_LOG_INTERNAL_CHECK_OK(val, val_text) \ - for (::std::pair, \ - absl::Nullable> \ + for (::std::pair \ absl_log_internal_check_ok_goo; \ absl_log_internal_check_ok_goo.first = \ ::absl::log_internal::AsStatus(val), \ @@ -149,12 +148,12 @@ " is OK")), \ !ABSL_PREDICT_TRUE(absl_log_internal_check_ok_goo.first->ok());) \ ABSL_LOG_INTERNAL_CONDITION_FATAL(STATELESS, true) \ - ABSL_LOG_INTERNAL_CHECK(absl::implicit_cast>( \ + ABSL_LOG_INTERNAL_CHECK(::absl::implicit_cast( \ absl_log_internal_check_ok_goo.second)) \ .InternalStream() #define ABSL_LOG_INTERNAL_QCHECK_OK(val, val_text) \ - for (::std::pair, \ - absl::Nullable> \ + for (::std::pair \ absl_log_internal_qcheck_ok_goo; \ absl_log_internal_qcheck_ok_goo.first = \ ::absl::log_internal::AsStatus(val), \ @@ -167,7 +166,7 @@ " is OK")), \ !ABSL_PREDICT_TRUE(absl_log_internal_qcheck_ok_goo.first->ok());) \ ABSL_LOG_INTERNAL_CONDITION_QFATAL(STATELESS, true) \ - ABSL_LOG_INTERNAL_QCHECK(absl::implicit_cast>( \ + ABSL_LOG_INTERNAL_QCHECK(::absl::implicit_cast( \ absl_log_internal_qcheck_ok_goo.second)) \ .InternalStream() @@ -179,9 +178,8 @@ template class StatusOr; namespace status_internal { -ABSL_ATTRIBUTE_PURE_FUNCTION absl::Nonnull MakeCheckFailString( - absl::Nonnull status, - absl::Nonnull prefix); +ABSL_ATTRIBUTE_PURE_FUNCTION const char* absl_nonnull MakeCheckFailString( + const absl::Status* absl_nonnull status, const char* absl_nonnull prefix); } // namespace status_internal namespace log_internal { @@ -189,11 +187,11 @@ namespace log_internal { // Convert a Status or a StatusOr to its underlying status value. // // (This implementation does not require a dep on absl::Status to work.) -inline absl::Nonnull AsStatus(const absl::Status& s) { +inline const absl::Status* absl_nonnull AsStatus(const absl::Status& s) { return &s; } template -absl::Nonnull AsStatus(const absl::StatusOr& s) { +const absl::Status* absl_nonnull AsStatus(const absl::StatusOr& s) { return &s.status(); } @@ -202,14 +200,14 @@ absl::Nonnull AsStatus(const absl::StatusOr& s) { class CheckOpMessageBuilder final { public: // Inserts `exprtext` and ` (` to the stream. - explicit CheckOpMessageBuilder(absl::Nonnull exprtext); + explicit CheckOpMessageBuilder(const char* absl_nonnull exprtext); ~CheckOpMessageBuilder() = default; // For inserting the first variable. std::ostream& ForVar1() { return stream_; } // For inserting the second variable (adds an intermediate ` vs. `). std::ostream& ForVar2(); // Get the result (inserts the closing `)`). - absl::Nonnull NewString(); + const char* absl_nonnull NewString(); private: std::ostringstream stream_; @@ -226,7 +224,7 @@ inline void MakeCheckOpValueString(std::ostream& os, const T& v) { void MakeCheckOpValueString(std::ostream& os, char v); void MakeCheckOpValueString(std::ostream& os, signed char v); void MakeCheckOpValueString(std::ostream& os, unsigned char v); -void MakeCheckOpValueString(std::ostream& os, const void* p); +void MakeCheckOpValueString(std::ostream& os, const void* absl_nullable p); namespace detect_specialization { @@ -268,8 +266,9 @@ float operator<<(std::ostream&, float value); double operator<<(std::ostream&, double value); long double operator<<(std::ostream&, long double value); bool operator<<(std::ostream&, bool value); -const void* operator<<(std::ostream&, const void* value); -const void* operator<<(std::ostream&, std::nullptr_t); +const void* absl_nullable operator<<(std::ostream&, + const void* absl_nullable value); +const void* absl_nullable operator<<(std::ostream&, std::nullptr_t); // These `char` overloads are specified like this in the standard, so we have to // write them exactly the same to ensure the call is ambiguous. @@ -283,13 +282,14 @@ signed char operator<<(std::basic_ostream&, signed char); template unsigned char operator<<(std::basic_ostream&, unsigned char); template -const char* operator<<(std::basic_ostream&, const char*); +const char* absl_nonnull operator<<(std::basic_ostream&, + const char* absl_nonnull); template -const signed char* operator<<(std::basic_ostream&, - const signed char*); +const signed char* absl_nonnull operator<<(std::basic_ostream&, + const signed char* absl_nonnull); template -const unsigned char* operator<<(std::basic_ostream&, - const unsigned char*); +const unsigned char* absl_nonnull operator<<(std::basic_ostream&, + const unsigned char* absl_nonnull); // This overload triggers when the call is not ambiguous. // It means that T is being printed with some overload not on this list. @@ -314,7 +314,8 @@ class StringifySink { void Append(absl::string_view text); void Append(size_t length, char ch); - friend void AbslFormatFlush(StringifySink* sink, absl::string_view text); + friend void AbslFormatFlush(StringifySink* absl_nonnull sink, + absl::string_view text); private: std::ostream& os_; @@ -352,12 +353,12 @@ using CheckOpStreamType = decltype(detect_specialization::Detect(0)); // Build the error message string. Specify no inlining for code size. template -ABSL_ATTRIBUTE_RETURNS_NONNULL absl::Nonnull MakeCheckOpString( - T1 v1, T2 v2, absl::Nonnull exprtext) ABSL_ATTRIBUTE_NOINLINE; +ABSL_ATTRIBUTE_RETURNS_NONNULL const char* absl_nonnull MakeCheckOpString( + T1 v1, T2 v2, const char* absl_nonnull exprtext) ABSL_ATTRIBUTE_NOINLINE; template -absl::Nonnull MakeCheckOpString( - T1 v1, T2 v2, absl::Nonnull exprtext) { +const char* absl_nonnull MakeCheckOpString(T1 v1, T2 v2, + const char* absl_nonnull exprtext) { CheckOpMessageBuilder comb(exprtext); MakeCheckOpValueString(comb.ForVar1(), v1); MakeCheckOpValueString(comb.ForVar2(), v2); @@ -367,8 +368,8 @@ absl::Nonnull MakeCheckOpString( // Add a few commonly used instantiations as extern to reduce size of objects // files. #define ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(x) \ - extern template absl::Nonnull MakeCheckOpString( \ - x, x, absl::Nonnull) + extern template const char* absl_nonnull MakeCheckOpString( \ + x, x, const char* absl_nonnull) ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(bool); ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(int64_t); ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(uint64_t); @@ -378,10 +379,12 @@ ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(char); ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(unsigned char); ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const std::string&); ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const absl::string_view&); -ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const char*); -ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const signed char*); -ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const unsigned char*); -ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const void*); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const char* absl_nonnull); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN( + const signed char* absl_nonnull); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN( + const unsigned char* absl_nonnull); +ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const void* absl_nonnull); #undef ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN // `ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT` skips formatting the Check_OP result @@ -404,8 +407,8 @@ ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const void*); // type. #define ABSL_LOG_INTERNAL_CHECK_OP_IMPL(name, op) \ template \ - inline constexpr absl::Nullable name##Impl( \ - const T1& v1, const T2& v2, absl::Nonnull exprtext) { \ + inline constexpr const char* absl_nullable name##Impl( \ + const T1& v1, const T2& v2, const char* absl_nonnull exprtext) { \ using U1 = CheckOpStreamType; \ using U2 = CheckOpStreamType; \ return ABSL_PREDICT_TRUE(v1 op v2) \ @@ -413,8 +416,8 @@ ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const void*); : ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT(U1, U2, U1(v1), \ U2(v2), exprtext); \ } \ - inline constexpr absl::Nullable name##Impl( \ - int v1, int v2, absl::Nonnull exprtext) { \ + inline constexpr const char* absl_nullable name##Impl( \ + int v1, int v2, const char* absl_nonnull exprtext) { \ return name##Impl(v1, v2, exprtext); \ } @@ -427,18 +430,18 @@ ABSL_LOG_INTERNAL_CHECK_OP_IMPL(Check_GT, >) #undef ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT #undef ABSL_LOG_INTERNAL_CHECK_OP_IMPL -absl::Nullable CheckstrcmptrueImpl( - absl::Nullable s1, absl::Nullable s2, - absl::Nonnull exprtext); -absl::Nullable CheckstrcmpfalseImpl( - absl::Nullable s1, absl::Nullable s2, - absl::Nonnull exprtext); -absl::Nullable CheckstrcasecmptrueImpl( - absl::Nullable s1, absl::Nullable s2, - absl::Nonnull exprtext); -absl::Nullable CheckstrcasecmpfalseImpl( - absl::Nullable s1, absl::Nullable s2, - absl::Nonnull exprtext); +const char* absl_nullable CheckstrcmptrueImpl( + const char* absl_nullable s1, const char* absl_nullable s2, + const char* absl_nonnull exprtext); +const char* absl_nullable CheckstrcmpfalseImpl( + const char* absl_nullable s1, const char* absl_nullable s2, + const char* absl_nonnull exprtext); +const char* absl_nullable CheckstrcasecmptrueImpl( + const char* absl_nullable s1, const char* absl_nullable s2, + const char* absl_nonnull exprtext); +const char* absl_nullable CheckstrcasecmpfalseImpl( + const char* absl_nullable s1, const char* absl_nullable s2, + const char* absl_nonnull exprtext); // `CHECK_EQ` and friends want to pass their arguments by reference, however // this winds up exposing lots of cases where people have defined and diff --git a/absl/log/internal/conditions.cc b/absl/log/internal/conditions.cc index a9f4966..a418c88 100644 --- a/absl/log/internal/conditions.cc +++ b/absl/log/internal/conditions.cc @@ -63,8 +63,9 @@ bool LogEveryNSecState::ShouldLog(double seconds) { // myriad2 does not have 8-byte compare and exchange. Use a racy version that // is "good enough" but will over-log in the face of concurrent logging. if (now_cycles > next_cycles) { - next_log_time_cycles_.store(now_cycles + seconds * CycleClock::Frequency(), - std::memory_order_relaxed); + next_log_time_cycles_.store( + static_cast(now_cycles + seconds * CycleClock::Frequency()), + std::memory_order_relaxed); return true; } return false; @@ -72,7 +73,8 @@ bool LogEveryNSecState::ShouldLog(double seconds) { do { if (now_cycles <= next_cycles) return false; } while (!next_log_time_cycles_.compare_exchange_weak( - next_cycles, now_cycles + seconds * CycleClock::Frequency(), + next_cycles, + static_cast(now_cycles + seconds * CycleClock::Frequency()), std::memory_order_relaxed, std::memory_order_relaxed)); return true; #endif diff --git a/absl/log/internal/conditions.h b/absl/log/internal/conditions.h index 9dc15db..6fb74b1 100644 --- a/absl/log/internal/conditions.h +++ b/absl/log/internal/conditions.h @@ -65,7 +65,7 @@ switch (0) \ case 0: \ default: \ - !(condition) ? (void)0 : ::absl::log_internal::Voidify()&& + !(condition) ? (void)0 : ::absl::log_internal::Voidify() && // `ABSL_LOG_INTERNAL_STATEFUL_CONDITION` applies a condition like // `ABSL_LOG_INTERNAL_STATELESS_CONDITION` but adds to that a series of variable @@ -96,7 +96,8 @@ for (const uint32_t COUNTER ABSL_ATTRIBUTE_UNUSED = \ absl_log_internal_stateful_condition_state.counter(); \ absl_log_internal_stateful_condition_do_log; \ - absl_log_internal_stateful_condition_do_log = false) + absl_log_internal_stateful_condition_do_log = false) \ + ::absl::log_internal::Voidify() && // `ABSL_LOG_INTERNAL_CONDITION_*` serve to combine any conditions from the // macro (e.g. `LOG_IF` or `VLOG`) with inherent conditions (e.g. @@ -117,6 +118,8 @@ ABSL_LOG_INTERNAL_##type##_CONDITION( \ (condition) && ::absl::LogSeverity::kError >= \ static_cast<::absl::LogSeverity>(ABSL_MIN_LOG_LEVEL)) +#define ABSL_LOG_INTERNAL_CONDITION_DO_NOT_SUBMIT(type, condition) \ + ABSL_LOG_INTERNAL_CONDITION_ERROR(type, condition) // NOTE: Use ternary operators instead of short-circuiting to mitigate // https://bugs.llvm.org/show_bug.cgi?id=51928. #define ABSL_LOG_INTERNAL_CONDITION_FATAL(type, condition) \ @@ -168,6 +171,8 @@ ABSL_LOG_INTERNAL_##type##_CONDITION(condition) #define ABSL_LOG_INTERNAL_CONDITION_ERROR(type, condition) \ ABSL_LOG_INTERNAL_##type##_CONDITION(condition) +#define ABSL_LOG_INTERNAL_CONDITION_DO_NOT_SUBMIT(type, condition) \ + ABSL_LOG_INTERNAL_CONDITION_ERROR(type, condition) #define ABSL_LOG_INTERNAL_CONDITION_FATAL(type, condition) \ ABSL_LOG_INTERNAL_##type##_CONDITION(condition) #define ABSL_LOG_INTERNAL_CONDITION_QFATAL(type, condition) \ diff --git a/absl/log/internal/log_message.cc b/absl/log/internal/log_message.cc index 9e7722d..3aed3a2 100644 --- a/absl/log/internal/log_message.cc +++ b/absl/log/internal/log_message.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include #include "absl/base/attributes.h" @@ -47,12 +48,14 @@ #include "absl/log/internal/globals.h" #include "absl/log/internal/log_format.h" #include "absl/log/internal/log_sink_set.h" +#include "absl/log/internal/nullguard.h" #include "absl/log/internal/proto.h" #include "absl/log/internal/structured_proto.h" #include "absl/log/log_entry.h" #include "absl/log/log_sink.h" #include "absl/log/log_sink_registry.h" #include "absl/memory/memory.h" +#include "absl/strings/internal/utf8.h" #include "absl/strings/string_view.h" #include "absl/time/clock.h" #include "absl/time/time.h" @@ -147,7 +150,7 @@ void WriteToStream(const char* data, void* os) { } // namespace struct LogMessage::LogMessageData final { - LogMessageData(absl::Nonnull file, int line, + LogMessageData(const char* absl_nonnull file, int line, absl::LogSeverity severity, absl::Time timestamp); LogMessageData(const LogMessageData&) = delete; LogMessageData& operator=(const LogMessageData&) = delete; @@ -163,7 +166,7 @@ struct LogMessage::LogMessageData final { bool is_perror; // Extra `LogSink`s to log to, in addition to `global_sinks`. - absl::InlinedVector, 16> extra_sinks; + absl::InlinedVector extra_sinks; // If true, log to `extra_sinks` but not to `global_sinks` or hardcoded // non-sink targets (e.g. stderr, log files). bool extra_sinks_only; @@ -199,7 +202,7 @@ struct LogMessage::LogMessageData final { void FinalizeEncodingAndFormat(); }; -LogMessage::LogMessageData::LogMessageData(absl::Nonnull file, +LogMessage::LogMessageData::LogMessageData(const char* absl_nonnull file, int line, absl::LogSeverity severity, absl::Time timestamp) : extra_sinks_only(false), manipulated(nullptr) { @@ -270,7 +273,7 @@ void LogMessage::LogMessageData::FinalizeEncodingAndFormat() { absl::MakeSpan(string_buf).subspan(0, chars_written); } -LogMessage::LogMessage(absl::Nonnull file, int line, +LogMessage::LogMessage(const char* absl_nonnull file, int line, absl::LogSeverity severity) : data_(absl::make_unique(file, line, severity, absl::Now())) { @@ -284,23 +287,15 @@ LogMessage::LogMessage(absl::Nonnull file, int line, LogBacktraceIfNeeded(); } -LogMessage::LogMessage(absl::Nonnull file, int line, InfoTag) +LogMessage::LogMessage(const char* absl_nonnull file, int line, InfoTag) : LogMessage(file, line, absl::LogSeverity::kInfo) {} -LogMessage::LogMessage(absl::Nonnull file, int line, WarningTag) +LogMessage::LogMessage(const char* absl_nonnull file, int line, WarningTag) : LogMessage(file, line, absl::LogSeverity::kWarning) {} -LogMessage::LogMessage(absl::Nonnull file, int line, ErrorTag) +LogMessage::LogMessage(const char* absl_nonnull file, int line, ErrorTag) : LogMessage(file, line, absl::LogSeverity::kError) {} -LogMessage::~LogMessage() { -#ifdef ABSL_MIN_LOG_LEVEL - if (data_->entry.log_severity() < - static_cast(ABSL_MIN_LOG_LEVEL) && - data_->entry.log_severity() < absl::LogSeverity::kFatal) { - return; - } -#endif - Flush(); -} +// This cannot go in the header since LogMessageData is defined in this file. +LogMessage::~LogMessage() = default; LogMessage& LogMessage::AtLocation(absl::string_view file, int line) { data_->entry.full_filename_ = file; @@ -351,13 +346,13 @@ LogMessage& LogMessage::WithPerror() { return *this; } -LogMessage& LogMessage::ToSinkAlso(absl::Nonnull sink) { +LogMessage& LogMessage::ToSinkAlso(absl::LogSink* absl_nonnull sink) { ABSL_INTERNAL_CHECK(sink, "null LogSink*"); data_->extra_sinks.push_back(sink); return *this; } -LogMessage& LogMessage::ToSinkOnly(absl::Nonnull sink) { +LogMessage& LogMessage::ToSinkOnly(absl::LogSink* absl_nonnull sink) { ABSL_INTERNAL_CHECK(sink, "null LogSink*"); data_->extra_sinks.clear(); data_->extra_sinks.push_back(sink); @@ -411,6 +406,34 @@ LogMessage& LogMessage::operator<<(absl::string_view v) { CopyToEncodedBuffer(v); return *this; } + +LogMessage& LogMessage::operator<<(const std::wstring& v) { + CopyToEncodedBuffer(v); + return *this; +} + +LogMessage& LogMessage::operator<<(std::wstring_view v) { + CopyToEncodedBuffer(v); + return *this; +} + +template <> +LogMessage& LogMessage::operator<< ( + const wchar_t* absl_nullable const& v) { + if (v == nullptr) { + CopyToEncodedBuffer( + absl::string_view(kCharNull.data(), kCharNull.size() - 1)); + } else { + CopyToEncodedBuffer(v); + } + return *this; +} + +LogMessage& LogMessage::operator<<(wchar_t v) { + CopyToEncodedBuffer(std::wstring_view(&v, 1)); + return *this; +} + LogMessage& LogMessage::operator<<(std::ostream& (*m)(std::ostream& os)) { OstreamView view(*data_); data_->manipulated << m; @@ -633,6 +656,37 @@ template void LogMessage::CopyToEncodedBuffer( template void LogMessage::CopyToEncodedBuffer< LogMessage::StringType::kNotLiteral>(char ch, size_t num); +template +void LogMessage::CopyToEncodedBuffer(std::wstring_view str) { + auto encoded_remaining_copy = data_->encoded_remaining(); + constexpr uint8_t tag_value = str_type == StringType::kLiteral + ? ValueTag::kStringLiteral + : ValueTag::kString; + size_t max_str_byte_length = + absl::strings_internal::kMaxEncodedUTF8Size * str.length(); + auto value_start = + EncodeMessageStart(EventTag::kValue, + BufferSizeFor(tag_value, WireType::kLengthDelimited) + + max_str_byte_length, + &encoded_remaining_copy); + auto str_start = EncodeMessageStart(tag_value, max_str_byte_length, + &encoded_remaining_copy); + if (str_start.data()) { + log_internal::AppendTruncated(str, encoded_remaining_copy); + EncodeMessageLength(str_start, &encoded_remaining_copy); + EncodeMessageLength(value_start, &encoded_remaining_copy); + data_->encoded_remaining() = encoded_remaining_copy; + } else { + // The field header(s) did not fit; zero `encoded_remaining()` so we don't + // write anything else later. + data_->encoded_remaining().remove_suffix(data_->encoded_remaining().size()); + } +} +template void LogMessage::CopyToEncodedBuffer( + std::wstring_view str); +template void LogMessage::CopyToEncodedBuffer< + LogMessage::StringType::kNotLiteral>(std::wstring_view str); + template void LogMessage::CopyToEncodedBufferWithStructuredProtoField< LogMessage::StringType::kLiteral>(StructuredProtoField field, absl::string_view str); @@ -681,57 +735,45 @@ void LogMessage::CopyToEncodedBufferWithStructuredProtoField( #pragma warning(disable : 4722) #endif -LogMessageFatal::LogMessageFatal(absl::Nonnull file, int line) +LogMessageFatal::LogMessageFatal(const char* absl_nonnull file, int line) : LogMessage(file, line, absl::LogSeverity::kFatal) {} -LogMessageFatal::LogMessageFatal(absl::Nonnull file, int line, - absl::Nonnull failure_msg) +LogMessageFatal::LogMessageFatal(const char* absl_nonnull file, int line, + const char* absl_nonnull failure_msg) : LogMessage(file, line, absl::LogSeverity::kFatal) { *this << "Check failed: " << failure_msg << " "; } -LogMessageFatal::~LogMessageFatal() { - Flush(); - FailWithoutStackTrace(); -} +LogMessageFatal::~LogMessageFatal() { FailWithoutStackTrace(); } -LogMessageDebugFatal::LogMessageDebugFatal(absl::Nonnull file, +LogMessageDebugFatal::LogMessageDebugFatal(const char* absl_nonnull file, int line) : LogMessage(file, line, absl::LogSeverity::kFatal) {} -LogMessageDebugFatal::~LogMessageDebugFatal() { - Flush(); - FailWithoutStackTrace(); -} +LogMessageDebugFatal::~LogMessageDebugFatal() { FailWithoutStackTrace(); } LogMessageQuietlyDebugFatal::LogMessageQuietlyDebugFatal( - absl::Nonnull file, int line) + const char* absl_nonnull file, int line) : LogMessage(file, line, absl::LogSeverity::kFatal) { SetFailQuietly(); } -LogMessageQuietlyDebugFatal::~LogMessageQuietlyDebugFatal() { - Flush(); - FailQuietly(); -} +LogMessageQuietlyDebugFatal::~LogMessageQuietlyDebugFatal() { FailQuietly(); } -LogMessageQuietlyFatal::LogMessageQuietlyFatal(absl::Nonnull file, +LogMessageQuietlyFatal::LogMessageQuietlyFatal(const char* absl_nonnull file, int line) : LogMessage(file, line, absl::LogSeverity::kFatal) { SetFailQuietly(); } LogMessageQuietlyFatal::LogMessageQuietlyFatal( - absl::Nonnull file, int line, - absl::Nonnull failure_msg) + const char* absl_nonnull file, int line, + const char* absl_nonnull failure_msg) : LogMessageQuietlyFatal(file, line) { *this << "Check failed: " << failure_msg << " "; } -LogMessageQuietlyFatal::~LogMessageQuietlyFatal() { - Flush(); - FailQuietly(); -} +LogMessageQuietlyFatal::~LogMessageQuietlyFatal() { FailQuietly(); } #if defined(_MSC_VER) && !defined(__clang__) #pragma warning(pop) #endif diff --git a/absl/log/internal/log_message.h b/absl/log/internal/log_message.h index 7d0e403..1aaf05e 100644 --- a/absl/log/internal/log_message.h +++ b/absl/log/internal/log_message.h @@ -17,22 +17,25 @@ // ----------------------------------------------------------------------------- // // This file declares `class absl::log_internal::LogMessage`. This class more or -// less represents a particular log message. LOG/CHECK macros create a -// temporary instance of `LogMessage` and then stream values to it. At the end -// of the LOG/CHECK statement, LogMessage instance goes out of scope and -// `~LogMessage` directs the message to the registered log sinks. -// Heap-allocation of `LogMessage` is unsupported. Construction outside of a -// `LOG` macro is unsupported. +// less represents a particular log message. LOG/CHECK macros create a temporary +// instance of `LogMessage` and then stream values to it. At the end of the +// LOG/CHECK statement, the LogMessage is voidified by operator&&, and `Flush()` +// directs the message to the registered log sinks. Heap-allocation of +// `LogMessage` is unsupported. Construction outside of a `LOG` macro is +// unsupported. #ifndef ABSL_LOG_INTERNAL_LOG_MESSAGE_H_ #define ABSL_LOG_INTERNAL_LOG_MESSAGE_H_ +#include + #include #include #include #include #include #include +#include #include #include "absl/base/attributes.h" @@ -62,15 +65,15 @@ class LogMessage { struct ErrorTag {}; // Used for `LOG`. - LogMessage(absl::Nonnull file, int line, + LogMessage(const char* absl_nonnull file, int line, absl::LogSeverity severity) ABSL_ATTRIBUTE_COLD; // These constructors are slightly smaller/faster to call; the severity is // curried into the function pointer. - LogMessage(absl::Nonnull file, int line, + LogMessage(const char* absl_nonnull file, int line, InfoTag) ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE; - LogMessage(absl::Nonnull file, int line, + LogMessage(const char* absl_nonnull file, int line, WarningTag) ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE; - LogMessage(absl::Nonnull file, int line, + LogMessage(const char* absl_nonnull file, int line, ErrorTag) ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE; LogMessage(const LogMessage&) = delete; LogMessage& operator=(const LogMessage&) = delete; @@ -102,9 +105,9 @@ class LogMessage { LogMessage& WithPerror(); // Sends this message to `*sink` in addition to whatever other sinks it would // otherwise have been sent to. - LogMessage& ToSinkAlso(absl::Nonnull sink); + LogMessage& ToSinkAlso(absl::LogSink* absl_nonnull sink); // Sends this message to `*sink` and no others. - LogMessage& ToSinkOnly(absl::Nonnull sink); + LogMessage& ToSinkOnly(absl::LogSink* absl_nonnull sink); // Don't call this method from outside this library. LogMessage& InternalStream() { return *this; } @@ -141,10 +144,10 @@ class LogMessage { LogMessage& operator<<(unsigned long long v) { return operator<< (v); } - LogMessage& operator<<(absl::Nullable v) { + LogMessage& operator<<(void* absl_nullable v) { return operator<< (v); } - LogMessage& operator<<(absl::Nullable v) { + LogMessage& operator<<(const void* absl_nullable v) { return operator<< (v); } LogMessage& operator<<(float v) { return operator<< (v); } @@ -158,10 +161,16 @@ class LogMessage { LogMessage& operator<<(const std::string& v); LogMessage& operator<<(absl::string_view v); + // Wide string overloads (since std::ostream does not provide them). + LogMessage& operator<<(const std::wstring& v); + LogMessage& operator<<(std::wstring_view v); + // `const wchar_t*` is handled by `operator<< `. + LogMessage& operator<<(wchar_t* absl_nullable v); + LogMessage& operator<<(wchar_t v); + // Handle stream manipulators e.g. std::endl. - LogMessage& operator<<(absl::Nonnull m); - LogMessage& operator<<( - absl::Nonnull m); + LogMessage& operator<<(std::ostream& (*absl_nonnull m)(std::ostream& os)); + LogMessage& operator<<(std::ios_base& (*absl_nonnull m)(std::ios_base& os)); // Literal strings. This allows us to record C string literals as literals in // the logging.proto.Value. @@ -170,31 +179,30 @@ class LogMessage { // this template for every value of `SIZE` encountered in each source code // file. That significantly increases linker input sizes. Inlining is cheap // because the argument to this overload is almost always a string literal so - // the call to `strlen` can be replaced at compile time. The overload for - // `char[]` below should not be inlined. The compiler typically does not have - // the string at compile time and cannot replace the call to `strlen` so - // inlining it increases the binary size. See the discussion on + // the call to `strlen` can be replaced at compile time. The overloads for + // `char[]`/`wchar_t[]` below should not be inlined. The compiler typically + // does not have the string at compile time and cannot replace the call to + // `strlen` so inlining it increases the binary size. See the discussion on // cl/107527369. template LogMessage& operator<<(const char (&buf)[SIZE]); + template + LogMessage& operator<<(const wchar_t (&buf)[SIZE]); // This prevents non-const `char[]` arrays from looking like literals. template LogMessage& operator<<(char (&buf)[SIZE]) ABSL_ATTRIBUTE_NOINLINE; + // `wchar_t[SIZE]` is handled by `operator<< `. // Types that support `AbslStringify()` are serialized that way. - template ::value, - int>::type = 0> - LogMessage& operator<<(const T& v) ABSL_ATTRIBUTE_NOINLINE; - // Types that don't support `AbslStringify()` but do support streaming into a // `std::ostream&` are serialized that way. - template ::value, - int>::type = 0> + template LogMessage& operator<<(const T& v) ABSL_ATTRIBUTE_NOINLINE; + // Dispatches the completed `absl::LogEntry` to applicable `absl::LogSink`s. + void Flush(); + // Note: We explicitly do not support `operator<<` for non-const references // because it breaks logging of non-integer bitfield types (i.e., enums). @@ -207,11 +215,6 @@ class LogMessage { // the process with an error exit code. [[noreturn]] static void FailQuietly(); - // Dispatches the completed `absl::LogEntry` to applicable `absl::LogSink`s. - // This might as well be inlined into `~LogMessage` except that - // `~LogMessageFatal` needs to call it early. - void Flush(); - // After this is called, failures are done as quiet as possible for this log // message. void SetFailQuietly(); @@ -253,6 +256,8 @@ class LogMessage { void CopyToEncodedBuffer(absl::string_view str) ABSL_ATTRIBUTE_NOINLINE; template void CopyToEncodedBuffer(char ch, size_t num) ABSL_ATTRIBUTE_NOINLINE; + template + void CopyToEncodedBuffer(std::wstring_view str) ABSL_ATTRIBUTE_NOINLINE; // Copies `field` to the encoded buffer, then appends `str` after it // (truncating `str` if necessary to fit). @@ -280,9 +285,25 @@ class LogMessage { // We keep the data in a separate struct so that each instance of `LogMessage` // uses less stack space. - absl::Nonnull> data_; + absl_nonnull std::unique_ptr data_; }; +// Explicitly specializes the generic operator<< for `const wchar_t*` +// arguments. +// +// This method is used instead of a non-template `const wchar_t*` overload, +// as the latter was found to take precedence over the array template +// (`operator<<(const wchar_t(&)[SIZE])`) when handling string literals. +// This specialization ensures the array template now correctly processes +// literals. +template <> +LogMessage& LogMessage::operator<< ( + const wchar_t* absl_nullable const& v); + +inline LogMessage& LogMessage::operator<<(wchar_t* absl_nullable v) { + return operator<<(const_cast(v)); +} + // Helper class so that `AbslStringify()` can modify the LogMessage. class StringifySink final { public: @@ -298,7 +319,7 @@ class StringifySink final { } // For types that implement `AbslStringify` using `absl::Format()`. - friend void AbslFormatFlush(absl::Nonnull sink, + friend void AbslFormatFlush(StringifySink* absl_nonnull sink, absl::string_view v) { sink->Append(v); } @@ -308,26 +329,27 @@ class StringifySink final { }; // Note: the following is declared `ABSL_ATTRIBUTE_NOINLINE` -template ::value, int>::type> +template LogMessage& LogMessage::operator<<(const T& v) { - StringifySink sink(*this); - // Replace with public API. - AbslStringify(sink, v); + if constexpr (absl::HasAbslStringify::value) { + StringifySink sink(*this); + // Replace with public API. + AbslStringify(sink, v); + } else { + OstreamView view(*data_); + view.stream() << log_internal::NullGuard().Guard(v); + } return *this; } -// Note: the following is declared `ABSL_ATTRIBUTE_NOINLINE` -template ::value, int>::type> -LogMessage& LogMessage::operator<<(const T& v) { - OstreamView view(*data_); - view.stream() << log_internal::NullGuard().Guard(v); +template +LogMessage& LogMessage::operator<<(const char (&buf)[SIZE]) { + CopyToEncodedBuffer(buf); return *this; } template -LogMessage& LogMessage::operator<<(const char (&buf)[SIZE]) { +LogMessage& LogMessage::operator<<(const wchar_t (&buf)[SIZE]) { CopyToEncodedBuffer(buf); return *this; } @@ -355,9 +377,9 @@ extern template LogMessage& LogMessage::operator<<(const unsigned long& v); extern template LogMessage& LogMessage::operator<<(const long long& v); extern template LogMessage& LogMessage::operator<<(const unsigned long long& v); extern template LogMessage& LogMessage::operator<<( - absl::Nullable const& v); + void* absl_nullable const& v); extern template LogMessage& LogMessage::operator<<( - absl::Nullable const& v); + const void* absl_nullable const& v); extern template LogMessage& LogMessage::operator<<(const float& v); extern template LogMessage& LogMessage::operator<<(const double& v); extern template LogMessage& LogMessage::operator<<(const bool& v); @@ -373,15 +395,18 @@ LogMessage::CopyToEncodedBuffer(char ch, size_t num); extern template void LogMessage::CopyToEncodedBuffer< LogMessage::StringType::kNotLiteral>(char ch, size_t num); +extern template void LogMessage::CopyToEncodedBuffer< + LogMessage::StringType::kLiteral>(std::wstring_view str); +extern template void LogMessage::CopyToEncodedBuffer< + LogMessage::StringType::kNotLiteral>(std::wstring_view str); // `LogMessageFatal` ensures the process will exit in failure after logging this // message. class LogMessageFatal final : public LogMessage { public: - LogMessageFatal(absl::Nonnull file, - int line) ABSL_ATTRIBUTE_COLD; - LogMessageFatal(absl::Nonnull file, int line, - absl::Nonnull failure_msg) ABSL_ATTRIBUTE_COLD; + LogMessageFatal(const char* absl_nonnull file, int line) ABSL_ATTRIBUTE_COLD; + LogMessageFatal(const char* absl_nonnull file, int line, + const char* absl_nonnull failure_msg) ABSL_ATTRIBUTE_COLD; [[noreturn]] ~LogMessageFatal(); }; @@ -390,7 +415,7 @@ class LogMessageFatal final : public LogMessage { // for DLOG(FATAL) variants. class LogMessageDebugFatal final : public LogMessage { public: - LogMessageDebugFatal(absl::Nonnull file, + LogMessageDebugFatal(const char* absl_nonnull file, int line) ABSL_ATTRIBUTE_COLD; ~LogMessageDebugFatal(); }; @@ -400,7 +425,7 @@ class LogMessageQuietlyDebugFatal final : public LogMessage { // DLOG(QFATAL) calls this instead of LogMessageQuietlyFatal to make sure the // destructor is not [[noreturn]] even if this is always FATAL as this is only // invoked when DLOG() is enabled. - LogMessageQuietlyDebugFatal(absl::Nonnull file, + LogMessageQuietlyDebugFatal(const char* absl_nonnull file, int line) ABSL_ATTRIBUTE_COLD; ~LogMessageQuietlyDebugFatal(); }; @@ -408,10 +433,10 @@ class LogMessageQuietlyDebugFatal final : public LogMessage { // Used for LOG(QFATAL) to make sure it's properly understood as [[noreturn]]. class LogMessageQuietlyFatal final : public LogMessage { public: - LogMessageQuietlyFatal(absl::Nonnull file, + LogMessageQuietlyFatal(const char* absl_nonnull file, int line) ABSL_ATTRIBUTE_COLD; - LogMessageQuietlyFatal(absl::Nonnull file, int line, - absl::Nonnull failure_msg) + LogMessageQuietlyFatal(const char* absl_nonnull file, int line, + const char* absl_nonnull failure_msg) ABSL_ATTRIBUTE_COLD; [[noreturn]] ~LogMessageQuietlyFatal(); }; diff --git a/absl/log/internal/nullstream.h b/absl/log/internal/nullstream.h index 973e91a..c87f9aa 100644 --- a/absl/log/internal/nullstream.h +++ b/absl/log/internal/nullstream.h @@ -79,6 +79,7 @@ class NullStream { return *this; } NullStream& InternalStream() { return *this; } + void Flush() {} }; template inline NullStream& operator<<(NullStream& str, const T&) { diff --git a/absl/log/internal/proto.cc b/absl/log/internal/proto.cc index 3513b15..821be2b 100644 --- a/absl/log/internal/proto.cc +++ b/absl/log/internal/proto.cc @@ -123,8 +123,9 @@ bool EncodeBytesTruncate(uint64_t tag, absl::Span value, return true; } -ABSL_MUST_USE_RESULT absl::Span EncodeMessageStart( - uint64_t tag, uint64_t max_size, absl::Span *buf) { +[[nodiscard]] absl::Span EncodeMessageStart(uint64_t tag, + uint64_t max_size, + absl::Span *buf) { const uint64_t tag_type = MakeTagType(tag, WireType::kLengthDelimited); const size_t tag_type_size = VarintSize(tag_type); max_size = std::min(max_size, buf->size()); diff --git a/absl/log/internal/proto.h b/absl/log/internal/proto.h index 20a9f3a..23f7954 100644 --- a/absl/log/internal/proto.h +++ b/absl/log/internal/proto.h @@ -169,9 +169,9 @@ inline bool EncodeStringTruncate(uint64_t tag, absl::string_view value, // safe to pass to `EncodeMessageLength` but need not be. // Used for string, bytes, message, and packed-repeated field type. // Consumes up to kMaxVarintSize * 2 bytes (20). -ABSL_MUST_USE_RESULT absl::Span EncodeMessageStart(uint64_t tag, - uint64_t max_size, - absl::Span *buf); +[[nodiscard]] absl::Span EncodeMessageStart(uint64_t tag, + uint64_t max_size, + absl::Span *buf); // Finalizes the length field in `msg` so that it encompasses all data encoded // since the call to `EncodeMessageStart` which returned `msg`. Does nothing if diff --git a/absl/log/internal/strip.h b/absl/log/internal/strip.h index 3e55010..60ef878 100644 --- a/absl/log/internal/strip.h +++ b/absl/log/internal/strip.h @@ -15,7 +15,8 @@ // ----------------------------------------------------------------------------- // File: log/internal/strip.h // ----------------------------------------------------------------------------- -// + +// SKIP_ABSL_INLINE_NAMESPACE_CHECK #ifndef ABSL_LOG_INTERNAL_STRIP_H_ #define ABSL_LOG_INTERNAL_STRIP_H_ @@ -31,15 +32,6 @@ // logging in subtly different ways for subtly different reasons (see below). #if defined(STRIP_LOG) && STRIP_LOG -// Attribute for marking variables used in implementation details of logging -// macros as unused, but only when `STRIP_LOG` is defined. -// With `STRIP_LOG` on, not marking them triggers `-Wunused-but-set-variable`, -// With `STRIP_LOG` off, marking them triggers `-Wused-but-marked-unused`. -// -// TODO(b/290784225): Replace this macro with attribute [[maybe_unused]] when -// Abseil stops supporting C++14. -#define ABSL_LOG_INTERNAL_ATTRIBUTE_UNUSED_IF_STRIP_LOG ABSL_ATTRIBUTE_UNUSED - #define ABSL_LOGGING_INTERNAL_LOG_INFO ::absl::log_internal::NullStream() #define ABSL_LOGGING_INTERNAL_LOG_WARNING ::absl::log_internal::NullStream() #define ABSL_LOGGING_INTERNAL_LOG_ERROR ::absl::log_internal::NullStream() @@ -62,8 +54,6 @@ #else // !defined(STRIP_LOG) || !STRIP_LOG -#define ABSL_LOG_INTERNAL_ATTRIBUTE_UNUSED_IF_STRIP_LOG - #define ABSL_LOGGING_INTERNAL_LOG_INFO \ ::absl::log_internal::LogMessage( \ __FILE__, __LINE__, ::absl::log_internal::LogMessage::InfoTag{}) @@ -105,4 +95,6 @@ #define ABSL_LOGGING_INTERNAL_DLOG_DFATAL ABSL_LOGGING_INTERNAL_LOG_DFATAL #define ABSL_LOGGING_INTERNAL_DLOG_LEVEL ABSL_LOGGING_INTERNAL_LOG_LEVEL +#define ABSL_LOGGING_INTERNAL_LOG_DO_NOT_SUBMIT ABSL_LOGGING_INTERNAL_LOG_ERROR + #endif // ABSL_LOG_INTERNAL_STRIP_H_ diff --git a/absl/log/internal/structured.h b/absl/log/internal/structured.h index 50783df..70b50e2 100644 --- a/absl/log/internal/structured.h +++ b/absl/log/internal/structured.h @@ -34,7 +34,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace log_internal { -class ABSL_MUST_USE_RESULT AsLiteralImpl final { +class [[nodiscard]] AsLiteralImpl final { public: explicit AsLiteralImpl(absl::string_view str ABSL_ATTRIBUTE_LIFETIME_BOUND) : str_(str) {} @@ -66,7 +66,7 @@ enum class StructuredStringType { // Structured log data for a string and associated structured proto field, // both of which must outlive this object. template -class ABSL_MUST_USE_RESULT AsStructuredStringTypeImpl final { +class [[nodiscard]] AsStructuredStringTypeImpl final { public: constexpr AsStructuredStringTypeImpl( absl::string_view str ABSL_ATTRIBUTE_LIFETIME_BOUND, @@ -105,7 +105,7 @@ using AsStructuredNotLiteralImpl = // Structured log data for a stringifyable type T and associated structured // proto field, both of which must outlive this object. template -class ABSL_MUST_USE_RESULT AsStructuredValueImpl final { +class [[nodiscard]] AsStructuredValueImpl final { public: using ValueFormatter = absl::AnyInvocable; @@ -139,8 +139,6 @@ class ABSL_MUST_USE_RESULT AsStructuredValueImpl final { } }; -#ifdef ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION - // Template deduction guide so `AsStructuredValueImpl(42, data)` works // without specifying the template type. template @@ -155,8 +153,6 @@ AsStructuredValueImpl( typename AsStructuredValueImpl::ValueFormatter value_formatter) -> AsStructuredValueImpl; -#endif // ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION - } // namespace log_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/log/internal/vlog_config.h b/absl/log/internal/vlog_config.h index b6e322c..84e817a 100644 --- a/absl/log/internal/vlog_config.h +++ b/absl/log/internal/vlog_config.h @@ -34,6 +34,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/base/thread_annotations.h" #include "absl/strings/string_view.h" @@ -45,7 +46,7 @@ namespace log_internal { class SyntheticBinary; class VLogSite; -int RegisterAndInitialize(VLogSite* v); +int RegisterAndInitialize(VLogSite* absl_nonnull v); void UpdateVLogSites(); constexpr int kUseFlag = (std::numeric_limits::min)(); @@ -60,7 +61,7 @@ constexpr int kUseFlag = (std::numeric_limits::min)(); class VLogSite final { public: // `f` must not be destroyed until the program exits. - explicit constexpr VLogSite(const char* f) + explicit constexpr VLogSite(const char* absl_nonnull f) : file_(f), v_(kUninitialized), next_(nullptr) {} VLogSite(const VLogSite&) = delete; VLogSite& operator=(const VLogSite&) = delete; @@ -93,7 +94,7 @@ class VLogSite final { } private: - friend int log_internal::RegisterAndInitialize(VLogSite* v); + friend int log_internal::RegisterAndInitialize(VLogSite* absl_nonnull v); friend void log_internal::UpdateVLogSites(); friend class log_internal::SyntheticBinary; static constexpr int kUninitialized = (std::numeric_limits::max)(); @@ -116,7 +117,7 @@ class VLogSite final { ABSL_ATTRIBUTE_NOINLINE bool SlowIsEnabled5(int stale_v); // This object is too size-sensitive to use absl::string_view. - const char* const file_; + const char* absl_nonnull const file_; std::atomic v_; std::atomic next_; }; @@ -130,7 +131,7 @@ int VLogLevel(absl::string_view file); // Registers a site `v` to get updated as `vmodule` and `v` change. Also // initializes the site based on their current values, and returns that result. // Does not allocate memory. -int RegisterAndInitialize(VLogSite* v); +int RegisterAndInitialize(VLogSite* absl_nonnull v); // Allocates memory. void UpdateVLogSites(); @@ -154,7 +155,8 @@ int PrependVModule(absl::string_view module_pattern, int log_level); void OnVLogVerbosityUpdate(std::function cb); // Does not allocate memory. -VLogSite* SetVModuleListHeadForTestOnly(VLogSite* v); +VLogSite* absl_nullable SetVModuleListHeadForTestOnly( + VLogSite* absl_nullable v); } // namespace log_internal ABSL_NAMESPACE_END diff --git a/absl/log/internal/voidify.h b/absl/log/internal/voidify.h index 8f62da2..f42859e 100644 --- a/absl/log/internal/voidify.h +++ b/absl/log/internal/voidify.h @@ -16,13 +16,15 @@ // File: log/internal/voidify.h // ----------------------------------------------------------------------------- // -// This class is used to explicitly ignore values in the conditional logging -// macros. This avoids compiler warnings like "value computed is not used" and -// "statement has no effect". +// This class does the dispatching of the completed `absl::LogEntry` to +// applicable `absl::LogSink`s, and is used to explicitly ignore values in the +// conditional logging macros. This avoids compiler warnings like "value +// computed is not used" and "statement has no effect". #ifndef ABSL_LOG_INTERNAL_VOIDIFY_H_ #define ABSL_LOG_INTERNAL_VOIDIFY_H_ +#include "absl/base/attributes.h" #include "absl/base/config.h" namespace absl { @@ -34,7 +36,11 @@ class Voidify final { // This has to be an operator with a precedence lower than << but higher than // ?: template - void operator&&(const T&) const&& {} + ABSL_ATTRIBUTE_COLD void operator&&(T&& message) const&& { + // The dispatching of the completed `absl::LogEntry` to applicable + // `absl::LogSink`s happens here. + message.Flush(); + } }; } // namespace log_internal diff --git a/absl/log/log.h b/absl/log/log.h index a4e1d1f..f1cab9d 100644 --- a/absl/log/log.h +++ b/absl/log/log.h @@ -34,6 +34,13 @@ // running registered error handlers. // * The `DFATAL` pseudo-severity level is defined as `FATAL` in debug mode and // as `ERROR` otherwise. +// * The `DO_NOT_SUBMIT` pseudo-severity level is an alias for `ERROR`, and is +// intended for debugging statements that won't be submitted. The name is +// chosen to be easy to spot in review and with tools in order to ensure that +// such statements aren't inadvertently checked in. +// The contract is that **it may not be checked in**, meaning that no +// in-contract uses will be affected if we decide in the future to remove it +// or change what it does. // Some preprocessor shenanigans are used to ensure that e.g. `LOG(INFO)` has // the same meaning even if a local symbol or preprocessor macro named `INFO` is // defined. To specify a severity level using an expression instead of a @@ -194,6 +201,8 @@ // LOG(INFO) << std::hex << 0xdeadbeef; // logs "0xdeadbeef" // LOG(INFO) << 0xdeadbeef; // logs "3735928559" +// SKIP_ABSL_INLINE_NAMESPACE_CHECK + #ifndef ABSL_LOG_LOG_H_ #define ABSL_LOG_LOG_H_ @@ -260,44 +269,55 @@ ABSL_LOG_INTERNAL_DLOG_IF_IMPL(_##severity, condition) // LOG_EVERY_N +// LOG_FIRST_N +// LOG_EVERY_POW_2 +// LOG_EVERY_N_SEC // -// An instance of `LOG_EVERY_N` increments a hidden zero-initialized counter -// every time execution passes through it and logs the specified message when -// the counter's value is a multiple of `n`, doing nothing otherwise. Each -// instance has its own counter. The counter's value can be logged by streaming -// the symbol `COUNTER`. `LOG_EVERY_N` is thread-safe. -// Example: +// These "stateful" macros log conditionally based on a hidden counter or timer. +// When the condition is false and no logging is done, streamed operands aren't +// evaluated either. Each instance has its own state (i.e. counter, timer) +// that's independent of other instances of the macros. The macros in this +// family are thread-safe in the sense that they are meant to be called +// concurrently and will not invoke undefined behavior, however their +// implementation prioritizes efficiency over exactness and may occasionally log +// more or less often than specified. +// +// * `LOG_EVERY_N` logs the first time and once every `n` times thereafter. +// * `LOG_FIRST_N` logs the first `n` times and then stops. +// * `LOG_EVERY_POW_2` logs the first, second, fourth, eighth, etc. times. +// * `LOG_EVERY_N_SEC` logs the first time and no more than once every `n` +// seconds thereafter. `n` is passed as a floating point value. +// +// The `LOG_IF`... variations with an extra condition evaluate the specified +// condition first and short-circuit if it is false. For example, an evaluation +// of `LOG_IF_FIRST_N` does not count against the first `n` if the specified +// condition is false. Stateful `VLOG`... variations likewise short-circuit +// if `VLOG` is disabled. +// +// An approximate count of the number of times a particular instance's stateful +// condition has been evaluated (i.e. excluding those where a specified `LOG_IF` +// condition was false) can be included in the logged message by streaming the +// symbol `COUNTER`. +// +// The `n` parameter need not be a constant. Conditional logging following a +// change to `n` isn't fully specified, but it should converge on the new value +// within roughly `max(old_n, new_n)` evaluations or seconds. +// +// Examples: // // LOG_EVERY_N(WARNING, 1000) << "Got a packet with a bad CRC (" << COUNTER // << " total)"; +// +// LOG_EVERY_N_SEC(INFO, 2.5) << "Got " << COUNTER << " cookies so far"; +// +// LOG_IF_EVERY_N(INFO, (size > 1024), 10) << "Got the " << COUNTER +// << "th big cookie"; #define LOG_EVERY_N(severity, n) \ ABSL_LOG_INTERNAL_LOG_EVERY_N_IMPL(_##severity, n) - -// LOG_FIRST_N -// -// `LOG_FIRST_N` behaves like `LOG_EVERY_N` except that the specified message is -// logged when the counter's value is less than `n`. `LOG_FIRST_N` is -// thread-safe. #define LOG_FIRST_N(severity, n) \ ABSL_LOG_INTERNAL_LOG_FIRST_N_IMPL(_##severity, n) - -// LOG_EVERY_POW_2 -// -// `LOG_EVERY_POW_2` behaves like `LOG_EVERY_N` except that the specified -// message is logged when the counter's value is a power of 2. -// `LOG_EVERY_POW_2` is thread-safe. #define LOG_EVERY_POW_2(severity) \ ABSL_LOG_INTERNAL_LOG_EVERY_POW_2_IMPL(_##severity) - -// LOG_EVERY_N_SEC -// -// An instance of `LOG_EVERY_N_SEC` uses a hidden state variable to log the -// specified message at most once every `n_seconds`. A hidden counter of -// executions (whether a message is logged or not) is also maintained and can be -// logged by streaming the symbol `COUNTER`. `LOG_EVERY_N_SEC` is thread-safe. -// Example: -// -// LOG_EVERY_N_SEC(INFO, 2.5) << "Got " << COUNTER << " cookies so far"; #define LOG_EVERY_N_SEC(severity, n_seconds) \ ABSL_LOG_INTERNAL_LOG_EVERY_N_SEC_IMPL(_##severity, n_seconds) @@ -328,13 +348,6 @@ #define VLOG_EVERY_N_SEC(severity, n_seconds) \ ABSL_LOG_INTERNAL_VLOG_EVERY_N_SEC_IMPL(severity, n_seconds) -// `LOG_IF_EVERY_N` and friends behave as the corresponding `LOG_EVERY_N` -// but neither increment a counter nor log a message if condition is false (as -// `LOG_IF`). -// Example: -// -// LOG_IF_EVERY_N(INFO, (size > 1024), 10) << "Got the " << COUNTER -// << "th big cookie"; #define LOG_IF_EVERY_N(severity, condition, n) \ ABSL_LOG_INTERNAL_LOG_IF_EVERY_N_IMPL(_##severity, condition, n) #define LOG_IF_FIRST_N(severity, condition, n) \ diff --git a/absl/log/log_sink_registry.h b/absl/log/log_sink_registry.h index 3aa3bf6..a3fa9a3 100644 --- a/absl/log/log_sink_registry.h +++ b/absl/log/log_sink_registry.h @@ -44,10 +44,10 @@ ABSL_NAMESPACE_BEGIN // sink instead which writes them to `stderr`. // // Do not call these inside `absl::LogSink::Send`. -inline void AddLogSink(absl::Nonnull sink) { +inline void AddLogSink(absl::LogSink* absl_nonnull sink) { log_internal::AddLogSink(sink); } -inline void RemoveLogSink(absl::Nonnull sink) { +inline void RemoveLogSink(absl::LogSink* absl_nonnull sink) { log_internal::RemoveLogSink(sink); } diff --git a/absl/meta/type_traits.h b/absl/meta/type_traits.h index 02da067..02c1e63 100644 --- a/absl/meta/type_traits.h +++ b/absl/meta/type_traits.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -48,10 +49,6 @@ #include // NOLINT(build/c++20) #endif -#ifdef ABSL_HAVE_STD_STRING_VIEW -#include -#endif - // Defines the default alignment. `__STDCPP_DEFAULT_NEW_ALIGNMENT__` is a C++17 // feature. #if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__) @@ -97,22 +94,6 @@ struct is_detected_impl>::type, Op, Args...> { template