Skip to content

Commit 7971b4a

Browse files
goldvitalycopybara-github
goldvitaly
authored and
copybara-github
committed
Store CharAlloc in SwissTable in order to simplify type erasure of functions accepting allocator as void*.
This way we can more naturally use CharAlloc as template parameter for type erased functions. PiperOrigin-RevId: 729010441 Change-Id: Ib733690335bbd349add0b290dc272ee9d02f3c7e
1 parent 2844cda commit 7971b4a

File tree

3 files changed

+109
-71
lines changed

3 files changed

+109
-71
lines changed

‎absl/container/internal/raw_hash_set.cc‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,7 @@ void ResizeNonSooImpl(CommonFields& common, size_t new_capacity,
477477

478478
common.set_capacity(new_capacity);
479479
RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz);
480-
void* alloc = policy.alloc_fn(common);
480+
void* alloc = policy.get_char_alloc(common);
481481
char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
482482
const GenerationType old_generation = common.generation();
483483
common.set_generation_ptr(
@@ -589,7 +589,7 @@ void ResizeFullSooTable(CommonFields& common, size_t new_capacity,
589589
common.set_capacity(new_capacity);
590590

591591
RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz);
592-
void* alloc = policy.alloc_fn(common);
592+
void* alloc = policy.get_char_alloc(common);
593593
char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
594594
const GenerationType old_generation = common.generation();
595595
common.set_generation_ptr(
@@ -767,7 +767,7 @@ size_t GrowToNextCapacityAndPrepareInsert(CommonFields& common, size_t new_hash,
767767
const bool has_infoz = infoz.IsSampled();
768768

769769
RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz);
770-
void* alloc = policy.alloc_fn(common);
770+
void* alloc = policy.get_char_alloc(common);
771771
char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
772772
const GenerationType old_generation = common.generation();
773773
common.set_generation_ptr(
@@ -928,8 +928,8 @@ void Rehash(CommonFields& common, size_t n, const PolicyFunctions& policy) {
928928
const size_t cap = common.capacity();
929929

930930
auto clear_backing_array = [&]() {
931-
ClearBackingArray(common, policy, policy.alloc_fn(common), /*reuse=*/false,
932-
policy.soo_capacity > 0);
931+
ClearBackingArray(common, policy, policy.get_char_alloc(common),
932+
/*reuse=*/false, policy.soo_capacity > 0);
933933
};
934934

935935
const size_t slot_size = policy.slot_size;
@@ -956,9 +956,9 @@ void Rehash(CommonFields& common, size_t n, const PolicyFunctions& policy) {
956956
assert(policy.slot_align <= alignof(HeapOrSoo));
957957
HeapOrSoo tmp_slot;
958958
size_t begin_offset = FindFirstFullSlot(0, cap, common.control());
959-
policy.transfer(
960-
&common, &tmp_slot,
961-
SlotAddress(common.slot_array(), begin_offset, slot_size), 1);
959+
policy.transfer(&common, &tmp_slot,
960+
SlotAddress(common.slot_array(), begin_offset, slot_size),
961+
1);
962962
clear_backing_array();
963963
policy.transfer(&common, common.soo_data(), &tmp_slot, 1);
964964
common.set_full_soo();

‎absl/container/internal/raw_hash_set.h‎

Lines changed: 52 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -2012,8 +2012,8 @@ struct PolicyFunctions {
20122012
// TODO(b/382423690): consider having separate `transfer` and `transfer_n`.
20132013
void (*transfer)(void* set, void* dst_slot, void* src_slot, size_t count);
20142014

2015-
// Returns the pointer to the allocator stored in the set.
2016-
void* (*alloc_fn)(CommonFields& common);
2015+
// Returns the pointer to the CharAlloc stored in the set.
2016+
void* (*get_char_alloc)(CommonFields& common);
20172017

20182018
// Allocates n bytes for the backing store for common.
20192019
void* (*alloc)(void* alloc, size_t n);
@@ -2159,7 +2159,7 @@ void GrowFullSooTableToNextCapacity(CommonFields& common, size_t soo_slot_hash,
21592159
// The decision to sample was already made during the first insertion.
21602160
RawHashSetLayout layout(kNewCapacity, slot_size, slot_align,
21612161
/*has_infoz=*/false);
2162-
void* alloc = policy.alloc_fn(common);
2162+
void* alloc = policy.get_char_alloc(common);
21632163
char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
21642164
const GenerationType old_generation = common.generation();
21652165
common.set_generation_ptr(
@@ -2675,7 +2675,7 @@ class raw_hash_set {
26752675

26762676
raw_hash_set(const raw_hash_set& that)
26772677
: raw_hash_set(that, AllocTraits::select_on_container_copy_construction(
2678-
that.alloc_ref())) {}
2678+
allocator_type(that.char_alloc_ref()))) {}
26792679

26802680
raw_hash_set(const raw_hash_set& that, const allocator_type& a)
26812681
: raw_hash_set(GrowthToLowerboundCapacity(that.size()), that.hash_ref(),
@@ -2758,7 +2758,7 @@ class raw_hash_set {
27582758
settings_(PolicyTraits::transfer_uses_memcpy() || !that.is_full_soo()
27592759
? std::move(that.common())
27602760
: CommonFields{full_soo_tag_t{}},
2761-
that.hash_ref(), that.eq_ref(), that.alloc_ref()) {
2761+
that.hash_ref(), that.eq_ref(), that.char_alloc_ref()) {
27622762
if (!PolicyTraits::transfer_uses_memcpy() && that.is_full_soo()) {
27632763
transfer(soo_slot(), that.soo_slot());
27642764
}
@@ -2769,7 +2769,7 @@ class raw_hash_set {
27692769
raw_hash_set(raw_hash_set&& that, const allocator_type& a)
27702770
: settings_(CommonFields::CreateDefault<SooEnabled()>(), that.hash_ref(),
27712771
that.eq_ref(), a) {
2772-
if (a == that.alloc_ref()) {
2772+
if (CharAlloc(a) == that.char_alloc_ref()) {
27732773
swap_common(that);
27742774
annotate_for_bug_detection_on_move(that);
27752775
} else {
@@ -2786,7 +2786,9 @@ class raw_hash_set {
27862786
// is an exact match for that.size(). If this->capacity() is too big, then
27872787
// it would make iteration very slow to reuse the allocation. Maybe we can
27882788
// do the same heuristic as clear() and reuse if it's small enough.
2789-
raw_hash_set tmp(that, propagate_alloc ? that.alloc_ref() : alloc_ref());
2789+
allocator_type alloc(propagate_alloc ? that.char_alloc_ref()
2790+
: char_alloc_ref());
2791+
raw_hash_set tmp(that, alloc);
27902792
// NOLINTNEXTLINE: not returning *this for performance.
27912793
return assign_impl<propagate_alloc>(std::move(tmp));
27922794
}
@@ -3112,7 +3114,8 @@ class raw_hash_set {
31123114
auto res = find_or_prepare_insert(key);
31133115
if (res.second) {
31143116
slot_type* slot = res.first.slot();
3115-
std::forward<F>(f)(constructor(&alloc_ref(), &slot));
3117+
allocator_type alloc(char_alloc_ref());
3118+
std::forward<F>(f)(constructor(&alloc, &slot));
31163119
ABSL_SWISSTABLE_ASSERT(!slot);
31173120
}
31183121
return res.first;
@@ -3216,7 +3219,8 @@ class raw_hash_set {
32163219
AssertNotDebugCapacity();
32173220
AssertIsFull(position.control(), position.inner_.generation(),
32183221
position.inner_.generation_ptr(), "extract()");
3219-
auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.slot());
3222+
allocator_type alloc(char_alloc_ref());
3223+
auto node = CommonAccess::Transfer<node_type>(alloc, position.slot());
32203224
if (is_soo()) {
32213225
common().set_empty_soo();
32223226
} else {
@@ -3242,7 +3246,7 @@ class raw_hash_set {
32423246
swap_common(that);
32433247
swap(hash_ref(), that.hash_ref());
32443248
swap(eq_ref(), that.eq_ref());
3245-
SwapAlloc(alloc_ref(), that.alloc_ref(),
3249+
SwapAlloc(char_alloc_ref(), that.char_alloc_ref(),
32463250
typename AllocTraits::propagate_on_container_swap{});
32473251
}
32483252

@@ -3364,7 +3368,9 @@ class raw_hash_set {
33643368

33653369
hasher hash_function() const { return hash_ref(); }
33663370
key_equal key_eq() const { return eq_ref(); }
3367-
allocator_type get_allocator() const { return alloc_ref(); }
3371+
allocator_type get_allocator() const {
3372+
return allocator_type(char_alloc_ref());
3373+
}
33683374

33693375
friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) {
33703376
if (a.size() != b.size()) return false;
@@ -3431,7 +3437,7 @@ class raw_hash_set {
34313437
struct EqualElement {
34323438
template <class K2, class... Args>
34333439
bool operator()(const K2& lhs, Args&&...) const {
3434-
return eq(lhs, rhs);
3440+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(eq(lhs, rhs));
34353441
}
34363442
const K1& rhs;
34373443
const key_equal& eq;
@@ -3469,16 +3475,21 @@ class raw_hash_set {
34693475
template <typename... Args>
34703476
inline void construct(slot_type* slot, Args&&... args) {
34713477
common().RunWithReentrancyGuard([&] {
3472-
PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...);
3478+
allocator_type alloc(char_alloc_ref());
3479+
PolicyTraits::construct(&alloc, slot, std::forward<Args>(args)...);
34733480
});
34743481
}
34753482
inline void destroy(slot_type* slot) {
3476-
common().RunWithReentrancyGuard(
3477-
[&] { PolicyTraits::destroy(&alloc_ref(), slot); });
3483+
common().RunWithReentrancyGuard([&] {
3484+
allocator_type alloc(char_alloc_ref());
3485+
PolicyTraits::destroy(&alloc, slot);
3486+
});
34783487
}
34793488
inline void transfer(slot_type* to, slot_type* from) {
3480-
common().RunWithReentrancyGuard(
3481-
[&] { PolicyTraits::transfer(&alloc_ref(), to, from); });
3489+
common().RunWithReentrancyGuard([&] {
3490+
allocator_type alloc(char_alloc_ref());
3491+
PolicyTraits::transfer(&alloc, to, from);
3492+
});
34823493
}
34833494

34843495
// TODO(b/289225379): consider having a helper class that has the impls for
@@ -3522,8 +3533,7 @@ class raw_hash_set {
35223533

35233534
void clear_backing_array(bool reuse) {
35243535
ABSL_SWISSTABLE_ASSERT(capacity() > DefaultCapacity());
3525-
CharAlloc alloc(alloc_ref());
3526-
ClearBackingArray(common(), GetPolicyFunctions(), &alloc, reuse,
3536+
ClearBackingArray(common(), GetPolicyFunctions(), &char_alloc_ref(), reuse,
35273537
SooEnabled());
35283538
}
35293539

@@ -3541,9 +3551,8 @@ class raw_hash_set {
35413551
// Unpoison before returning the memory to the allocator.
35423552
SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * capacity());
35433553
infoz().Unregister();
3544-
CharAlloc alloc(alloc_ref());
35453554
DeallocateBackingArray<BackingArrayAlignment(alignof(slot_type)),
3546-
CharAlloc>(&alloc, capacity(), control(),
3555+
CharAlloc>(&char_alloc_ref(), capacity(), control(),
35473556
sizeof(slot_type), alignof(slot_type),
35483557
common().has_infoz());
35493558
}
@@ -3598,7 +3607,7 @@ class raw_hash_set {
35983607
static slot_type* to_slot(void* buf) { return static_cast<slot_type*>(buf); }
35993608

36003609
// Requires that lhs does not have a full SOO slot.
3601-
static void move_common(bool rhs_is_full_soo, allocator_type& rhs_alloc,
3610+
static void move_common(bool rhs_is_full_soo, CharAlloc& rhs_alloc,
36023611
CommonFields& lhs, CommonFields&& rhs) {
36033612
if (PolicyTraits::transfer_uses_memcpy() || !rhs_is_full_soo) {
36043613
lhs = std::move(rhs);
@@ -3623,10 +3632,12 @@ class raw_hash_set {
36233632
}
36243633
CommonFields tmp = CommonFields(uninitialized_tag_t{});
36253634
const bool that_is_full_soo = that.is_full_soo();
3626-
move_common(that_is_full_soo, that.alloc_ref(), tmp,
3635+
move_common(that_is_full_soo, that.char_alloc_ref(), tmp,
36273636
std::move(that.common()));
3628-
move_common(is_full_soo(), alloc_ref(), that.common(), std::move(common()));
3629-
move_common(that_is_full_soo, that.alloc_ref(), common(), std::move(tmp));
3637+
move_common(is_full_soo(), char_alloc_ref(), that.common(),
3638+
std::move(common()));
3639+
move_common(that_is_full_soo, that.char_alloc_ref(), common(),
3640+
std::move(tmp));
36303641
}
36313642

36323643
void annotate_for_bug_detection_on_move(
@@ -3653,11 +3664,11 @@ class raw_hash_set {
36533664
// We don't bother checking for this/that aliasing. We just need to avoid
36543665
// breaking the invariants in that case.
36553666
destructor_impl();
3656-
move_common(that.is_full_soo(), that.alloc_ref(), common(),
3667+
move_common(that.is_full_soo(), that.char_alloc_ref(), common(),
36573668
std::move(that.common()));
36583669
hash_ref() = that.hash_ref();
36593670
eq_ref() = that.eq_ref();
3660-
CopyAlloc(alloc_ref(), that.alloc_ref(),
3671+
CopyAlloc(char_alloc_ref(), that.char_alloc_ref(),
36613672
std::integral_constant<bool, propagate_alloc>());
36623673
that.common() = CommonFields::CreateDefault<SooEnabled()>();
36633674
annotate_for_bug_detection_on_move(that);
@@ -3684,7 +3695,7 @@ class raw_hash_set {
36843695
}
36853696
raw_hash_set& move_assign(raw_hash_set&& that,
36863697
std::false_type /*propagate_alloc*/) {
3687-
if (alloc_ref() == that.alloc_ref()) {
3698+
if (char_alloc_ref() == that.char_alloc_ref()) {
36883699
return assign_impl<false>(std::move(that));
36893700
}
36903701
// Aliasing can't happen here because allocs would compare equal above.
@@ -3913,10 +3924,12 @@ class raw_hash_set {
39133924
}
39143925
slot_type* soo_slot() {
39153926
ABSL_SWISSTABLE_ASSERT(is_soo());
3916-
return static_cast<slot_type*>(common().soo_data());
3927+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(
3928+
static_cast<slot_type*>(common().soo_data()));
39173929
}
39183930
const slot_type* soo_slot() const {
3919-
return const_cast<raw_hash_set*>(this)->soo_slot();
3931+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(
3932+
const_cast<raw_hash_set*>(this)->soo_slot());
39203933
}
39213934
iterator soo_iterator() {
39223935
return {SooControl(), soo_slot(), common().generation_ptr()};
@@ -3933,14 +3946,14 @@ class raw_hash_set {
39333946
const hasher& hash_ref() const { return settings_.template get<1>(); }
39343947
key_equal& eq_ref() { return settings_.template get<2>(); }
39353948
const key_equal& eq_ref() const { return settings_.template get<2>(); }
3936-
allocator_type& alloc_ref() { return settings_.template get<3>(); }
3937-
const allocator_type& alloc_ref() const {
3949+
CharAlloc& char_alloc_ref() { return settings_.template get<3>(); }
3950+
const CharAlloc& char_alloc_ref() const {
39383951
return settings_.template get<3>();
39393952
}
39403953

3941-
static void* get_alloc_ref_fn(CommonFields& common) {
3954+
static void* get_char_alloc_ref_fn(CommonFields& common) {
39423955
auto* h = reinterpret_cast<raw_hash_set*>(&common);
3943-
return &h->alloc_ref();
3956+
return &h->char_alloc_ref();
39443957
}
39453958
static void* get_hash_ref_fn(CommonFields& common) {
39463959
auto* h = reinterpret_cast<raw_hash_set*>(&common);
@@ -3989,11 +4002,6 @@ class raw_hash_set {
39894002
static_assert(sizeof(value_type) <= (std::numeric_limits<uint32_t>::max)());
39904003
static constexpr size_t kBackingArrayAlignment =
39914004
BackingArrayAlignment(alignof(slot_type));
3992-
// TODO(b/397461659): store CharAlloc in the table instead of Alloc.
3993-
// If both allocators are empty, we can use the same pointer for both
3994-
// allocators.
3995-
static constexpr bool kAllocAndCharAllocPointersCompatible =
3996-
std::is_empty_v<CharAlloc> && std::is_empty_v<Alloc>;
39974005
static constexpr PolicyFunctions value = {
39984006
sizeof(key_type), sizeof(value_type), sizeof(slot_type),
39994007
alignof(slot_type), SooEnabled() ? SooCapacity() : 0,
@@ -4007,13 +4015,9 @@ class raw_hash_set {
40074015
? TransferRelocatable<sizeof(slot_type)>
40084016
: &raw_hash_set::transfer_slots_fn,
40094017
std::is_empty_v<Alloc> ? &GetRefForEmptyClass
4010-
: &raw_hash_set::get_alloc_ref_fn,
4011-
kAllocAndCharAllocPointersCompatible
4012-
? &AllocateBackingArray<kBackingArrayAlignment, CharAlloc>
4013-
: &AllocateBackingArray<kBackingArrayAlignment, Alloc>,
4014-
kAllocAndCharAllocPointersCompatible
4015-
? &DeallocateBackingArray<kBackingArrayAlignment, CharAlloc>
4016-
: &DeallocateBackingArray<kBackingArrayAlignment, Alloc>,
4018+
: &raw_hash_set::get_char_alloc_ref_fn,
4019+
&AllocateBackingArray<kBackingArrayAlignment, CharAlloc>,
4020+
&DeallocateBackingArray<kBackingArrayAlignment, CharAlloc>,
40174021
&raw_hash_set::find_new_positions_and_transfer_slots_fn};
40184022
return value;
40194023
}
@@ -4022,9 +4026,9 @@ class raw_hash_set {
40224026
// CompressedTuple will ensure that sizeof is not affected by any of the empty
40234027
// fields that occur after CommonFields.
40244028
absl::container_internal::CompressedTuple<CommonFields, hasher, key_equal,
4025-
allocator_type>
4029+
CharAlloc>
40264030
settings_{CommonFields::CreateDefault<SooEnabled()>(), hasher{},
4027-
key_equal{}, allocator_type{}};
4031+
key_equal{}, CharAlloc{}};
40284032
};
40294033

40304034
// Friend access for free functions in raw_hash_set.h.

0 commit comments

Comments
 (0)