Skip to content

Commit 05ba922

Browse files
goldvitalycopybara-github
authored andcommitted
Small cleanup of infoz processing to get the logic out of the line or removed.
1. For SOO growth from 1 to 3: remove recording since table couldn't be sampled in full SOO state. 2. For non-SOO growth for 0 to 1 and from 1 to 3: move reporting out of the line and reuse the code. Microbenchmarks show some wins for SOO case. ``` name old CYCLES/op new CYCLES/op delta BM_SWISSMAP_InsertManyToEmpty_Hot<::absl::flat_hash_set, 4>/set_size:1 14.2 ± 0% 14.2 ± 0% -0.15% (p=0.000 n=51+47) BM_SWISSMAP_InsertManyToEmpty_Hot<::absl::flat_hash_set, 4>/set_size:2 34.4 ± 0% 32.4 ± 0% -5.83% (p=0.000 n=51+54) ``` PiperOrigin-RevId: 761052883 Change-Id: Icfe1503e5234af6f1b61fee12078d0e0cebbfcac
1 parent f0835ec commit 05ba922

File tree

1 file changed

+50
-24
lines changed

1 file changed

+50
-24
lines changed

absl/container/internal/raw_hash_set.cc

Lines changed: 50 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,45 @@ size_t FindNewPositionsAndTransferSlots(
601601
return total_probe_length;
602602
}
603603

604+
void ReportGrowthToInfozImpl(CommonFields& common, HashtablezInfoHandle infoz,
605+
size_t hash, size_t total_probe_length,
606+
size_t distance_from_desired) {
607+
ABSL_SWISSTABLE_ASSERT(infoz.IsSampled());
608+
infoz.RecordStorageChanged(common.size() - 1, common.capacity());
609+
infoz.RecordRehash(total_probe_length);
610+
infoz.RecordInsert(hash, distance_from_desired);
611+
common.set_has_infoz();
612+
// TODO(b/413062340): we could potentially store infoz in place of the
613+
// control pointer for the capacity 1 case.
614+
common.set_infoz(infoz);
615+
}
616+
617+
// Specialization to avoid passing two 0s from hot function.
618+
ABSL_ATTRIBUTE_NOINLINE void ReportSingleGroupTableGrowthToInfoz(
619+
CommonFields& common, HashtablezInfoHandle infoz, size_t hash) {
620+
ReportGrowthToInfozImpl(common, infoz, hash, /*total_probe_length=*/0,
621+
/*distance_from_desired=*/0);
622+
}
623+
624+
ABSL_ATTRIBUTE_NOINLINE void ReportGrowthToInfoz(CommonFields& common,
625+
HashtablezInfoHandle infoz,
626+
size_t hash,
627+
size_t total_probe_length,
628+
size_t distance_from_desired) {
629+
ReportGrowthToInfozImpl(common, infoz, hash, total_probe_length,
630+
distance_from_desired);
631+
}
632+
633+
ABSL_ATTRIBUTE_NOINLINE void ReportResizeToInfoz(CommonFields& common,
634+
HashtablezInfoHandle infoz,
635+
size_t total_probe_length) {
636+
ABSL_SWISSTABLE_ASSERT(infoz.IsSampled());
637+
infoz.RecordStorageChanged(common.size(), common.capacity());
638+
infoz.RecordRehash(total_probe_length);
639+
common.set_has_infoz();
640+
common.set_infoz(infoz);
641+
}
642+
604643
struct BackingArrayPtrs {
605644
ctrl_t* ctrl;
606645
void* slots;
@@ -662,11 +701,8 @@ void ResizeNonSooImpl(CommonFields& common,
662701
CapacityToGrowth(new_capacity));
663702
}
664703

665-
if (has_infoz) {
666-
common.set_has_infoz();
667-
infoz.RecordStorageChanged(common.size(), new_capacity);
668-
infoz.RecordRehash(total_probe_length);
669-
common.set_infoz(infoz);
704+
if (ABSL_PREDICT_FALSE(has_infoz)) {
705+
ReportResizeToInfoz(common, infoz, total_probe_length);
670706
}
671707
}
672708

@@ -1268,14 +1304,11 @@ std::pair<ctrl_t*, void*> Grow1To3AndPrepareInsert(
12681304
policy.dealloc(alloc, kOldCapacity, old_ctrl, slot_size, slot_align,
12691305
has_infoz);
12701306
PrepareInsertCommon(common);
1271-
GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(1);
1307+
ABSL_SWISSTABLE_ASSERT(common.size() == 2);
1308+
GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(kNewCapacity - 2);
12721309

12731310
if (ABSL_PREDICT_FALSE(has_infoz)) {
1274-
common.set_has_infoz();
1275-
infoz.RecordStorageChanged(common.size() - 1, kNewCapacity);
1276-
infoz.RecordRehash(0);
1277-
infoz.RecordInsert(new_hash, 0);
1278-
common.set_infoz(infoz);
1311+
ReportSingleGroupTableGrowthToInfoz(common, infoz, new_hash);
12791312
}
12801313
return {new_ctrl + offset, new_element_target_slot};
12811314
}
@@ -1342,11 +1375,8 @@ size_t GrowToNextCapacityAndPrepareInsert(
13421375
common.size());
13431376

13441377
if (ABSL_PREDICT_FALSE(has_infoz)) {
1345-
common.set_has_infoz();
1346-
infoz.RecordStorageChanged(common.size() - 1, new_capacity);
1347-
infoz.RecordRehash(total_probe_length);
1348-
infoz.RecordInsert(new_hash, find_info.probe_length);
1349-
common.set_infoz(infoz);
1378+
ReportGrowthToInfoz(common, infoz, new_hash, total_probe_length,
1379+
find_info.probe_length);
13501380
}
13511381
return find_info.offset;
13521382
}
@@ -1396,13 +1426,7 @@ std::pair<ctrl_t*, void*> SmallNonSooPrepareInsert(
13961426
GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(0);
13971427

13981428
if (ABSL_PREDICT_FALSE(has_infoz)) {
1399-
// TODO(b/413062340): we could potentially store infoz in place of the
1400-
// control pointer for the capacity 1 case.
1401-
common.set_has_infoz();
1402-
infoz.RecordStorageChanged(/*size=*/0, kNewCapacity);
1403-
infoz.RecordRehash(/*total_probe_length=*/0);
1404-
infoz.RecordInsert(get_hash(), /*distance_from_desired=*/0);
1405-
common.set_infoz(infoz);
1429+
ReportSingleGroupTableGrowthToInfoz(common, infoz, get_hash());
14061430
}
14071431
return {SooControl(), new_slots};
14081432
}
@@ -1652,7 +1676,9 @@ size_t GrowSooTableToNextCapacityAndPrepareInsert(
16521676
common.set_control</*kGenerateSeed=*/false>(new_ctrl);
16531677
common.set_slots(new_slots);
16541678

1655-
common.infoz().RecordInsert(new_hash, /*distance_from_desired=*/0);
1679+
// Full SOO table couldn't be sampled. If SOO table is sampled, it would
1680+
// have been resized to the next capacity.
1681+
ABSL_SWISSTABLE_ASSERT(!common.infoz().IsSampled());
16561682
SanitizerUnpoisonMemoryRegion(SlotAddress(new_slots, offset, slot_size),
16571683
slot_size);
16581684
return offset;

0 commit comments

Comments
 (0)