Skip to content

Commit 1756fcb

Browse files
cferris1000cferris1000
authored
[scudo] Add primary option to enable/disable cache blocks. (llvm#129794)
When configured this way, no primary blocks will be cached except the batch class. Nothing else changes, no change in the page releasing strategy.
1 parent e8b506b commit 1756fcb

File tree

12 files changed

+404
-175
lines changed

12 files changed

+404
-175
lines changed

‎compiler-rt/lib/scudo/standalone/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ set(SCUDO_HEADERS
7474
internal_defs.h
7575
linux.h
7676
list.h
77-
local_cache.h
7877
memtag.h
7978
mem_map.h
8079
mem_map_base.h
@@ -90,6 +89,7 @@ set(SCUDO_HEADERS
9089
report.h
9190
report_linux.h
9291
secondary.h
92+
size_class_allocator.h
9393
size_class_map.h
9494
stack_depot.h
9595
stats.h

‎compiler-rt/lib/scudo/standalone/allocator_config.def‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,10 @@ PRIMARY_REQUIRED(const s32, MaxReleaseToOsIntervalMs)
8181

8282
// PRIMARY_OPTIONAL(TYPE, NAME, DEFAULT)
8383
//
84+
85+
// Enables/disables primary block caching. Batch class still caches.
86+
PRIMARY_OPTIONAL(const bool, EnableBlockCache, true)
87+
8488
// The scale of a compact pointer. E.g., Ptr = Base + (CompactPtr << Scale).
8589
PRIMARY_OPTIONAL(const uptr, CompactPtrScale, SCUDO_MIN_ALIGNMENT_LOG)
8690

‎compiler-rt/lib/scudo/standalone/combined.h‎

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@
1515
#include "common.h"
1616
#include "flags.h"
1717
#include "flags_parser.h"
18-
#include "local_cache.h"
1918
#include "mem_map.h"
2019
#include "memtag.h"
2120
#include "mutex.h"
2221
#include "options.h"
2322
#include "quarantine.h"
2423
#include "report.h"
2524
#include "secondary.h"
25+
#include "size_class_allocator.h"
2626
#include "stack_depot.h"
2727
#include "string_utils.h"
2828
#include "tsd.h"
@@ -54,7 +54,7 @@ class Allocator {
5454
typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
5555
using SecondaryT =
5656
typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
57-
using CacheT = typename PrimaryT::CacheT;
57+
using SizeClassAllocatorT = typename PrimaryT::SizeClassAllocatorT;
5858
typedef Allocator<Config, PostInitCallback> ThisT;
5959
typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
6060

@@ -63,8 +63,9 @@ class Allocator {
6363
}
6464

6565
struct QuarantineCallback {
66-
explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache)
67-
: Allocator(Instance), Cache(LocalCache) {}
66+
explicit QuarantineCallback(ThisT &Instance,
67+
SizeClassAllocatorT &SizeClassAllocator)
68+
: Allocator(Instance), SizeClassAllocator(SizeClassAllocator) {}
6869

6970
// Chunk recycling function, returns a quarantined chunk to the backend,
7071
// first making sure it hasn't been tampered with.
@@ -80,7 +81,7 @@ class Allocator {
8081
if (allocatorSupportsMemoryTagging<AllocatorConfig>())
8182
Ptr = untagPointer(Ptr);
8283
void *BlockBegin = Allocator::getBlockBegin(Ptr, &Header);
83-
Cache.deallocate(Header.ClassId, BlockBegin);
84+
SizeClassAllocator.deallocate(Header.ClassId, BlockBegin);
8485
}
8586

8687
// We take a shortcut when allocating a quarantine batch by working with the
@@ -89,7 +90,7 @@ class Allocator {
8990
void *allocate(UNUSED uptr Size) {
9091
const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
9192
sizeof(QuarantineBatch) + Chunk::getHeaderSize());
92-
void *Ptr = Cache.allocate(QuarantineClassId);
93+
void *Ptr = SizeClassAllocator.allocate(QuarantineClassId);
9394
// Quarantine batch allocation failure is fatal.
9495
if (UNLIKELY(!Ptr))
9596
reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
@@ -126,14 +127,15 @@ class Allocator {
126127

127128
Header.State = Chunk::State::Available;
128129
Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);
129-
Cache.deallocate(QuarantineClassId,
130-
reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
131-
Chunk::getHeaderSize()));
130+
SizeClassAllocator.deallocate(
131+
QuarantineClassId,
132+
reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
133+
Chunk::getHeaderSize()));
132134
}
133135

134136
private:
135137
ThisT &Allocator;
136-
CacheT &Cache;
138+
SizeClassAllocatorT &SizeClassAllocator;
137139
};
138140

139141
typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
@@ -263,7 +265,9 @@ class Allocator {
263265
QuarantineT *getQuarantine() { return &Quarantine; }
264266

265267
// The Cache must be provided zero-initialized.
266-
void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); }
268+
void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {
269+
SizeClassAllocator->init(&Stats, &Primary);
270+
}
267271

268272
// Release the resources used by a TSD, which involves:
269273
// - draining the local quarantine cache to the global quarantine;
@@ -273,15 +277,16 @@ class Allocator {
273277
void commitBack(TSD<ThisT> *TSD) {
274278
TSD->assertLocked(/*BypassCheck=*/true);
275279
Quarantine.drain(&TSD->getQuarantineCache(),
276-
QuarantineCallback(*this, TSD->getCache()));
277-
TSD->getCache().destroy(&Stats);
280+
QuarantineCallback(*this, TSD->getSizeClassAllocator()));
281+
TSD->getSizeClassAllocator().destroy(&Stats);
278282
}
279283

280284
void drainCache(TSD<ThisT> *TSD) {
281285
TSD->assertLocked(/*BypassCheck=*/true);
282-
Quarantine.drainAndRecycle(&TSD->getQuarantineCache(),
283-
QuarantineCallback(*this, TSD->getCache()));
284-
TSD->getCache().drain();
286+
Quarantine.drainAndRecycle(
287+
&TSD->getQuarantineCache(),
288+
QuarantineCallback(*this, TSD->getSizeClassAllocator()));
289+
TSD->getSizeClassAllocator().drain();
285290
}
286291
void drainCaches() { TSDRegistry.drainCaches(this); }
287292

@@ -390,13 +395,13 @@ class Allocator {
390395
ClassId = SizeClassMap::getClassIdBySize(NeededSize);
391396
DCHECK_NE(ClassId, 0U);
392397
typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
393-
Block = TSD->getCache().allocate(ClassId);
398+
Block = TSD->getSizeClassAllocator().allocate(ClassId);
394399
// If the allocation failed, retry in each successively larger class until
395400
// it fits. If it fails to fit in the largest class, fallback to the
396401
// Secondary.
397402
if (UNLIKELY(!Block)) {
398403
while (ClassId < SizeClassMap::LargestClassId && !Block)
399-
Block = TSD->getCache().allocate(++ClassId);
404+
Block = TSD->getSizeClassAllocator().allocate(++ClassId);
400405
if (!Block)
401406
ClassId = 0;
402407
}
@@ -1280,7 +1285,8 @@ class Allocator {
12801285
bool CacheDrained;
12811286
{
12821287
typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1283-
CacheDrained = TSD->getCache().deallocate(ClassId, BlockBegin);
1288+
CacheDrained =
1289+
TSD->getSizeClassAllocator().deallocate(ClassId, BlockBegin);
12841290
}
12851291
// When we have drained some blocks back to the Primary from TSD, that
12861292
// implies that we may have the chance to release some pages as well.
@@ -1296,7 +1302,8 @@ class Allocator {
12961302
retagBlock(Options, TaggedPtr, Ptr, Header, Size, false);
12971303
typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
12981304
Quarantine.put(&TSD->getQuarantineCache(),
1299-
QuarantineCallback(*this, TSD->getCache()), Ptr, Size);
1305+
QuarantineCallback(*this, TSD->getSizeClassAllocator()),
1306+
Ptr, Size);
13001307
}
13011308
}
13021309

‎compiler-rt/lib/scudo/standalone/primary32.h‎

Lines changed: 43 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@
1313
#include "bytemap.h"
1414
#include "common.h"
1515
#include "list.h"
16-
#include "local_cache.h"
1716
#include "options.h"
1817
#include "release.h"
1918
#include "report.h"
19+
#include "size_class_allocator.h"
2020
#include "stats.h"
2121
#include "string_utils.h"
2222
#include "thread_annotations.h"
@@ -52,7 +52,10 @@ template <typename Config> class SizeClassAllocator32 {
5252
static_assert((1UL << Config::getRegionSizeLog()) >= SizeClassMap::MaxSize,
5353
"");
5454
typedef SizeClassAllocator32<Config> ThisT;
55-
typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
55+
using SizeClassAllocatorT =
56+
typename Conditional<Config::getEnableBlockCache(),
57+
SizeClassAllocatorLocalCache<ThisT>,
58+
SizeClassAllocatorNoCache<ThisT>>::type;
5659
typedef TransferBatch<ThisT> TransferBatchT;
5760
typedef BatchGroup<ThisT> BatchGroupT;
5861

@@ -191,25 +194,28 @@ template <typename Config> class SizeClassAllocator32 {
191194
return BlockSize > PageSize;
192195
}
193196

194-
u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray,
195-
const u16 MaxBlockCount) {
197+
u16 popBlocks(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
198+
CompactPtrT *ToArray, const u16 MaxBlockCount) {
196199
DCHECK_LT(ClassId, NumClasses);
197200
SizeClassInfo *Sci = getSizeClassInfo(ClassId);
198201
ScopedLock L(Sci->Mutex);
199202

200-
u16 PopCount = popBlocksImpl(C, ClassId, Sci, ToArray, MaxBlockCount);
203+
u16 PopCount =
204+
popBlocksImpl(SizeClassAllocator, ClassId, Sci, ToArray, MaxBlockCount);
201205
if (UNLIKELY(PopCount == 0)) {
202-
if (UNLIKELY(!populateFreeList(C, ClassId, Sci)))
206+
if (UNLIKELY(!populateFreeList(SizeClassAllocator, ClassId, Sci)))
203207
return 0U;
204-
PopCount = popBlocksImpl(C, ClassId, Sci, ToArray, MaxBlockCount);
208+
PopCount = popBlocksImpl(SizeClassAllocator, ClassId, Sci, ToArray,
209+
MaxBlockCount);
205210
DCHECK_NE(PopCount, 0U);
206211
}
207212

208213
return PopCount;
209214
}
210215

211216
// Push the array of free blocks to the designated batch group.
212-
void pushBlocks(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size) {
217+
void pushBlocks(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
218+
CompactPtrT *Array, u32 Size) {
213219
DCHECK_LT(ClassId, NumClasses);
214220
DCHECK_GT(Size, 0);
215221

@@ -240,7 +246,7 @@ template <typename Config> class SizeClassAllocator32 {
240246
}
241247

242248
ScopedLock L(Sci->Mutex);
243-
pushBlocksImpl(C, ClassId, Sci, Array, Size, SameGroup);
249+
pushBlocksImpl(SizeClassAllocator, ClassId, Sci, Array, Size, SameGroup);
244250
}
245251

246252
void disable() NO_THREAD_SAFETY_ANALYSIS {
@@ -529,8 +535,8 @@ template <typename Config> class SizeClassAllocator32 {
529535
// memory group here.
530536
BG->CompactPtrGroupBase = 0;
531537
BG->BytesInBGAtLastCheckpoint = 0;
532-
BG->MaxCachedPerBatch =
533-
CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId));
538+
BG->MaxCachedPerBatch = SizeClassAllocatorT::getMaxCached(
539+
getSizeByClassId(SizeClassMap::BatchClassId));
534540

535541
Sci->FreeListInfo.BlockList.push_front(BG);
536542
}
@@ -597,18 +603,18 @@ template <typename Config> class SizeClassAllocator32 {
597603
// same group then we will skip checking the group id of each block.
598604
//
599605
// The region mutex needs to be held while calling this method.
600-
void pushBlocksImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci,
601-
CompactPtrT *Array, u32 Size, bool SameGroup = false)
602-
REQUIRES(Sci->Mutex) {
606+
void pushBlocksImpl(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
607+
SizeClassInfo *Sci, CompactPtrT *Array, u32 Size,
608+
bool SameGroup = false) REQUIRES(Sci->Mutex) {
603609
DCHECK_NE(ClassId, SizeClassMap::BatchClassId);
604610
DCHECK_GT(Size, 0U);
605611

606612
auto CreateGroup = [&](uptr CompactPtrGroupBase) {
607-
BatchGroupT *BG =
608-
reinterpret_cast<BatchGroupT *>(C->getBatchClassBlock());
613+
BatchGroupT *BG = reinterpret_cast<BatchGroupT *>(
614+
SizeClassAllocator->getBatchClassBlock());
609615
BG->Batches.clear();
610-
TransferBatchT *TB =
611-
reinterpret_cast<TransferBatchT *>(C->getBatchClassBlock());
616+
TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(
617+
SizeClassAllocator->getBatchClassBlock());
612618
TB->clear();
613619

614620
BG->CompactPtrGroupBase = CompactPtrGroupBase;
@@ -629,8 +635,8 @@ template <typename Config> class SizeClassAllocator32 {
629635
u16 UnusedSlots =
630636
static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
631637
if (UnusedSlots == 0) {
632-
CurBatch =
633-
reinterpret_cast<TransferBatchT *>(C->getBatchClassBlock());
638+
CurBatch = reinterpret_cast<TransferBatchT *>(
639+
SizeClassAllocator->getBatchClassBlock());
634640
CurBatch->clear();
635641
Batches.push_front(CurBatch);
636642
UnusedSlots = BG->MaxCachedPerBatch;
@@ -704,9 +710,9 @@ template <typename Config> class SizeClassAllocator32 {
704710
InsertBlocks(Cur, Array + Size - Count, Count);
705711
}
706712

707-
u16 popBlocksImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci,
708-
CompactPtrT *ToArray, const u16 MaxBlockCount)
709-
REQUIRES(Sci->Mutex) {
713+
u16 popBlocksImpl(SizeClassAllocatorT *SizeClassAllocator, uptr ClassId,
714+
SizeClassInfo *Sci, CompactPtrT *ToArray,
715+
const u16 MaxBlockCount) REQUIRES(Sci->Mutex) {
710716
if (Sci->FreeListInfo.BlockList.empty())
711717
return 0U;
712718

@@ -730,11 +736,11 @@ template <typename Config> class SizeClassAllocator32 {
730736
// So far, instead of always filling the blocks to `MaxBlockCount`, we only
731737
// examine single `TransferBatch` to minimize the time spent on the primary
732738
// allocator. Besides, the sizes of `TransferBatch` and
733-
// `CacheT::getMaxCached()` may also impact the time spent on accessing the
734-
// primary allocator.
739+
// `SizeClassAllocatorT::getMaxCached()` may also impact the time spent on
740+
// accessing the primary allocator.
735741
// TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
736742
// blocks and/or adjust the size of `TransferBatch` according to
737-
// `CacheT::getMaxCached()`.
743+
// `SizeClassAllocatorT::getMaxCached()`.
738744
TransferBatchT *B = Batches.front();
739745
DCHECK_NE(B, nullptr);
740746
DCHECK_GT(B->getCount(), 0U);
@@ -754,7 +760,7 @@ template <typename Config> class SizeClassAllocator32 {
754760
// deallocate. Read the comment in `pushBatchClassBlocks()` for more
755761
// details.
756762
if (ClassId != SizeClassMap::BatchClassId)
757-
C->deallocate(SizeClassMap::BatchClassId, B);
763+
SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, B);
758764

759765
if (Batches.empty()) {
760766
BatchGroupT *BG = Sci->FreeListInfo.BlockList.front();
@@ -766,15 +772,16 @@ template <typename Config> class SizeClassAllocator32 {
766772
// Which means, once we pop the last TransferBatch, the block is
767773
// implicitly deallocated.
768774
if (ClassId != SizeClassMap::BatchClassId)
769-
C->deallocate(SizeClassMap::BatchClassId, BG);
775+
SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, BG);
770776
}
771777
}
772778

773779
Sci->FreeListInfo.PoppedBlocks += PopCount;
774780
return PopCount;
775781
}
776782

777-
NOINLINE bool populateFreeList(CacheT *C, uptr ClassId, SizeClassInfo *Sci)
783+
NOINLINE bool populateFreeList(SizeClassAllocatorT *SizeClassAllocator,
784+
uptr ClassId, SizeClassInfo *Sci)
778785
REQUIRES(Sci->Mutex) {
779786
uptr Region;
780787
uptr Offset;
@@ -791,13 +798,13 @@ template <typename Config> class SizeClassAllocator32 {
791798
Region = allocateRegion(Sci, ClassId);
792799
if (UNLIKELY(!Region))
793800
return false;
794-
C->getStats().add(StatMapped, RegionSize);
801+
SizeClassAllocator->getStats().add(StatMapped, RegionSize);
795802
Sci->CurrentRegion = Region;
796803
Offset = 0;
797804
}
798805

799806
const uptr Size = getSizeByClassId(ClassId);
800-
const u16 MaxCount = CacheT::getMaxCached(Size);
807+
const u16 MaxCount = SizeClassAllocatorT::getMaxCached(Size);
801808
DCHECK_GT(MaxCount, 0U);
802809
// The maximum number of blocks we should carve in the region is dictated
803810
// by the maximum number of batches we want to fill, and the amount of
@@ -827,7 +834,8 @@ template <typename Config> class SizeClassAllocator32 {
827834
for (u32 I = 1; I < NumberOfBlocks; I++) {
828835
if (UNLIKELY(compactPtrGroupBase(ShuffleArray[I]) != CurGroup)) {
829836
shuffle(ShuffleArray + I - N, N, &Sci->RandState);
830-
pushBlocksImpl(C, ClassId, Sci, ShuffleArray + I - N, N,
837+
pushBlocksImpl(SizeClassAllocator, ClassId, Sci, ShuffleArray + I - N,
838+
N,
831839
/*SameGroup=*/true);
832840
N = 1;
833841
CurGroup = compactPtrGroupBase(ShuffleArray[I]);
@@ -837,7 +845,8 @@ template <typename Config> class SizeClassAllocator32 {
837845
}
838846

839847
shuffle(ShuffleArray + NumberOfBlocks - N, N, &Sci->RandState);
840-
pushBlocksImpl(C, ClassId, Sci, &ShuffleArray[NumberOfBlocks - N], N,
848+
pushBlocksImpl(SizeClassAllocator, ClassId, Sci,
849+
&ShuffleArray[NumberOfBlocks - N], N,
841850
/*SameGroup=*/true);
842851
} else {
843852
pushBatchClassBlocks(Sci, ShuffleArray, NumberOfBlocks);
@@ -850,7 +859,7 @@ template <typename Config> class SizeClassAllocator32 {
850859
Sci->FreeListInfo.PushedBlocks -= NumberOfBlocks;
851860

852861
const uptr AllocatedUser = Size * NumberOfBlocks;
853-
C->getStats().add(StatFree, AllocatedUser);
862+
SizeClassAllocator->getStats().add(StatFree, AllocatedUser);
854863
DCHECK_LE(Sci->CurrentRegionAllocated + AllocatedUser, RegionSize);
855864
// If there is not enough room in the region currently associated to fit
856865
// more blocks, we deassociate the region by resetting CurrentRegion and

0 commit comments

Comments
 (0)