diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.def b/compiler-rt/lib/scudo/standalone/allocator_config.def index 84fcec0877d40..748530820cd64 100644 --- a/compiler-rt/lib/scudo/standalone/allocator_config.def +++ b/compiler-rt/lib/scudo/standalone/allocator_config.def @@ -54,6 +54,9 @@ BASE_REQUIRED_TEMPLATE_TYPE(SecondaryT) // Indicates possible support for Memory Tagging. BASE_OPTIONAL(const bool, MaySupportMemoryTagging, false) +// Disable the quarantine code. +BASE_OPTIONAL(const bool, QuarantineDisabled, false) + // PRIMARY_REQUIRED_TYPE(NAME) // // SizeClassMap to use with the Primary. diff --git a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h index ac639ee9dd943..5bfa700c2f8d8 100644 --- a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h @@ -60,6 +60,10 @@ template struct PrimaryConfig { return BaseConfig::getMaySupportMemoryTagging(); } + static constexpr bool getQuarantineDisabled() { + return BaseConfig::getQuarantineDisabled(); + } + #define PRIMARY_REQUIRED_TYPE(NAME) \ using NAME = typename AllocatorConfig::Primary::NAME; @@ -92,6 +96,10 @@ template struct SecondaryConfig { return BaseConfig::getMaySupportMemoryTagging(); } + static constexpr bool getQuarantineDisabled() { + return BaseConfig::getQuarantineDisabled(); + } + #define SECONDARY_REQUIRED_TEMPLATE_TYPE(NAME) \ template \ using NAME = typename AllocatorConfig::Secondary::template NAME; @@ -111,6 +119,10 @@ template struct SecondaryConfig { return BaseConfig::getMaySupportMemoryTagging(); } + static constexpr bool getQuarantineDisabled() { + return BaseConfig::getQuarantineDisabled(); + } + #define SECONDARY_CACHE_OPTIONAL(TYPE, NAME, DEFAULT) \ OPTIONAL_TEMPLATE(TYPE, NAME, DEFAULT, Cache::NAME) \ static constexpr removeConst::type get##NAME() { \ diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index 87acdec2a3bac..48a30ad0306ec 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -184,9 +184,10 @@ class Allocator { const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms; Primary.init(ReleaseToOsIntervalMs); Secondary.init(&Stats, ReleaseToOsIntervalMs); - Quarantine.init( - static_cast(getFlags()->quarantine_size_kb << 10), - static_cast(getFlags()->thread_local_quarantine_size_kb << 10)); + if (!AllocatorConfig::getQuarantineDisabled()) + Quarantine.init( + static_cast(getFlags()->quarantine_size_kb << 10), + static_cast(getFlags()->thread_local_quarantine_size_kb << 10)); } void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS { @@ -276,16 +277,18 @@ class Allocator { // the last two items). void commitBack(TSD *TSD) { TSD->assertLocked(/*BypassCheck=*/true); - Quarantine.drain(&TSD->getQuarantineCache(), - QuarantineCallback(*this, TSD->getSizeClassAllocator())); + if (!AllocatorConfig::getQuarantineDisabled()) + Quarantine.drain(&TSD->getQuarantineCache(), + QuarantineCallback(*this, TSD->getSizeClassAllocator())); TSD->getSizeClassAllocator().destroy(&Stats); } void drainCache(TSD *TSD) { TSD->assertLocked(/*BypassCheck=*/true); - Quarantine.drainAndRecycle( - &TSD->getQuarantineCache(), - QuarantineCallback(*this, TSD->getSizeClassAllocator())); + if (!AllocatorConfig::getQuarantineDisabled()) + Quarantine.drainAndRecycle( + &TSD->getQuarantineCache(), + QuarantineCallback(*this, TSD->getSizeClassAllocator())); TSD->getSizeClassAllocator().drain(); } void drainCaches() { TSDRegistry.drainCaches(this); } @@ -612,7 +615,8 @@ class Allocator { #endif TSDRegistry.disable(); Stats.disable(); - Quarantine.disable(); + if (!AllocatorConfig::getQuarantineDisabled()) + Quarantine.disable(); Primary.disable(); Secondary.disable(); disableRingBuffer(); @@ -623,7 +627,8 @@ class Allocator { enableRingBuffer(); Secondary.enable(); Primary.enable(); - Quarantine.enable(); + if (!AllocatorConfig::getQuarantineDisabled()) + Quarantine.enable(); Stats.enable(); TSDRegistry.enable(); #ifdef GWP_ASAN_HOOKS @@ -1252,7 +1257,8 @@ class Allocator { // If the quarantine is disabled, the actual size of a chunk is 0 or larger // than the maximum allowed, we return a chunk directly to the backend. // This purposefully underflows for Size == 0. - const bool BypassQuarantine = !Quarantine.getCacheSize() || + const bool BypassQuarantine = AllocatorConfig::getQuarantineDisabled() || + !Quarantine.getCacheSize() || ((Size - 1) >= QuarantineMaxChunkSize) || !Header->ClassId; if (BypassQuarantine) @@ -1642,7 +1648,8 @@ class Allocator { uptr getStats(ScopedString *Str) { Primary.getStats(Str); Secondary.getStats(Str); - Quarantine.getStats(Str); + if (!AllocatorConfig::getQuarantineDisabled()) + Quarantine.getStats(Str); TSDRegistry.getStats(Str); return Str->length(); } diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h index 286e5d332f57c..f04c5b7cfc2ea 100644 --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -269,7 +269,8 @@ class MapAllocatorCache { Entry.MemMap = MemMap; Entry.Time = UINT64_MAX; - if (useMemoryTagging(Options)) { + bool MemoryTaggingEnabled = useMemoryTagging(Options); + if (MemoryTaggingEnabled) { if (Interval == 0 && !SCUDO_FUCHSIA) { // Release the memory and make it inaccessible at the same time by // creating a new MAP_NOACCESS mapping on top of the existing mapping. @@ -302,7 +303,7 @@ class MapAllocatorCache { if (Entry.Time != 0) Entry.Time = Time; - if (useMemoryTagging(Options) && QuarantinePos == -1U) { + if (MemoryTaggingEnabled && !useMemoryTagging(Options)) { // If we get here then memory tagging was disabled in between when we // read Options and when we locked Mutex. We can't insert our entry into // the quarantine or the cache because the permissions would be wrong so @@ -310,7 +311,8 @@ class MapAllocatorCache { unmapCallBack(Entry.MemMap); break; } - if (Config::getQuarantineSize() && useMemoryTagging(Options)) { + + if (Config::getQuarantineSize()) { QuarantinePos = (QuarantinePos + 1) % Max(Config::getQuarantineSize(), 1u); if (!Quarantine[QuarantinePos].isValid()) { @@ -513,9 +515,10 @@ class MapAllocatorCache { Quarantine[I].invalidate(); } } + QuarantinePos = -1U; + for (CachedBlock &Entry : LRUEntries) Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0); - QuarantinePos = -1U; } void disable() NO_THREAD_SAFETY_ANALYSIS { Mutex.lock(); } diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp index 7e8d5b4396d2e..8b55c6570650e 100644 --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -623,20 +623,20 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DisableMemoryTagging) { SCUDO_TYPED_TEST(ScudoCombinedTest, Stats) { auto *Allocator = this->Allocator.get(); - scudo::uptr BufferSize = 8192; - std::vector Buffer(BufferSize); - scudo::uptr ActualSize = Allocator->getStats(Buffer.data(), BufferSize); - while (ActualSize > BufferSize) { - BufferSize = ActualSize + 1024; - Buffer.resize(BufferSize); - ActualSize = Allocator->getStats(Buffer.data(), BufferSize); + std::string Stats(10000, '\0'); + scudo::uptr ActualSize = Allocator->getStats(Stats.data(), Stats.size()); + if (ActualSize > Stats.size()) { + Stats.resize(ActualSize); + ActualSize = Allocator->getStats(Stats.data(), Stats.size()); } - std::string Stats(Buffer.begin(), Buffer.end()); + EXPECT_GE(Stats.size(), ActualSize); + // Basic checks on the contents of the statistics output, which also allows us // to verify that we got it all. EXPECT_NE(Stats.find("Stats: SizeClassAllocator"), std::string::npos); EXPECT_NE(Stats.find("Stats: MapAllocator"), std::string::npos); - EXPECT_NE(Stats.find("Stats: Quarantine"), std::string::npos); + // Do not explicitly check for quarantine stats since a config can disable + // them. Other tests verify this (QuarantineEnabled/QuarantineDisabled). } SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, Drain) { @@ -1076,3 +1076,107 @@ TEST(ScudoCombinedTest, BasicTrustyConfig) { #endif #endif + +struct TestQuarantineSizeClassConfig { + static const scudo::uptr NumBits = 1; + static const scudo::uptr MinSizeLog = 10; + static const scudo::uptr MidSizeLog = 10; + static const scudo::uptr MaxSizeLog = 13; + static const scudo::u16 MaxNumCachedHint = 8; + static const scudo::uptr MaxBytesCachedLog = 12; + static const scudo::uptr SizeDelta = 0; +}; + +struct TestQuarantineConfig { + static const bool MaySupportMemoryTagging = false; + + template using TSDRegistryT = scudo::TSDRegistrySharedT; + + struct Primary { + // Tiny allocator, its Primary only serves chunks of four sizes. + using SizeClassMap = scudo::FixedSizeClassMap; + static const scudo::uptr RegionSizeLog = DeathRegionSizeLog; + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::uptr CompactPtrT; + static const scudo::uptr CompactPtrScale = 0; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + static const scudo::uptr GroupSizeLog = 18; + }; + template + using PrimaryT = scudo::SizeClassAllocator64; + + struct Secondary { + template + using CacheT = scudo::MapAllocatorNoCache; + }; + + template using SecondaryT = scudo::MapAllocator; +}; + +TEST(ScudoCombinedTest, QuarantineEnabled) { + // Enable quarantine options. + EXPECT_EQ(0, setenv("SCUDO_OPTIONS", + "quarantine_max_chunk_size=65535:quarantine_size_kb=500:" + "thread_local_quarantine_size_kb=600", + true)); + + using AllocatorT = scudo::Allocator; + auto Allocator = std::unique_ptr(new AllocatorT()); + + void *P = Allocator->allocate(100U, Origin); + EXPECT_NE(P, nullptr); + Allocator->deallocate(P, Origin); + + std::string Stats(10000, '\0'); + scudo::uptr ActualSize = Allocator->getStats(Stats.data(), Stats.size()); + if (ActualSize > Stats.size()) { + Stats.resize(ActualSize); + ActualSize = Allocator->getStats(Stats.data(), Stats.size()); + } + EXPECT_GE(Stats.size(), ActualSize); + + // Verify we get quarantine stats. + EXPECT_NE(Stats.find("Stats: Quarantine"), std::string::npos); + // Verify that quarantine values get set properly. + EXPECT_NE(Stats.find("Quarantine limits: global: 500K; thread local: 600K"), + std::string::npos); + + // Clear the options for all other tests. + EXPECT_EQ(0, setenv("SCUDO_OPTIONS", "", true)); +} + +struct TestQuarantineDisabledConfig : TestQuarantineConfig { + static const bool QuarantineDisabled = true; +}; + +TEST(ScudoCombinedTest, QuarantineDisabled) { + // Enable quarantine options. + EXPECT_EQ(0, setenv("SCUDO_OPTIONS", + "quarantine_max_chunk_size=65535:quarantine_size_kb=500:" + "thread_local_quarantine_size_kb=600", + true)); + + using AllocatorT = scudo::Allocator; + auto Allocator = std::unique_ptr(new AllocatorT()); + + const scudo::uptr Size = 1000U; + void *P = Allocator->allocate(Size, Origin); + EXPECT_NE(P, nullptr); + Allocator->deallocate(P, Origin); + + std::string Stats(10000, '\0'); + scudo::uptr ActualSize = Allocator->getStats(Stats.data(), Stats.size()); + if (ActualSize > Stats.size()) { + Stats.resize(ActualSize); + ActualSize = Allocator->getStats(Stats.data(), Stats.size()); + } + EXPECT_GE(Stats.size(), ActualSize); + + // No quarantine stats should exist because the code is not enabled. + EXPECT_EQ(Stats.find("Stats: Quarantine"), std::string::npos); + + // Clear the options for all other tests. + EXPECT_EQ(0, setenv("SCUDO_OPTIONS", "", true)); +}