[scudo] Add a method to use a hard-coded page size (#106646)

Currently, only Android supports using a hard-code page size. Make this
a bit more generic so any platform that wants to can use this.

In addition, add a getPageSizeLogCached() function since this value is
used in release.h and can avoid keeping this value around in objects.

Finally, change some of the release.h page size multiplies to shifts
using the new page size log value.

Author: cferris1000

compiler-rt/lib/scudo/standalone/combined.h

@@ -140,6 +140,9 @@ class Allocator {
   typedef typename QuarantineT::CacheT QuarantineCacheT;
 
   void init() {
+    // Make sure that the page size is initialized if it's not a constant.
+    CHECK_NE(getPageSizeCached(), 0U);
+
     performSanityChecks();
 
     // Check if hardware CRC32 is supported in the binary and by the platform,

compiler-rt/lib/scudo/standalone/common.cpp

@@ -12,13 +12,21 @@
 
 namespace scudo {
 
-uptr PageSizeCached;
+#if !defined(SCUDO_PAGE_SIZE)
+uptr PageSizeCached = 0;
+uptr PageSizeLogCached = 0;
+
+// Must be defined in platform specific code.
 uptr getPageSize();
 
+// This must be called in the init path or there could be a race if multiple
+// threads try to set the cached values.
 uptr getPageSizeSlow() {
   PageSizeCached = getPageSize();
   CHECK_NE(PageSizeCached, 0);
+  PageSizeLogCached = getLog2(PageSizeCached);
   return PageSizeCached;
 }
+#endif
 
 } // namespace scudo

compiler-rt/lib/scudo/standalone/common.h

@@ -133,18 +133,40 @@ inline void computePercentage(uptr Numerator, uptr Denominator, uptr *Integral,
 
 // Platform specific functions.
 
+#if defined(SCUDO_PAGE_SIZE)
+
+inline constexpr uptr getPageSizeCached() { return SCUDO_PAGE_SIZE; }
+
+inline constexpr uptr getPageSizeSlow() { return getPageSizeCached(); }
+
+inline constexpr uptr getPageSizeLogCached() {
+  return static_cast<uptr>(__builtin_ctzl(SCUDO_PAGE_SIZE));
+}
+
+#else
+
 extern uptr PageSizeCached;
+extern uptr PageSizeLogCached;
+
 uptr getPageSizeSlow();
+
 inline uptr getPageSizeCached() {
-#if SCUDO_ANDROID && defined(PAGE_SIZE)
-  // Most Android builds have a build-time constant page size.
-  return PAGE_SIZE;
-#endif
   if (LIKELY(PageSizeCached))
     return PageSizeCached;
   return getPageSizeSlow();
 }
 
+inline uptr getPageSizeLogCached() {
+  if (LIKELY(PageSizeLogCached))
+    return PageSizeLogCached;
+  // PageSizeLogCached and PageSizeCached are both set in getPageSizeSlow()
+  getPageSizeSlow();
+  DCHECK_NE(PageSizeLogCached, 0);
+  return PageSizeLogCached;
+}
+
+#endif
+
 // Returns 0 if the number of CPUs could not be determined.
 u32 getNumberOfCPUs();
 

compiler-rt/lib/scudo/standalone/linux.cpp

@@ -40,7 +40,10 @@
 
 namespace scudo {
 
+#if !defined(SCUDO_PAGE_SIZE)
+// This function is only used when page size is not hard-coded.
 uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); }
+#endif
 
 void NORETURN die() { abort(); }
 

compiler-rt/lib/scudo/standalone/platform.h

@@ -21,6 +21,11 @@
 // See https://android.googlesource.com/platform/bionic/+/master/docs/defines.md
 #if defined(__BIONIC__)
 #define SCUDO_ANDROID 1
+// Transitive includes of unistd.h will get PAGE_SIZE if it is defined.
+#include <unistd.h>
+#if defined(PAGE_SIZE)
+#define SCUDO_PAGE_SIZE PAGE_SIZE
+#endif
 #else
 #define SCUDO_ANDROID 0
 #endif

compiler-rt/lib/scudo/standalone/release.h

@@ -88,7 +88,7 @@ class FragmentationRecorder {
 
   void releasePageRangeToOS(uptr From, uptr To) {
     DCHECK_EQ((To - From) % getPageSizeCached(), 0U);
-    ReleasedPagesCount += (To - From) / getPageSizeCached();
+    ReleasedPagesCount += (To - From) >> getPageSizeLogCached();
   }
 
 private:
@@ -348,7 +348,7 @@ class RegionPageMap {
 template <class ReleaseRecorderT> class FreePagesRangeTracker {
 public:
   explicit FreePagesRangeTracker(ReleaseRecorderT &Recorder)
-      : Recorder(Recorder), PageSizeLog(getLog2(getPageSizeCached())) {}
+      : Recorder(Recorder) {}
 
   void processNextPage(bool Released) {
     if (Released) {
@@ -372,14 +372,14 @@ template <class ReleaseRecorderT> class FreePagesRangeTracker {
 private:
   void closeOpenedRange() {
     if (InRange) {
+      const uptr PageSizeLog = getPageSizeLogCached();
       Recorder.releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog),
                                     (CurrentPage << PageSizeLog));
       InRange = false;
     }
   }
 
   ReleaseRecorderT &Recorder;
-  const uptr PageSizeLog;
   bool InRange = false;
   uptr CurrentPage = 0;
   uptr CurrentRangeStatePage = 0;
@@ -389,7 +389,7 @@ struct PageReleaseContext {
   PageReleaseContext(uptr BlockSize, uptr NumberOfRegions, uptr ReleaseSize,
                      uptr ReleaseOffset = 0)
       : BlockSize(BlockSize), NumberOfRegions(NumberOfRegions) {
-    PageSize = getPageSizeCached();
+    const uptr PageSize = getPageSizeCached();
     if (BlockSize <= PageSize) {
       if (PageSize % BlockSize == 0) {
         // Same number of chunks per page, no cross overs.
@@ -408,7 +408,7 @@ struct PageReleaseContext {
         SameBlockCountPerPage = false;
       }
     } else {
-      if (BlockSize % PageSize == 0) {
+      if ((BlockSize & (PageSize - 1)) == 0) {
         // One chunk covers multiple pages, no cross overs.
         FullPagesBlockCountMax = 1;
         SameBlockCountPerPage = true;
@@ -427,8 +427,8 @@ struct PageReleaseContext {
     if (NumberOfRegions != 1)
       DCHECK_EQ(ReleaseOffset, 0U);
 
-    PagesCount = roundUp(ReleaseSize, PageSize) / PageSize;
-    PageSizeLog = getLog2(PageSize);
+    const uptr PageSizeLog = getPageSizeLogCached();
+    PagesCount = roundUp(ReleaseSize, PageSize) >> PageSizeLog;
     ReleasePageOffset = ReleaseOffset >> PageSizeLog;
   }
 
@@ -451,6 +451,7 @@ struct PageReleaseContext {
   // RegionSize, it's not necessary to be aligned with page size.
   bool markRangeAsAllCounted(uptr From, uptr To, uptr Base,
                              const uptr RegionIndex, const uptr RegionSize) {
+    const uptr PageSize = getPageSizeCached();
     DCHECK_LT(From, To);
     DCHECK_LE(To, Base + RegionSize);
     DCHECK_EQ(From % PageSize, 0U);
@@ -544,6 +545,7 @@ struct PageReleaseContext {
     if (!ensurePageMapAllocated())
       return false;
 
+    const uptr PageSize = getPageSizeCached();
     if (MayContainLastBlockInRegion) {
       const uptr LastBlockInRegion =
           ((RegionSize / BlockSize) - 1U) * BlockSize;
@@ -605,17 +607,19 @@ struct PageReleaseContext {
     return true;
   }
 
-  uptr getPageIndex(uptr P) { return (P >> PageSizeLog) - ReleasePageOffset; }
-  uptr getReleaseOffset() { return ReleasePageOffset << PageSizeLog; }
+  uptr getPageIndex(uptr P) {
+    return (P >> getPageSizeLogCached()) - ReleasePageOffset;
+  }
+  uptr getReleaseOffset() {
+    return ReleasePageOffset << getPageSizeLogCached();
+  }
 
   uptr BlockSize;
   uptr NumberOfRegions;
   // For partial region marking, some pages in front are not needed to be
   // counted.
   uptr ReleasePageOffset;
-  uptr PageSize;
   uptr PagesCount;
-  uptr PageSizeLog;
   uptr FullPagesBlockCountMax;
   bool SameBlockCountPerPage;
   RegionPageMap PageMap;
@@ -628,7 +632,7 @@ template <class ReleaseRecorderT, typename SkipRegionT>
 NOINLINE void
 releaseFreeMemoryToOS(PageReleaseContext &Context,
                       ReleaseRecorderT &Recorder, SkipRegionT SkipRegion) {
-  const uptr PageSize = Context.PageSize;
+  const uptr PageSize = getPageSizeCached();
   const uptr BlockSize = Context.BlockSize;
   const uptr PagesCount = Context.PagesCount;
   const uptr NumberOfRegions = Context.NumberOfRegions;
@@ -671,7 +675,7 @@ releaseFreeMemoryToOS(PageReleaseContext &Context,
       uptr PrevPageBoundary = 0;
       uptr CurrentBoundary = 0;
       if (ReleasePageOffset > 0) {
-        PrevPageBoundary = ReleasePageOffset * PageSize;
+        PrevPageBoundary = ReleasePageOffset << getPageSizeLogCached();
         CurrentBoundary = roundUpSlow(PrevPageBoundary, BlockSize);
       }
       for (uptr J = 0; J < PagesCount; J++) {