Port to Mac OS X + Apple Silicon (16K pages).

Author: emeryberger

.bazelrc

@@ -16,5 +16,5 @@ build:disable-meshing --define disable_meshing=true
 build:disable-randomization --define disable_randomization=true
 build:shuffle-on-free --define shuffle_on_free=true
 
-build:modern-amd64 --copt=-mavx --copt=-march=westmere
-build:modern-amd64 --linkopt=-mavx --linkopt=-march=westmere
+build:modern-amd64 --copt=-mavx # --copt=-march=westmere
+build:modern-amd64 --linkopt=-mavx # --linkopt=-march=westmere

Makefile

@@ -3,10 +3,21 @@
 # Version 2.0, that can be found in the LICENSE file.
 
 PREFIX       = /usr
-BAZEL_CONFIG = --config=modern-amd64
 LIB_SUFFIX   =
 
 UNAME_S = $(shell uname -s)
+UNAME_M = $(shell uname -m)
+
+# Set BAZEL_CONFIG based on architecture
+ifeq ($(UNAME_M),x86_64)
+BAZEL_CONFIG = --config=modern-amd64
+else ifeq ($(UNAME_M),amd64)
+BAZEL_CONFIG = --config=modern-amd64
+else
+# ARM64 and other architectures - no special flags
+BAZEL_CONFIG =
+endif
+
 ifeq ($(UNAME_S),Darwin)
 LIB_EXT      = dylib
 BAZEL_PREFIX = darwin

src/BUILD

@@ -194,6 +194,7 @@ cc_library(
 
 cc_test(
     name = "unit-tests",
+    size = "small",  # Test runs in ~2.5s
     srcs = glob([
         "testing/unit/*.cc",
     ]),

src/bitmap.h

@@ -44,8 +44,15 @@ using std::atomic_size_t;
 
 // enables iteration through the set bits of the bitmap
 template <typename Container>
-class BitmapIter : public std::iterator<std::forward_iterator_tag, size_t> {
+class BitmapIter {
 public:
+  // Iterator traits (modern C++17 approach, not inheriting from deprecated std::iterator)
+  using iterator_category = std::forward_iterator_tag;
+  using value_type = size_t;
+  using difference_type = std::ptrdiff_t;
+  using pointer = size_t*;
+  using reference = size_t&;
+
   BitmapIter(const Container &a, const size_t i) : _i(i), _cont(a) {
   }
   BitmapIter &operator++() {
@@ -86,28 +93,22 @@ class AtomicBitmapBase {
   AtomicBitmapBase(size_t bitCount) {
     d_assert_msg(bitCount <= maxBits, "max bits (%zu) exceeded: %zu", maxBits, bitCount);
 
-    static_assert(wordCount(representationSize(maxBits)) == 4, "unexpected representation size");
-    // for (size_t i = 0; i < wordCount(representationSize(maxBits)); i++) {
-    //   _bits[i].store(0, std::memory_order_relaxed);
-    // }
-    _bits[0].store(0, std::memory_order_relaxed);
-    _bits[1].store(0, std::memory_order_relaxed);
-    _bits[2].store(0, std::memory_order_relaxed);
-    _bits[3].store(0, std::memory_order_relaxed);
+    // Initialize all words to 0
+    constexpr size_t wordCnt = wordCount(representationSize(maxBits));
+    for (size_t i = 0; i < wordCnt; i++) {
+      _bits[i].store(0, std::memory_order_relaxed);
+    }
     std::atomic_thread_fence(std::memory_order_release);
   }
 
   ~AtomicBitmapBase() {
   }
 
   inline void ATTRIBUTE_ALWAYS_INLINE setAndExchangeAll(size_t *oldBits, const size_t *newBits) {
-    // for (size_t i = 0; i < wordCount(representationSize(maxBits)); i++) {
-    //   oldBits[i] = _bits[i].exchange(newBits[i]);
-    // }
-    oldBits[0] = _bits[0].exchange(newBits[0], std::memory_order_acq_rel);
-    oldBits[1] = _bits[1].exchange(newBits[1], std::memory_order_acq_rel);
-    oldBits[2] = _bits[2].exchange(newBits[2], std::memory_order_acq_rel);
-    oldBits[3] = _bits[3].exchange(newBits[3], std::memory_order_acq_rel);
+    constexpr size_t wordCnt = wordCount(representationSize(maxBits));
+    for (size_t i = 0; i < wordCnt; i++) {
+      oldBits[i] = _bits[i].exchange(newBits[i], std::memory_order_acq_rel);
+    }
   }
 
 public:
@@ -140,8 +141,12 @@ class AtomicBitmapBase {
   }
 
   inline uint32_t ATTRIBUTE_ALWAYS_INLINE inUseCount() const {
-    return __builtin_popcountl(_bits[0]) + __builtin_popcountl(_bits[1]) + __builtin_popcountl(_bits[2]) +
-           __builtin_popcountl(_bits[3]);
+    constexpr size_t wordCnt = wordCount(representationSize(maxBits));
+    uint32_t count = 0;
+    for (size_t i = 0; i < wordCnt; i++) {
+      count += __builtin_popcountl(_bits[i].load(std::memory_order_relaxed));
+    }
+    return count;
   }
 
 protected:
@@ -273,14 +278,10 @@ class RelaxedFixedBitmapBase {
 
 public:
   inline void ATTRIBUTE_ALWAYS_INLINE invert() {
-    // constexpr size_t numWords = wordCount(representationSize(maxBits));
-    // for (size_t i = 0; i < numWords; i++) {
-    //   _bits[i] = ~_bits[i];
-    // }
-    _bits[0] = ~_bits[0];
-    _bits[1] = ~_bits[1];
-    _bits[2] = ~_bits[2];
-    _bits[3] = ~_bits[3];
+    constexpr size_t numWords = wordCount(representationSize(maxBits));
+    for (size_t i = 0; i < numWords; i++) {
+      _bits[i] = ~_bits[i];
+    }
   }
 
   inline void ATTRIBUTE_ALWAYS_INLINE setAll(uint64_t bitCount) {
@@ -315,15 +316,11 @@ class RelaxedFixedBitmapBase {
   }
 
   inline uint64_t ATTRIBUTE_ALWAYS_INLINE inUseCount() const {
-    constexpr auto wordCount = representationSize(maxBits) / sizeof(size_t);
+    constexpr auto wordCnt = representationSize(maxBits) / sizeof(size_t);
     uint32_t count = 0;
-    // for (size_t i = 0; i < wordCount; i++) {
-    //   count += __builtin_popcountl(_bits[i]);
-    // }
-    count += __builtin_popcountl(_bits[0]);
-    count += __builtin_popcountl(_bits[1]);
-    count += __builtin_popcountl(_bits[2]);
-    count += __builtin_popcountl(_bits[3]);
+    for (size_t i = 0; i < wordCnt; i++) {
+      count += __builtin_popcountl(_bits[i]);
+    }
     return count;
   }
 
@@ -332,10 +329,10 @@ class RelaxedFixedBitmapBase {
   }
 
   void ATTRIBUTE_ALWAYS_INLINE clear() {
-    _bits[0] = 0;
-    _bits[1] = 0;
-    _bits[2] = 0;
-    _bits[3] = 0;
+    constexpr size_t wordCnt = wordCount(representationSize(maxBits));
+    for (size_t i = 0; i < wordCnt; i++) {
+      _bits[i] = 0;
+    }
   }
 
   inline size_t ATTRIBUTE_ALWAYS_INLINE bitCount() const {
@@ -477,11 +474,15 @@ class BitmapBase : public Super {
     return Super::unsetAt(item, position);
   }
 
-  // FIXME: who uses this? bad idea with atomics
+  // NOTE: This is not fully atomic-safe, but only used in debug assertions
+  // For RelaxedBitmapBase (non-atomic), direct access is fine
+  // For AtomicBitmapBase, this creates a potential race, but acceptable for debug
   inline bool ATTRIBUTE_ALWAYS_INLINE isSet(uint64_t index) const {
     uint32_t item, position;
     computeItemPosition(index, item, position);
 
+    // Use direct access for both atomic and non-atomic variants
+    // This works because on x86/ARM, reading a size_t is atomic
     return Super::_bits[item] & getMask(position);
   }
 
@@ -577,12 +578,12 @@ class BitmapBase : public Super {
 }  // namespace bitmap
 
 namespace internal {
-typedef bitmap::BitmapBase<bitmap::AtomicBitmapBase<256>> Bitmap;
-typedef bitmap::BitmapBase<bitmap::RelaxedFixedBitmapBase<256>> RelaxedFixedBitmap;
+typedef bitmap::BitmapBase<bitmap::AtomicBitmapBase<1024>> Bitmap;
+typedef bitmap::BitmapBase<bitmap::RelaxedFixedBitmapBase<1024>> RelaxedFixedBitmap;
 typedef bitmap::BitmapBase<bitmap::RelaxedBitmapBase> RelaxedBitmap;
 
-static_assert(sizeof(Bitmap) == sizeof(size_t) * 4, "Bitmap unexpected size");
-static_assert(sizeof(RelaxedFixedBitmap) == sizeof(size_t) * 4, "Bitmap unexpected size");
+static_assert(sizeof(Bitmap) == sizeof(size_t) * 16, "Bitmap unexpected size (expected 128 bytes for 1024 bits)");
+static_assert(sizeof(RelaxedFixedBitmap) == sizeof(size_t) * 16, "Bitmap unexpected size (expected 128 bytes for 1024 bits)");
 static_assert(sizeof(RelaxedBitmap) == sizeof(size_t) * 2, "Bitmap unexpected size");
 }  // namespace internal
 }  // namespace mesh

src/common.h

@@ -79,7 +79,29 @@ static constexpr size_t kMaxSize = 16384;
 static constexpr size_t kClassSizesMax = 25;
 static constexpr size_t kAlignment = 8;
 static constexpr int kMinAlign = 16;
-static constexpr uint64_t kPageSize = 4096;
+
+// Runtime page size detection for Apple Silicon (16KB) and x86 (4KB) compatibility
+namespace internal {
+  inline size_t initPageSize() {
+#if defined(_WIN32)
+    SYSTEM_INFO sysInfo;
+    GetSystemInfo(&sysInfo);
+    return sysInfo.dwPageSize;
+#else
+    return static_cast<size_t>(sysconf(_SC_PAGESIZE));
+#endif
+  }
+}
+
+// Runtime page size - initialized on first access
+inline size_t getPageSize() {
+  static const size_t kPageSize = internal::initPageSize();
+  return kPageSize;
+}
+
+// Keep a constexpr version for compile-time calculations (assume minimum 4KB)
+static constexpr uint64_t kPageSizeMin = 4096;
+
 static constexpr size_t kMaxFastLargeSize = 256 * 1024;  // 256Kb
 
 static constexpr size_t kMaxSplitListSize = 16384;
@@ -100,23 +122,25 @@ static constexpr size_t kMaxMeshesPerIteration = 2500;
 
 // maximum number of dirty pages to hold onto before we flush them
 // back to the OS (via MeshableArena::scavenge()
-static constexpr size_t kMaxDirtyPageThreshold = 1 << 14;  // 64 MB in pages
-static constexpr size_t kMinDirtyPageThreshold = 32;       // 128 KB in pages
+// Note: These are in 4KB page units for backward compatibility
+// Actual byte limits are calculated at runtime based on system page size
+static constexpr size_t kMaxDirtyPageThreshold = 1 << 14;  // 64 MB in 4KB pages
+static constexpr size_t kMinDirtyPageThreshold = 32;       // 128 KB in 4KB pages
 
 static constexpr uint32_t kSpanClassCount = 256;
 
 static constexpr int kNumBins = 25;  // 16Kb max object size
 static constexpr int kDefaultMeshPeriod = 10000;
 
-static constexpr size_t kMinArenaExpansion = 4096;  // 16 MB in pages
+static constexpr size_t kMinArenaExpansion = 4096;  // 16 MB in 4KB pages (4096 * 4KB = 16MB)
 
 // ensures we amortize the cost of going to the global heap enough
 static constexpr uint64_t kMinStringLen = 8;
 static constexpr size_t kMiniheapRefillGoalSize = 4 * 1024;
 static constexpr size_t kMaxMiniheapsPerShuffleVector = 24;
 
 // shuffle vector features
-static constexpr int16_t kMaxShuffleVectorLength = 256;  // sizeof(uint8_t) << 8
+static constexpr int16_t kMaxShuffleVectorLength = 1024;  // increased to support 16KB pages with 16-byte objects
 static constexpr bool kEnableShuffleOnInit = SHUFFLE_ON_INIT == 1;
 static constexpr bool kEnableShuffleOnFree = SHUFFLE_ON_FREE == 1;
 
@@ -129,24 +153,36 @@ static constexpr std::chrono::milliseconds kMeshPeriodMs{100};  // 100 ms
 // controls aspects of miniheaps
 static constexpr size_t kMaxMeshes = 256;  // 1 per bit
 #ifdef __APPLE__
-static constexpr size_t kArenaSize = 32ULL * 1024ULL * 1024ULL * 1024ULL;  // 16 GB
+static constexpr size_t kArenaSize = 32ULL * 1024ULL * 1024ULL * 1024ULL;  // 32 GB
 #else
 static constexpr size_t kArenaSize = 64ULL * 1024ULL * 1024ULL * 1024ULL;  // 64 GB
 #endif
-static constexpr size_t kAltStackSize = 16 * 1024UL;  // 16k sigaltstacks
+static constexpr size_t kAltStackSize = 16 * 1024UL;  // 16KB sigaltstacks
 #define SIGQUIESCE (SIGRTMIN + 7)
 #define SIGDUMP (SIGRTMIN + 8)
 
 // BinnedTracker
 static constexpr size_t kBinnedTrackerBinCount = 1;
 static constexpr size_t kBinnedTrackerMaxEmpty = 128;
 
-static inline constexpr size_t PageCount(size_t sz) {
-  return (sz + (kPageSize - 1)) / kPageSize;
+// Runtime page count calculation
+static inline size_t PageCount(size_t sz) {
+  const auto pageSize = getPageSize();
+  return (sz + (pageSize - 1)) / pageSize;
+}
+
+// Runtime page rounding
+static inline size_t RoundUpToPage(size_t sz) {
+  return getPageSize() * PageCount(sz);
+}
+
+// Constexpr versions for compile-time calculations (use minimum page size)
+static inline constexpr size_t PageCountMin(size_t sz) {
+  return (sz + (kPageSizeMin - 1)) / kPageSizeMin;
 }
 
-static inline constexpr size_t RoundUpToPage(size_t sz) {
-  return kPageSize * PageCount(sz);
+static inline constexpr size_t RoundUpToPageMin(size_t sz) {
+  return kPageSizeMin * PageCountMin(sz);
 }
 
 namespace powerOfTwo {
@@ -186,7 +222,8 @@ using std::unique_lock;
 #define CACHELINE_SIZE 64
 #define CACHELINE_ALIGNED ATTRIBUTE_ALIGNED(CACHELINE_SIZE)
 #define CACHELINE_ALIGNED_FN CACHELINE_ALIGNED
-#define PAGE_ALIGNED ATTRIBUTE_ALIGNED(kPageSize)
+// Use maximum page size for alignment (16KB for Apple Silicon compatibility)
+#define PAGE_ALIGNED ATTRIBUTE_ALIGNED(16384)
 
 #define MESH_EXPORT __attribute__((visibility("default")))
 

src/fixed_array.h

@@ -13,8 +13,15 @@ namespace mesh {
 
 // enables iteration through the miniheaps in an array
 template <typename FixedArray>
-class FixedArrayIter : public std::iterator<std::forward_iterator_tag, typename FixedArray::value_type> {
+class FixedArrayIter {
 public:
+  // Iterator traits (modern C++17 approach, not inheriting from deprecated std::iterator)
+  using iterator_category = std::forward_iterator_tag;
+  using value_type = typename FixedArray::value_type;
+  using difference_type = std::ptrdiff_t;
+  using pointer = value_type*;
+  using reference = value_type&;
+
   FixedArrayIter(const FixedArray &a, const uint32_t i) : _i(i), _array(a) {
   }
   FixedArrayIter &operator++() {

src/global_heap.cc

@@ -384,14 +384,14 @@ size_t GlobalHeap::meshSizeClassLocked(size_t sizeClass, MergeSetArray &mergeSet
 void GlobalHeap::meshAllSizeClassesLocked() {
   static MergeSetArray PAGE_ALIGNED MergeSets;
   static_assert(sizeof(MergeSets) == sizeof(void *) * 2 * 4096, "array too big");
-  d_assert((reinterpret_cast<uintptr_t>(&MergeSets) & (kPageSize - 1)) == 0);
+  d_assert((reinterpret_cast<uintptr_t>(&MergeSets) & (getPageSize() - 1)) == 0);
 
   static SplitArray PAGE_ALIGNED Left;
   static SplitArray PAGE_ALIGNED Right;
   static_assert(sizeof(Left) == sizeof(void *) * 16384, "array too big");
   static_assert(sizeof(Right) == sizeof(void *) * 16384, "array too big");
-  d_assert((reinterpret_cast<uintptr_t>(&Left) & (kPageSize - 1)) == 0);
-  d_assert((reinterpret_cast<uintptr_t>(&Right) & (kPageSize - 1)) == 0);
+  d_assert((reinterpret_cast<uintptr_t>(&Left) & (getPageSize() - 1)) == 0);
+  d_assert((reinterpret_cast<uintptr_t>(&Right) & (getPageSize() - 1)) == 0);
 
   // if we have freed but not reset meshed mappings, this will reset
   // them to the identity mapping, ensuring we don't blow past our VMA
@@ -505,7 +505,8 @@ shiftedSplitting(MWC &prng, MiniHeapListEntry *miniheaps, SplitArray &left, Spli
     return;
   }
 
-  constexpr size_t nBytes = 32;
+  // Bitmap size increased from 32 bytes (256 bits) to 128 bytes (1024 bits)
+  constexpr size_t nBytes = 128;
   const size_t limit = rightSize < t ? rightSize : t;
   d_assert(nBytes == left[0]->bitmap().byteCount());
 

src/global_heap.h

@@ -116,7 +116,7 @@ class GlobalHeap : public MeshableArena {
     Span span{0, 0};
     char *spanBegin = Super::pageAlloc(span, pageCount, pageAlignment);
     d_assert(spanBegin != nullptr);
-    d_assert((reinterpret_cast<uintptr_t>(spanBegin) / kPageSize) % pageAlignment == 0);
+    d_assert((reinterpret_cast<uintptr_t>(spanBegin) / getPageSize()) % pageAlignment == 0);
 
     MiniHeap *mh = new (buf) MiniHeap(arenaBegin(), span, objectCount, objectSize);
 
@@ -142,11 +142,12 @@ class GlobalHeap : public MeshableArena {
 
     lock_guard<mutex> lock(_miniheapLock);
 
-    MiniHeap *mh = allocMiniheapLocked(-1, pageCount, 1, pageCount * kPageSize, pageAlignment);
+    const size_t pageSize = getPageSize();
+    MiniHeap *mh = allocMiniheapLocked(-1, pageCount, 1, pageCount * pageSize, pageAlignment);
 
     d_assert(mh->isLargeAlloc());
-    d_assert(mh->spanSize() == pageCount * kPageSize);
-    // d_assert(mh->objectSize() == pageCount * kPageSize);
+    d_assert(mh->spanSize() == pageCount * pageSize);
+    // d_assert(mh->objectSize() == pageCount * pageSize);
 
     void *ptr = mh->mallocAt(arenaBegin(), 0);
 
@@ -305,8 +306,9 @@ class GlobalHeap : public MeshableArena {
     // if we have objects bigger than the size of a page, allocate
     // multiple pages to amortize the cost of creating a
     // miniheap/globally locking the heap.  For example, asking for
-    // 2048 byte objects would allocate 4 4KB pages.
-    const size_t objectCount = max(kPageSize / objectSize, kMinStringLen);
+    // 2048 byte objects would allocate 4 4KB pages (or 16KB pages on Apple Silicon).
+    // Cap at 1024 to fit within the MiniHeap bitmap size limit (128 bytes = 1024 bits)
+    const size_t objectCount = min(max(getPageSize() / objectSize, static_cast<size_t>(kMinStringLen)), static_cast<size_t>(1024));
     const size_t pageCount = PageCount(objectSize * objectCount);
 
     while (bytesFree < kMiniheapRefillGoalSize && !miniheaps.full()) {