First draft of f16 library

Author: Snehasish Kumar

compiler-rt/include/profile/MemProfData.inc

@@ -218,11 +218,7 @@ void Merge(const MemInfoBlock &newMIB) {
     ShorterHistogramSize = newMIB.AccessHistogramSize;
   }
   for (size_t i = 0; i < ShorterHistogramSize; ++i) {
-    // Cast to uint8_t* and cap the sum at 255 to prevent overflow
-    uint8_t *CurrentHistPtr = (uint8_t *)AccessHistogram;
-    uint8_t *ShorterHistPtr = (uint8_t *)ShorterHistogram;
-    uint32_t sum = CurrentHistPtr[i] + ShorterHistPtr[i];
-    CurrentHistPtr[i] = (sum > 255) ? 255 : (uint8_t)sum;
+    ((uint64_t *)AccessHistogram)[i] += ((uint64_t *)ShorterHistogram)[i];
   }
 }
 
@@ -232,6 +228,41 @@ void Merge(const MemInfoBlock &newMIB) {
 } __attribute__((__packed__));
 #endif
 
+constexpr int MantissaBits = 12;
+constexpr int ExponentBits = 4;
+constexpr uint16_t MaxMantissa = (1U << MantissaBits) - 1;
+constexpr uint16_t MaxExponent = (1U << ExponentBits) - 1;
+
+// Encodes a 64-bit unsigned integer into a 16-bit scaled integer format.
+inline uint16_t encodeHistogramCount(uint64_t Count) {
+  if (Count == 0)
+    return 0;
+
+  const uint64_t MaxRepresentableValue = static_cast<uint64_t>(MaxMantissa)
+                                         << MaxExponent;
+  if (Count > MaxRepresentableValue)
+    Count = MaxRepresentableValue;
+
+  if (Count <= MaxMantissa)
+    return Count;
+
+  uint64_t M = Count;
+  uint16_t E = 0;
+  while (M > MaxMantissa) {
+    M = (M + 1) >> 1;
+    E++;
+  }
+  return (E << MantissaBits) | static_cast<uint16_t>(M);
+}
+
+// Decodes a 16-bit scaled integer and returns the
+// decoded 64-bit unsigned integer.
+inline uint64_t decodeHistogramCount(uint16_t EncodedValue) {
+  const uint16_t E = EncodedValue >> MantissaBits;
+  const uint16_t M = EncodedValue & MaxMantissa;
+  return static_cast<uint64_t>(M) << E;
+}
+
 } // namespace memprof
 } // namespace llvm
 

compiler-rt/lib/memprof/memprof_allocator.cpp

@@ -77,7 +77,7 @@ void Print(const MemInfoBlock &M, const u64 id, bool print_terse) {
                              ? MAX_HISTOGRAM_PRINT_SIZE
                              : M.AccessHistogramSize;
     for (size_t i = 0; i < PrintSize; ++i) {
-      Printf("%u ", ((uint8_t *)M.AccessHistogram)[i]);
+      Printf("%llu ", ((uint64_t *)M.AccessHistogram)[i]);
     }
     Printf("\n");
   }
@@ -327,14 +327,12 @@ struct Allocator {
     uint32_t HistogramSize =
         RoundUpTo(user_size, HISTOGRAM_GRANULARITY) / HISTOGRAM_GRANULARITY;
     uintptr_t Histogram =
-        (uintptr_t)InternalAlloc(HistogramSize * sizeof(uint8_t));
-    memset((void *)Histogram, 0, HistogramSize * sizeof(uint8_t));
+        (uintptr_t)InternalAlloc(HistogramSize * sizeof(uint64_t));
+    memset((void *)Histogram, 0, HistogramSize * sizeof(uint64_t));
     for (size_t i = 0; i < HistogramSize; ++i) {
       u8 Counter =
           *((u8 *)HISTOGRAM_MEM_TO_SHADOW(p + HISTOGRAM_GRANULARITY * i));
-      // Cap the counter at HISTOGRAM_MAX_COUNTER (255) to prevent overflow
-      ((uint8_t *)Histogram)[i] =
-          (Counter > HISTOGRAM_MAX_COUNTER) ? HISTOGRAM_MAX_COUNTER : Counter;
+      ((uint64_t *)Histogram)[i] = (uint64_t)Counter;
     }
     MemInfoBlock newMIB(user_size, c, m->timestamp_ms, curtime, m->cpu_id,
                         GetCpuId(), Histogram, HistogramSize);

compiler-rt/lib/memprof/memprof_rawprofile.cpp

@@ -19,6 +19,7 @@ using ::__sanitizer::Vector;
 using ::llvm::memprof::MemInfoBlock;
 using SegmentEntry = ::llvm::memprof::SegmentEntry;
 using Header = ::llvm::memprof::Header;
+using ::llvm::memprof::encodeHistogramCount;
 
 namespace {
 template <class T> char *WriteBytes(const T &Pod, char *Buffer) {
@@ -169,14 +170,15 @@ void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds,
     // FIXME: We unnecessarily serialize the AccessHistogram pointer. Adding a
     // serialization schema will fix this issue. See also FIXME in
     // deserialization.
-    Ptr = WriteBytes((*h)->mib, Ptr);
-    for (u64 j = 0; j < (*h)->mib.AccessHistogramSize; ++j) {
-      // Read as uint8_t and write as uint8_t
-      uint8_t HistogramEntry = ((uint8_t *)((*h)->mib.AccessHistogram))[j];
+    auto &MIB = (*h)->mib;
+    Ptr = WriteBytes(MIB, Ptr);
+    for (u64 j = 0; j < MIB.AccessHistogramSize; ++j) {
+      u16 HistogramEntry =
+          encodeHistogramCount(((u64 *)(MIB.AccessHistogram))[j]);
       Ptr = WriteBytes(HistogramEntry, Ptr);
     }
-    if ((*h)->mib.AccessHistogramSize > 0) {
-      InternalFree((void *)((*h)->mib.AccessHistogram));
+    if (MIB.AccessHistogramSize > 0) {
+      InternalFree((void *)MIB.AccessHistogram);
     }
   }
   CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
@@ -250,7 +252,7 @@ u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules,
       },
       reinterpret_cast<void *>(&TotalAccessHistogramEntries));
   const u64 NumHistogramBytes =
-      RoundUpTo(TotalAccessHistogramEntries * sizeof(uint8_t), 8);
+      RoundUpTo(TotalAccessHistogramEntries * sizeof(uint16_t), 8);
 
   const u64 NumStackBytes = RoundUpTo(StackSizeBytes(StackIds), 8);
 

compiler-rt/lib/memprof/tests/CMakeLists.txt

@@ -26,6 +26,7 @@ set(MEMPROF_SOURCES
   ../memprof_rawprofile.cpp)
 
 set(MEMPROF_UNITTESTS
+  histogram_encoding.cpp
   rawprofile.cpp
   driver.cpp)
 

compiler-rt/lib/memprof/tests/histogram_encoding.cpp

@@ -0,0 +1,35 @@
+#include <cstdint>
+#include <vector>
+
+#include "profile/MemProfData.inc"
+#include "gtest/gtest.h"
+
+namespace llvm {
+namespace memprof {
+namespace {
+TEST(MemProf, F16EncodeDecode) {
+  const std::vector<uint64_t> TestCases = {
+      0, 100, 4095, 4096, 5000, 8191, 65535, 1000000, 134213640, 200000000,
+  };
+
+  for (const uint64_t TestCase : TestCases) {
+    const uint16_t Encoded = encodeHistogramCount(TestCase);
+    const uint64_t Decoded = decodeHistogramCount(Encoded);
+
+    const uint64_t MaxRepresentable = static_cast<uint64_t>(MaxMantissa)
+                                      << MaxExponent;
+
+    if (TestCase >= MaxRepresentable) {
+      EXPECT_EQ(Decoded, MaxRepresentable);
+    } else if (TestCase == 0) {
+      EXPECT_EQ(Decoded, TestCase);
+    } else {
+      // The decoded value should be close to the original value.
+      // The error should be less than 1/1024 for larger numbers.
+      EXPECT_NEAR(Decoded, TestCase, static_cast<double>(TestCase) / 1024.0);
+    }
+  }
+}
+} // namespace
+} // namespace memprof
+} // namespace llvm

compiler-rt/test/memprof/TestCases/memprof_histogram_uint8.cpp

@@ -1,3 +1,7 @@
+// Test the histogram support in memprof using the text format output.
+// Shadow memory counters per object are limited to 8b. In memory counters
+// aggregating counts across multiple objects are 64b.
+
 // RUN: %clangxx_memprof -O0 -mllvm -memprof-histogram -mllvm -memprof-use-callbacks=true %s -o %t && %env_memprof_opts=print_text=1:histogram=1:log_path=stdout %run %t 2>&1 | FileCheck %s
 
 #include <stdio.h>

llvm/include/llvm/ProfileData/MemProfData.inc

@@ -218,11 +218,7 @@ void Merge(const MemInfoBlock &newMIB) {
     ShorterHistogramSize = newMIB.AccessHistogramSize;
   }
   for (size_t i = 0; i < ShorterHistogramSize; ++i) {
-    // Cast to uint8_t* and cap the sum at 255 to prevent overflow
-    uint8_t *CurrentHistPtr = (uint8_t *)AccessHistogram;
-    uint8_t *ShorterHistPtr = (uint8_t *)ShorterHistogram;
-    uint32_t sum = CurrentHistPtr[i] + ShorterHistPtr[i];
-    CurrentHistPtr[i] = (sum > 255) ? 255 : (uint8_t)sum;
+    ((uint64_t *)AccessHistogram)[i] += ((uint64_t *)ShorterHistogram)[i];
   }
 }
 
@@ -232,6 +228,41 @@ void Merge(const MemInfoBlock &newMIB) {
 } __attribute__((__packed__));
 #endif
 
+constexpr int MantissaBits = 12;
+constexpr int ExponentBits = 4;
+constexpr uint16_t MaxMantissa = (1U << MantissaBits) - 1;
+constexpr uint16_t MaxExponent = (1U << ExponentBits) - 1;
+
+// Encodes a 64-bit unsigned integer into a 16-bit scaled integer format.
+inline uint16_t encodeHistogramCount(uint64_t Count) {
+  if (Count == 0)
+    return 0;
+
+  const uint64_t MaxRepresentableValue = static_cast<uint64_t>(MaxMantissa)
+                                         << MaxExponent;
+  if (Count > MaxRepresentableValue)
+    Count = MaxRepresentableValue;
+
+  if (Count <= MaxMantissa)
+    return Count;
+
+  uint64_t M = Count;
+  uint16_t E = 0;
+  while (M > MaxMantissa) {
+    M = (M + 1) >> 1;
+    E++;
+  }
+  return (E << MantissaBits) | static_cast<uint16_t>(M);
+}
+
+// Decodes a 16-bit scaled integer and returns the
+// decoded 64-bit unsigned integer.
+inline uint64_t decodeHistogramCount(uint16_t EncodedValue) {
+  const uint16_t E = EncodedValue >> MantissaBits;
+  const uint16_t M = EncodedValue & MaxMantissa;
+  return static_cast<uint64_t>(M) << E;
+}
+
 } // namespace memprof
 } // namespace llvm
 

llvm/lib/ProfileData/MemProfReader.cpp

@@ -178,49 +178,20 @@ readMemInfoBlocksV5(const char *Ptr) {
     const uint64_t Id =
         endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
 
-    MemInfoBlock MIB;
-#define READ_MIB_FIELD(FIELD)                                                  \
-  MIB.FIELD = endian::readNext<decltype(MIB.FIELD), llvm::endianness::little,  \
-                               unaligned>(Ptr)
-
-    READ_MIB_FIELD(AllocCount);
-    READ_MIB_FIELD(TotalAccessCount);
-    READ_MIB_FIELD(MinAccessCount);
-    READ_MIB_FIELD(MaxAccessCount);
-    READ_MIB_FIELD(TotalSize);
-    READ_MIB_FIELD(MinSize);
-    READ_MIB_FIELD(MaxSize);
-    READ_MIB_FIELD(AllocTimestamp);
-    READ_MIB_FIELD(DeallocTimestamp);
-    READ_MIB_FIELD(TotalLifetime);
-    READ_MIB_FIELD(MinLifetime);
-    READ_MIB_FIELD(MaxLifetime);
-    READ_MIB_FIELD(AllocCpuId);
-    READ_MIB_FIELD(DeallocCpuId);
-    READ_MIB_FIELD(NumMigratedCpu);
-    READ_MIB_FIELD(NumLifetimeOverlaps);
-    READ_MIB_FIELD(NumSameAllocCpu);
-    READ_MIB_FIELD(NumSameDeallocCpu);
-    READ_MIB_FIELD(DataTypeId);
-    READ_MIB_FIELD(TotalAccessDensity);
-    READ_MIB_FIELD(MinAccessDensity);
-    READ_MIB_FIELD(MaxAccessDensity);
-    READ_MIB_FIELD(TotalLifetimeAccessDensity);
-    READ_MIB_FIELD(MinLifetimeAccessDensity);
-    READ_MIB_FIELD(MaxLifetimeAccessDensity);
-    READ_MIB_FIELD(AccessHistogramSize);
-    READ_MIB_FIELD(AccessHistogram);
-#undef READ_MIB_FIELD
+    MemInfoBlock MIB = *reinterpret_cast<const MemInfoBlock *>(Ptr);
+    Ptr += sizeof(MemInfoBlock);
 
     if (MIB.AccessHistogramSize > 0) {
       // The in-memory representation uses uint64_t for histogram entries.
       MIB.AccessHistogram =
           (uintptr_t)malloc(MIB.AccessHistogramSize * sizeof(uint64_t));
       for (uint64_t J = 0; J < MIB.AccessHistogramSize; J++) {
-        // The on-disk format for V5 uses uint8_t.
-        const uint8_t Val =
-            endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Ptr);
-        ((uint64_t *)MIB.AccessHistogram)[J] = Val;
+        // The on-disk format for V5 uses uint16_t which is then decoded to
+        // uint64_t.
+        const uint16_t Val =
+            endian::readNext<uint16_t, llvm::endianness::little, unaligned>(
+                Ptr);
+        ((uint64_t *)MIB.AccessHistogram)[J] = decodeHistogramCount(Val);
       }
     }
     Items.push_back({Id, MIB});