New: succinct counting blocked Bloom filter

thomasmueller · thomasmueller · commit d026b8abfec8 · 2018-12-21T07:06:07.000+01:00
diff --git a/benchmarks/bulk-insert-and-query.cc b/benchmarks/bulk-insert-and-query.cc
@@ -547,6 +547,25 @@ struct FilterAPI<SuccinctCountingBloomFilter<ItemType, bits_per_item, branchless
   }
 };
 
+template <typename ItemType, size_t bits_per_item, typename HashFamily>
+struct FilterAPI<SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily>> {
+  using Table = SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily>;
+  static Table ConstructFromAddCount(size_t add_count) { return Table(add_count); }
+  static void Add(uint64_t key, Table* table) {
+    table->Add(key);
+  }
+  static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
+    throw std::runtime_error("Unsupported");
+    // table->AddAll(keys, start, end);
+  }
+  static void Remove(uint64_t key, Table * table) {
+    table->Remove(key);
+  }
+  CONTAIN_ATTRIBUTES static bool Contain(uint64_t key, const Table * table) {
+    return table->Contain(key);
+  }
+};
+
 // assuming that first1,last1 and first2, last2 are sorted,
 // this tries to find out how many of first1,last1 can be
 // found in first2, last2, this includes duplicates
@@ -822,6 +841,7 @@ int main(int argc, char * argv[]) {
     // Counting Bloom
     {60, "CountingBloom10 (addall)"},
     {61, "SuccCountingBloom10 (addall)"},
+    {62, "SuccCountBlockBloom10"},
     // Sort
     {100, "Sort"},
   };
@@ -1263,6 +1283,13 @@ int main(int argc, char * argv[]) {
           add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed, true, true);
       cout << setw(NAME_WIDTH) << names[a] << cf << endl;
   }
+  a = 62;
+  if (algorithmId == a || algorithmId < 0 || (algos.find(a) != algos.end())) {
+      auto cf = FilterBenchmark<
+          SuccinctCountingBlockedBloomFilter<uint64_t, 10, SimpleMixSplit>>(
+          add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed, false, true);
+      cout << setw(NAME_WIDTH) << names[a] << cf << endl;
+  }
 
   // Sort ----------------------------------------------------------
   a = 100;
diff --git a/src/counting_bloom.h b/src/counting_bloom.h
@@ -65,6 +65,8 @@ inline uint32_t reduce(uint32_t hash, uint32_t n) {
   return (uint32_t)(((uint64_t)hash * n) >> 32);
 }
 
+// CountingBloomFilter --------------------------------------------------------------------------------------
+
 template <typename ItemType, size_t bits_per_item, bool branchless,
           typename HashFamily = TwoIndependentMultiplyShift,
           int k = (int)((double)bits_per_item * 0.693147180559945 + 0.5)>
@@ -183,7 +185,7 @@ Status CountingBloomFilter<ItemType, bits_per_item, branchless, HashFamily, k>::
   return Ok;
 }
 
-// --------------------------------------------------------------------------------------
+// SuccinctCountingBloomFilter --------------------------------------------------------------------------------------
 
 // #define VERIFY_COUNT
 
@@ -467,5 +469,283 @@ Status SuccinctCountingBloomFilter<ItemType, bits_per_item, branchless, HashFami
   return Ok;
 }
 
+// SuccinctCountingBlockedBloomFilter --------------------------------------------------------------------------------------
+
+
+// #define VERIFY_COUNT
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily,
+          int k = (int)((double)bits_per_item * 0.693147180559945 + 0.5)>
+class SuccinctCountingBlockedBloomFilter {
+private:
+  const int bucketCount;
+  HashFamily hasher;
+  uint64_t *data;
+  uint64_t *counts;
+  uint64_t *overflow;
+  size_t overflowLength;
+  size_t nextFreeOverflow;
+#ifdef VERIFY_COUNT
+  uint8_t *realCount;
+#endif
+
+  void Increment(size_t group, int bit);
+  void Decrement(size_t group, int bit);
+  int ReadCount(size_t group, int bit);
+
+public:
+  explicit SuccinctCountingBlockedBloomFilter(const int capacity);
+  ~SuccinctCountingBlockedBloomFilter() noexcept;
+  void Add(const uint64_t key) noexcept;
+  void Remove(const uint64_t key) noexcept;
+  bool Contain(const uint64_t key) const noexcept;
+  uint64_t SizeInBytes() const {
+      return 2 * 64 * bucketCount + 8 * overflowLength;
+  }
+};
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    SuccinctCountingBlockedBloomFilter(const int capacity)
+    : bucketCount(capacity * bits_per_item / 512), hasher() {
+  const size_t alloc_size = bucketCount * (512 / 8);
+  const int malloc_failed =
+      posix_memalign(reinterpret_cast<void **>(&data), 64, alloc_size);
+  if (malloc_failed)
+    throw ::std::bad_alloc();
+  memset(data, 0, alloc_size);
+  size_t arrayLength = bucketCount * 8;
+  overflowLength = 100 + arrayLength / 100 * 36;
+  counts = new uint64_t[arrayLength]();
+  overflow = new uint64_t[overflowLength]();
+#ifdef VERIFY_COUNT
+  realCount = new uint8_t[arrayLength * 64]();
+#endif
+  nextFreeOverflow = 0;
+  for (size_t i = 0; i < overflowLength; i += 8) {
+      overflow[i] = i + 8;
+  }
+}
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    ~SuccinctCountingBlockedBloomFilter() noexcept {
+  free(data);
+  delete[] counts;
+  delete[] overflow;
+}
+
+static inline uint64_t rotl64(uint64_t n, unsigned int c) {
+  // assumes width is a power of 2
+  const unsigned int mask = (CHAR_BIT * sizeof(n) - 1);
+  // assert ( (c<=mask) &&"rotate by type width or more");
+  c &= mask;
+  return (n << c) | (n >> ((-c) & mask));
+}
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+void SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    Add(const uint64_t key) noexcept {
+  const auto hash = hasher(key);
+  const uint32_t bucket_start = reduce(rotl64(hash, 32), bucketCount) * 8;
+  uint32_t a = (uint32_t)hash;
+  if (k >= 3) {
+    Increment(bucket_start + ((a >> 0) & 7), (a >> 3) & 0x3f);
+    Increment(bucket_start + ((a >> 9) & 7), (a >> 12) & 0x3f);
+    Increment(bucket_start + ((a >> 18) & 7), (a >> 21) & 0x3f);
+//    data[bucket_start + ((a >> 0) & 7)] |= 1ULL << ((a >> 3) & 0x3f);
+//    data[bucket_start + ((a >> 9) & 7)] |= 1ULL << ((a >> 12) & 0x3f);
+//    data[bucket_start + ((a >> 18) & 7)] |= 1ULL << ((a >> 21) & 0x3f);
+  }
+  uint32_t b = (uint32_t)(hash >> 32);
+  for (int i = 3; i < k; i++) {
+    a += b;
+    Increment(bucket_start + (a & 7), (a >> 3) & 0x3f);
+//    data[bucket_start + (a & 7)] |= 1ULL << ((a >> 3) & 0x3f);
+  }
+}
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+void SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    Increment(size_t group, int bit) {
+#ifdef VERIFY_COUNT
+    realCount[(group << 6) + bit]++;
+#endif
+    uint64_t m = data[group];
+    uint64_t c = counts[group];
+    if ((c & 0xc000000000000000ULL) != 0) {
+        // an overflow entry, or overflowing now
+        size_t index;
+        if ((c & 0x8000000000000000ULL) == 0) {
+            // convert to an overflow entry
+            // allocate overflow
+            index = nextFreeOverflow;
+            if (index >= overflowLength) {
+                ::std::cout << "ERROR: overflow too small\n";
+                data[group] |= 1ULL << bit;
+                return;
+            }
+            nextFreeOverflow = (size_t) overflow[index];
+            for (int i = 0; i < 8; i++) {
+                overflow[index + i] = 0;
+            }
+            // convert to a pointer
+            for (int i = 0; i < 64; i++) {
+                int n = ReadCount(group, i);
+                overflow[index + i / 8] += n * (1ULL << (i * 8));
+            }
+            uint64_t count = 64;
+            c = 0x8000000000000000ULL | (count << 32) | index;
+            counts[group] = c;
+        } else {
+            // already
+            index = (size_t) (c & 0x0fffffffULL);
+            c += 1ULL << 32;
+            counts[group] = c;
+        }
+        overflow[index + bit / 8] += (1ULL << (bit * 8));
+        data[group] |= 1ULL << bit;
+    } else {
+        data[group] |= 1ULL << bit;
+        int bitsBefore = bitCount64(m & (0xffffffffffffffffULL >> (63 - bit)));
+        int before = select64((c << 1) | 1, bitsBefore);
+        int d = (m >> bit) & 1;
+        int insertAt = before - d;
+        uint64_t mask = (1ULL << insertAt) - 1;
+        uint64_t left = c & ~mask;
+        uint64_t right = c & mask;
+        c = (left << 1) | ((1ULL ^ d) << insertAt) | right;
+        counts[group] = c;
+    }
+#ifdef VERIFY_COUNT
+    for(int b = 0; b < 64; b++) {
+        if (realCount[(group << 6) + b] != ReadCount(group, b)) {
+            ::std::cout << "group " << group << "/" << b << " of " << bit << "\n";
+        }
+    }
+#endif
+}
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+int SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    ReadCount(size_t group, int bit) {
+    uint64_t m = data[group];
+    uint64_t d = (m >> bit) & 1;
+    if (d == 0) {
+        return 0;
+    }
+    uint64_t c = counts[group];
+    if ((c & 0x8000000000000000ULL) != 0) {
+        size_t index = (size_t) (c & 0x0fffffffULL);
+        uint64_t n = overflow[index + bit / 8];
+        n >>= 8 * (bit & 0xff);
+        return (int) (n & 0xff);
+    }
+    int bitsBefore = bitCount64(m & (0xffffffffffffffffULL >> (63 - bit)));
+    int bitPos = select64(c, bitsBefore - 1);
+    uint64_t y = ((c << (63 - bitPos)) << 1) | (1ULL << (63 - bitPos));
+    return numberOfLeadingZeros64(y) + 1;
+}
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+void SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    Remove(const uint64_t key) noexcept {
+  const auto hash = hasher(key);
+  const uint32_t bucket_start = reduce(rotl64(hash, 32), bucketCount) * 8;
+  uint32_t a = (uint32_t)hash;
+  if (k >= 3) {
+    Decrement(bucket_start + ((a >> 0) & 7), (a >> 3) & 0x3f);
+    Decrement(bucket_start + ((a >> 9) & 7), (a >> 12) & 0x3f);
+    Decrement(bucket_start + ((a >> 18) & 7), (a >> 21) & 0x3f);
+  }
+  uint32_t b = (uint32_t)(hash >> 32);
+  for (int i = 3; i < k; i++) {
+    a += b;
+    Decrement(bucket_start + (a & 7), (a >> 3) & 0x3f);
+  }
+}
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+void SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    Decrement(size_t group, int bit) {
+#ifdef VERIFY_COUNT
+    realCount[(group << 6) + bit]--;
+#endif
+    uint64_t m = data[group];
+    uint64_t c = counts[group];
+    if ((c & 0x8000000000000000ULL) != 0) {
+        // an overflow entry
+        size_t index = (size_t) (c & 0x0fffffffULL);
+        size_t count = (size_t) (c >> 32) & 0x0fffffffULL;
+        c -= 1ULL << 32;
+        counts[group] = c;
+        uint64_t n = overflow[index + bit / 8];
+        overflow[index + bit / 8] = n - (1ULL << (bit * 8));
+        n >>= 8 * (bit & 0xf);
+        if ((n & 0xff) == 1) {
+            data[group] &= ~(1ULL << bit);
+        }
+        if (count < 64) {
+            // convert back to an inline entry, and free up the overflow entry
+            uint64_t c2 = 0;
+            for (int j = 63; j >= 0; j--) {
+                int cj = (int) ((overflow[index + j / 8] >> (8 * j)) & 0xff);
+                if (cj > 0) {
+                    c2 = ((c2 << 1) | 1) << (cj - 1);
+                }
+            }
+            counts[group] = c2;
+            // free overflow
+            overflow[index] = nextFreeOverflow;
+            nextFreeOverflow = index;
+        }
+    } else {
+        int bitsBefore = bitCount64(m & (0xffffffffffffffffULL >> (63 - bit)));
+        int before = select64((c << 1) | 1, bitsBefore) - 1;
+        int removeAt = max(0, before - 1);
+        // remove the bit from the counter
+        uint64_t mask = (1ULL << removeAt) - 1;
+        uint64_t left = (c >> 1) & ~mask;
+        uint64_t right= c & mask;
+        counts[group] = left | right;
+        uint64_t removed = (c >> removeAt) & 1;
+        // possibly reset the data bit
+        data[group] = m & ~(removed << bit);
+    }
+#ifdef VERIFY_COUNT
+    for(int b = 0; b < 64; b++) {
+        if (realCount[(group << 6) + b] != ReadCount(group, b)) {
+            ::std::cout << "group- " << group << "/" << b << " of " << bit << "\n";
+        }
+    }
+#endif
+}
+
+template <typename ItemType, size_t bits_per_item, typename HashFamily, int k>
+bool SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, HashFamily, k>::
+    Contain(const uint64_t key) const noexcept {
+  const auto hash = hasher(key);
+  const uint32_t bucket_start = reduce(rotl64(hash, 32), bucketCount) * 8;
+  uint32_t a = (uint32_t)hash;
+  char ok = 1;
+  if (k >= 3) {
+    ok &= data[bucket_start + ((a >> 0) & 7)] >> ((a >> 3) & 0x3f);
+    ok &= data[bucket_start + ((a >> 9) & 7)] >> ((a >> 12) & 0x3f);
+    ok &= data[bucket_start + ((a >> 18) & 7)] >> ((a >> 21) & 0x3f);
+  }
+  if (!ok) {
+    return ok;
+  }
+  uint32_t b = (uint32_t)(hash >> 32);
+  for (int i = 3; i < k; i++) {
+    a += b;
+    ok &= data[bucket_start + (a & 7)] >> ((a >> 3) & 63);
+    if (!ok) {
+      return ok;
+    }
+  }
+  return ok;
+}
+
 }
 #endif
