Added succinct counting blocked Bloom using rank instead of select

Author: thomasmueller

benchmarks/bulk-insert-and-query.cc

@@ -666,6 +666,25 @@ struct FilterAPI<SuccinctCountingBlockedBloomFilter<ItemType, bits_per_item, Has
   }
 };
 
+template <typename ItemType, size_t bits_per_item, typename HashFamily>
+struct FilterAPI<SuccinctCountingBlockedBloomRankFilter<ItemType, bits_per_item, HashFamily>> {
+  using Table = SuccinctCountingBlockedBloomRankFilter<ItemType, bits_per_item, HashFamily>;
+  static Table ConstructFromAddCount(size_t add_count) { return Table(add_count); }
+  static void Add(uint64_t key, Table* table) {
+    table->Add(key);
+  }
+  static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
+    throw std::runtime_error("Unsupported");
+    // table->AddAll(keys, start, end);
+  }
+  static void Remove(uint64_t key, Table * table) {
+    table->Remove(key);
+  }
+  CONTAIN_ATTRIBUTES static bool Contain(uint64_t key, const Table * table) {
+    return table->Contain(key);
+  }
+};
+
 // assuming that first1,last1 and first2, last2 are sorted,
 // this tries to find out how many of first1,last1 can be
 // found in first2, last2, this includes duplicates
@@ -942,6 +961,7 @@ int main(int argc, char * argv[]) {
     {60, "CountingBloom10 (addall)"},
     {61, "SuccCountingBloom10 (addall)"},
     {62, "SuccCountBlockBloom10"},
+    {63, "SuccCountBlockBloomRank10"},
 
     {70, "Xor8-singleheader"},
     {80, "Morton"},
@@ -1394,6 +1414,13 @@ int main(int argc, char * argv[]) {
           add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed, false, true);
       cout << setw(NAME_WIDTH) << names[a] << cf << endl;
   }
+  a = 63;
+  if (algorithmId == a  || (algos.find(a) != algos.end())) {
+      auto cf = FilterBenchmark<
+          SuccinctCountingBlockedBloomRankFilter<uint64_t, 10, SimpleMixSplit>>(
+          add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed, false, true);
+      cout << setw(NAME_WIDTH) << names[a] << cf << endl;
+  }
 
   a = 70;
   if (algorithmId == a || (algos.find(a) != algos.end())) {

src/bloom/bloom.h

@@ -192,6 +192,8 @@ BloomFilter<ItemType, bits_per_item, branchless, HashFamily, k>::Info() const {
   return ss.str();
 }
 
+
+
 /***************
  * Simple block filter (naive implementation)
  ***************/
@@ -200,82 +202,81 @@ template <size_t blocksize, int k,
           typename HashFamily = ::hashing::TwoIndependentMultiplyShift>
 class SimpleBlockFilter {
 private:
-  // The filter is divided up into Buckets:
-  using Bucket = uint64_t[blocksize];
-
-  const int bucketCount;
-
-  Bucket *directory_;
-
+  const size_t arrayLength;
+  uint64_t* data;
   HashFamily hasher_;
-
 public:
   // Consumes at most (1 << log_heap_space) bytes on the heap:
   explicit SimpleBlockFilter(const int bits);
   ~SimpleBlockFilter() noexcept;
   void Add(const uint64_t key) noexcept;
-
   bool Find(const uint64_t key) const noexcept;
-  uint64_t SizeInBytes() const { return sizeof(Bucket) * bucketCount; }
+  uint64_t SizeInBytes() const {
+    return arrayLength * 8;
+  }
 };
 
 template <size_t blocksize, int k, typename HashFamily>
 SimpleBlockFilter<blocksize, k, HashFamily>::SimpleBlockFilter(
     const int capacity)
-    : bucketCount(capacity * k / (blocksize * 47)), directory_(nullptr),
+    : arrayLength((capacity * 10) / 64 + 8),
       hasher_() {
-  const size_t alloc_size = bucketCount * sizeof(Bucket);
-  const int malloc_failed =
-      posix_memalign(reinterpret_cast<void **>(&directory_), 64, alloc_size);
-  if (malloc_failed)
-    throw ::std::bad_alloc();
-  memset(directory_, 0, alloc_size);
+  data = new uint64_t[arrayLength]();
 }
 
 template <size_t blocksize, int k, typename HashFamily>
 SimpleBlockFilter<blocksize, k, HashFamily>::~SimpleBlockFilter() noexcept {
-  free(directory_);
-  directory_ = nullptr;
+  free(data);
+  data = nullptr;
 }
 
 static inline uint64_t rotl64(uint64_t n, unsigned int c) {
   // assumes width is a power of 2
   const unsigned int mask = (CHAR_BIT * sizeof(n) - 1);
-  // assert ( (c<=mask) &&"rotate by type width or more");
   c &= mask;
   return (n << c) | (n >> ((-c) & mask));
 }
 
-char setbit64(uint64_t *t, uint64_t bit) { return *t |= (1L << (bit & 63)); }
-
 template <size_t blocksize, int k, typename HashFamily>
 inline void
 SimpleBlockFilter<blocksize, k, HashFamily>::Add(const uint64_t key) noexcept {
   const auto hash = hasher_(key);
-  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
-  Bucket *bucket = directory_ + bucket_idx;
-  uint32_t a = (uint32_t)(hash >> 32);
-  uint32_t b = (uint32_t)hash;
-  for (int i = 0; i < k; i++) {
-    setbit64((uint64_t *)bucket + (a % blocksize), a / blocksize);
-    a += b;
-  }
+  const uint32_t idx = reduce(hash, arrayLength);
+  uint64_t *bucket = data + idx;
+//  uint32_t a = (uint32_t)(hash ^ (hash >> 32));
+
+//  *bucket++ |= (uint64_t) ((1L << (a & 63)) | (1L << ((a >> 6) & 63)));
+//  *bucket |= (uint64_t) ((1L << ((a >> 12) & 63)) | (1L << ((a >> 18) & 63)));
+
+//  *bucket++ |= (uint64_t) (a & (a >> 1));
+  uint64_t m1 = 1L << hash;
+  uint64_t m2 = 1L << (hash >> 8);
+  uint64_t m = m1 | m2;
+  *bucket |= m;
+
 }
 template <size_t blocksize, int k, typename HashFamily>
 inline bool
 SimpleBlockFilter<blocksize, k, HashFamily>::Find(const uint64_t key) const
     noexcept {
   const auto hash = hasher_(key);
-  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
-  const Bucket *bucket = directory_ + bucket_idx;
-  uint32_t a = (uint32_t)(hash >> 32);
-  uint32_t b = (uint32_t)hash;
-  char ok = 1;
-  for (int i = 0; i < k; i++) {
-    ok &= bittest64(((const uint64_t *)bucket) + (a % blocksize), a / blocksize);
-    a += b;
-  }
-  return ok;
+  const uint32_t idx = reduce(hash, arrayLength);
+  uint64_t *bucket = data + idx;
+//  uint32_t a = (uint32_t)(hash ^ (hash >> 32));
+//  uint64_t m1 = (uint64_t) ((1L << (a & 63)) | (1L << ((a >> 6) & 63)));
+//  uint64_t m2 = (uint64_t) ((1L << ((a >> 12) & 63)) | (1L << ((a >> 18) & 63)));
+  uint64_t m1 = 1L << hash;
+  uint64_t m2 = 1L << (hash >> 8);
+  uint64_t m = m1 | m2;
+  return !((m & *bucket) - m);
+
+/*
+  uint64_t x = *bucket++;
+//  a += b;
+//  x = *bucket++;
+//  y &= (x >> (a & 63)) & (x >> ((a >> 8) & 63));
+  return y & 1;
+*/
 }
 
 } // namespace bloomfilter