Implemented a naive block Bloom

Author: lemire

benchmarks/bulk-insert-and-query.cc

@@ -291,6 +291,28 @@ struct FilterAPI<XorFilter<ItemType, FingerprintType>> {
   }
 };
 
+
+template<size_t blocksize, int k, typename HashFamily>
+struct FilterAPI<SimpleBlockFilter<blocksize,k,HashFamily>> {
+  using Table = SimpleBlockFilter<blocksize,k,HashFamily>;
+  static Table ConstructFromAddCount(size_t add_count) {
+    Table ans(ceil(add_count * 8.0 / CHAR_BIT));
+    return ans;
+  }
+  static void Add(uint64_t key, Table* table) {
+    table->Add(key);
+  }
+  static void AddAll(const vector<uint64_t> keys, const size_t start, const size_t end, Table* table) {
+    throw std::runtime_error("Unsupported");
+  }
+
+  CONTAIN_ATTRIBUTES
+  static bool Contain(uint64_t key, const Table * table) {
+    return table->Find(key);
+  }
+};
+
+
 template <typename ItemType, typename FingerprintType, typename HashFamily>
 struct FilterAPI<XorFilter<ItemType, FingerprintType, HashFamily>> {
   using Table = XorFilter<ItemType, FingerprintType, HashFamily>;
@@ -678,43 +700,6 @@ void parse_comma_separated(char * c, std::set<int> & answer ) {
     }
 }
 
-/*
-#define MUL 1625L
-#define MUL2 (MUL*MUL)
-// (1<<64) / MUL
-#define INVMUL 11351842506898185L
-// (1<<64) / MUL2
-#define INVMUL2 6985749235014L
-int main() {
-    printf("start\n");
-    for (int a = 0; a < MUL; a++) {
-        for (int b = 0; b < MUL; b++) {
-            for (int c = 0; c < MUL; c++) {
-                uint32_t x = a * MUL2 + b * MUL + c;
-                int aa = (int) (((__uint128_t) x * (INVMUL2 + 1)) >> 64);
-                if (aa != a) {
-                    printf("wrong a");
-                    return -1;
-                }
-                int bb = (int) (((__uint128_t) x * (INVMUL + 1)) >> 64);
-                int rb = bb % MUL;
-                if (rb != b) {
-                    printf("wrong b");
-                    return -1;
-                }
-                int expected = (a + b + c) % MUL;
-                int got = (aa + bb + x) % MUL;
-                if (expected != got) {
-                    printf("wrong modulo");
-                    return -1;
-                }
-            }
-        }
-    }
-    printf("end\n");
-    return 0;
-}
-*/
 
 int main(int argc, char * argv[]) {
 #ifdef __aarch64__
@@ -726,7 +711,8 @@ int main(int argc, char * argv[]) {
    {14,"GCS"},  {22, "Xor10 (NBitArray)"}, {23, "Xor14 (NBitArray)"},
    {25, "Xor10"},{26, "Xor10.666"}, {37,"Bloom8 (addall)"},
    {38,"Bloom12 (addall)"},
-   {40,"BlockedBloom (addall)"}
+   {40,"BlockedBloom (addall)"},
+   {70,"SimpleBlockedBloom"}
   };
 #elif defined( __AVX2__)
   std::map<int,std::string> names = {{0,"Xor8"},{1,"Xor12"},
@@ -737,7 +723,8 @@ int main(int argc, char * argv[]) {
    {14,"GCS"}, {15,"CQF"}, {22, "Xor10 (NBitArray)"}, {23, "Xor14 (NBitArray)"}, 
    {25, "Xor10"},{26, "Xor10.666"}, {37,"Bloom8 (addall)"},
    {38,"Bloom12 (addall)"},{39,"Bloom16 (addall)"},
-   {40,"BlockedBloom (addall)"}, {63,"BlockedBloom16"}, {64,"BlockedBloom64"}
+   {40,"BlockedBloom (addall)"}, {63,"BlockedBloom16"}, {64,"BlockedBloom64"},
+   {70,"SimpleBlockedBloom"}
   };
 #else
   std::map<int,std::string> names = {{0,"Xor8"},{1,"Xor12"},
@@ -747,7 +734,8 @@ int main(int argc, char * argv[]) {
    {11,"sort"}, {12,"Xor+8"}, {13,"Xor+16"},
    {14,"GCS"}, {22, "Xor10 (NBitArray)"}, {23, "Xor14 (NBitArray)"},
    {25, "Xor10"},{26, "Xor10.666"}, {37,"Bloom8 (addall)"},
-   {38,"Bloom12 (addall)"},{39,"Bloom16 (addall)"}
+   {38,"Bloom12 (addall)"},{39,"Bloom16 (addall)"},
+   {70,"SimpleBlockedBloom"}
   };
 #endif
 
@@ -1091,6 +1079,13 @@ int main(int argc, char * argv[]) {
           add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed, true);
       cout << setw(NAME_WIDTH) << names[23] << cf << endl;
   }
+  if (algorithmId == 70 || (algos.find(70) != algos.end())) {
+      auto cf = FilterBenchmark<
+          SimpleBlockFilter<8, 8, SimpleMixSplit>>(
+          add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed, false);
+      cout << setw(NAME_WIDTH) << names[70] << cf << endl;
+  }
+
 
   // this algo overflows and crashes
   /*if (algorithmId == 24 || (algos.find(24) != algos.end())) {

src/bloom.h

@@ -172,5 +172,98 @@ std::string BloomFilter<ItemType, bits_per_item, HashFamily, k>::Info() const {
   }
   return ss.str();
 }
+
+
+
+
+/***************
+ * Simple block filter (naive implementation)
+ ***************/
+
+template<size_t blocksize, int k, typename HashFamily = ::hashing::TwoIndependentMultiplyShift>
+class SimpleBlockFilter {
+ private:
+  // The filter is divided up into Buckets:
+  using Bucket = uint64_t[blocksize];
+
+  const int bucketCount;
+
+  Bucket* directory_;
+
+  HashFamily hasher_;
+
+ public:
+  // Consumes at most (1 << log_heap_space) bytes on the heap:
+  explicit SimpleBlockFilter(const int bits);
+  ~SimpleBlockFilter() noexcept;
+  void Add(const uint64_t key) noexcept;
+
+  bool Find(const uint64_t key) const noexcept;
+  uint64_t SizeInBytes() const { return sizeof(Bucket) * bucketCount; }
+
+
+};
+
+template<size_t blocksize,int k,  typename HashFamily>
+SimpleBlockFilter<blocksize,k,HashFamily>::SimpleBlockFilter(const int capacity)
+  : bucketCount(capacity * k / (  blocksize * 64)),
+    directory_(nullptr),
+    hasher_() {
+  const size_t alloc_size = bucketCount * sizeof(Bucket);
+  const int malloc_failed =
+      posix_memalign(reinterpret_cast<void**>(&directory_), 64, alloc_size);
+  if (malloc_failed) throw ::std::bad_alloc();
+  memset(directory_, 0, alloc_size);
+}
+
+template<size_t blocksize, int k, typename HashFamily>
+SimpleBlockFilter<blocksize,k,HashFamily>::~SimpleBlockFilter() noexcept {
+  free(directory_);
+  directory_ = nullptr;
+}
+
+static inline uint64_t rotl64(uint64_t n, unsigned int c) {
+    // assumes width is a power of 2
+    const unsigned int mask = (CHAR_BIT * sizeof(n) - 1);
+    // assert ( (c<=mask) &&"rotate by type width or more");
+    c &= mask;
+    return (n << c) | ( n >> ((-c) & mask));
+}
+
+template<size_t blocksize,int k,  typename HashFamily>
+inline void
+SimpleBlockFilter<blocksize,k, HashFamily>::Add(const uint64_t key) noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  Bucket * bucket = directory_ + bucket_idx;
+  uint32_t a = (uint32_t) (hash >> 32);
+  uint32_t b = (uint32_t) hash;
+  for (int i = 0; i < k; i++) {
+        ((uint64_t *)bucket)[reduce(a, blocksize)] |= getBit(a);
+        a += b;
+  }
+
+}
+
+
+template<size_t blocksize, int k, typename HashFamily>
+inline bool
+SimpleBlockFilter<blocksize,k,HashFamily>::Find(const uint64_t key) const noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  const Bucket * bucket = directory_ + bucket_idx;
+  uint32_t a = (uint32_t) (hash >> 32);
+  uint32_t b = (uint32_t) hash;
+  for (int i = 0; i < k; i++) {
+        if ((((const uint64_t *)bucket)[reduce(a, blocksize)] & getBit(a)) == 0) {
+            return false;
+        }
+        a += b;
+  }
+  return true;
+}
+
+
+
 }  // namespace bloomfilter
 #endif  // BLOOM_FILTER_BLOOM_FILTER_H_

src/simd-block-fixed-fpp.h

@@ -30,7 +30,7 @@ inline uint32_t reduce(uint32_t hash, uint32_t n) {
     return (uint32_t) (((uint64_t) hash * n) >> 32);
 }
 
-inline uint64_t rotl64(uint64_t n, unsigned int c) {
+static inline uint64_t rotl64(uint64_t n, unsigned int c) {
     // assumes width is a power of 2
     const unsigned int mask = (CHAR_BIT * sizeof(n) - 1);
     // assert ( (c<=mask) &&"rotate by type width or more");