Add VQF

Author: thomasmueller

benchmarks/Makefile

@@ -7,6 +7,7 @@ OPT = -O3 -DNDEBUG -std=c++17
 CXXFLAGS += -fno-strict-aliasing -Wall -std=c++11 -I. -I../src/ \
     -I../src/bloom/ -I../src/cuckoo/ -I../src/gcs \
     -I../src/gqf/ -I../src/morton/ -I../src/xorfilter -I../src/ribbon \
+    -I../src/vqf \
     $(OPT)
 
 UNAME_P := $(shell uname -p)
@@ -20,6 +21,7 @@ LDFLAGS = -Wall -Wextra
 HEADERS = $(wildcard ../src/*.h \
     ../src/bloom/*.h ../src/cuckoo/*.h ../src/gcs/*.h \
     ../src/gqf/*.h ../src/morton/*.h ../src/xorfilter/*.h ../src/ribbon/*.h \
+    ../src/vqf/*.h \
     ) *.h
 
 .PHONY: all

benchmarks/bulk-insert-and-query.cc

@@ -356,8 +356,9 @@ int main(int argc, char * argv[]) {
     // GCS
     {20,"GCS"},
 #ifdef __AVX2__
-    // CQF
+    // CQF + VQF
     {30,"CQF"},
+    {31,"VQF"},
 #endif
     // Bloom
     {40, "Bloom8"}, {41, "Bloom12" }, {42, "Bloom16"},
@@ -729,6 +730,13 @@ int main(int argc, char * argv[]) {
           add_count, to_add, intersectionsize, mixed_sets,  false, true);
       cout << setw(NAME_WIDTH) << names[a] << cf << endl;
   }
+  a = 31;
+  if (algorithmId == a || (algos.find(a) != algos.end())) {
+      auto cf = FilterBenchmark<
+          VQFilter<uint64_t, SimpleMixSplit>>(
+          add_count, to_add, intersectionsize, mixed_sets,  true, false);
+      cout << setw(NAME_WIDTH) << names[a] << cf << endl;
+  }  
 #endif
 
   // Bloom ----------------------------------------------------------

benchmarks/filterapi.h

@@ -24,6 +24,7 @@
 #include "gcs.h"
 #ifdef __AVX2__
 #include "gqf_cpp.h"
+#include "vqf_cpp.h"
 #include "simd-block.h"
 #endif
 #include "simd-block-fixed-fpp.h"
@@ -41,6 +42,7 @@ using namespace gcsfilter;
 using namespace CompressedCuckoo; // Morton filter namespace
 #ifdef __AVX2__
 using namespace gqfilter;
+using namespace vqfilter;
 #endif
 using namespace ribbon;
 
@@ -853,6 +855,27 @@ struct FilterAPI<GQFilter<ItemType, bits_per_item, HashFamily>> {
     return (0 == table->Contain(key));
   }
 };
+
+template <typename ItemType, typename HashFamily>
+struct FilterAPI<VQFilter<ItemType, HashFamily>> {
+  using Table = VQFilter<ItemType, HashFamily>;
+  static Table ConstructFromAddCount(size_t add_count) { return Table(add_count); }
+  static void Add(uint64_t key, Table* table) {
+    table->Add(key);
+  }
+  static void AddAll(const vector<uint64_t>& keys, const size_t start, const size_t end, Table* table) {
+    table->AddAll(keys, start, end);
+  // for(size_t i = start; i < end; i++) { Add(keys[i],table); }
+  }
+  static void Remove(uint64_t, Table *) {
+    throw std::runtime_error("Unsupported");
+    // sometimes fails with the original VQF implementation
+    // table->Remove(key);
+  }
+  CONTAIN_ATTRIBUTES static bool Contain(uint64_t key, const Table * table) {
+    return (0 == table->Contain(key));
+  }
+};
 #endif
 
 template <typename ItemType, size_t bits_per_item, bool branchless, typename HashFamily>

src/vqf/vqf_cpp.h

@@ -0,0 +1,170 @@
+#ifndef VQ_FILTER_VQ_FILTER_H_
+#define VQ_FILTER_VQ_FILTER_H_
+
+#include <assert.h>
+#include <algorithm>
+
+#include "hashutil.h"
+
+#include "vqf_filter.h"
+#include "vqf_filter.c"
+
+using namespace std;
+using namespace hashing;
+
+namespace vqfilter {
+// status returned by a VQ filter operation
+enum Status {
+  Ok = 0,
+  NotFound = 1,
+  NotEnoughSpace = 2,
+  NotSupported = 3,
+};
+
+template <typename ItemType, typename HashFamily = SimpleMixSplit>
+class VQFilter {
+
+  vqf_filter *filter;
+  uint64_t bytesUsed;
+  uint64_t range;
+  double bitsPerItem;
+  HashFamily hasher;
+
+  double BitsPerItem() const { return bitsPerItem; }
+  
+  void ApplyBlock(uint64_t *tmp, int block, int len);
+
+ public:
+  explicit VQFilter(const size_t n) : hasher() {
+
+    // when inserting in random order
+    // uint64_t nslots = (uint64_t) (n / 0.94);
+    // when inserting in sorted order
+    uint64_t nslots = (uint64_t) (n / 0.89);
+
+    if ((filter = vqf_init(nslots)) == NULL) {
+      std::cout << "Can't allocate.\n";
+      abort();
+    }
+    range = filter->metadata.range;
+    bytesUsed = filter->metadata.total_size_in_bytes;
+    bitsPerItem = (double) bytesUsed / n;
+
+  }
+
+  ~VQFilter() {
+      free(filter);
+  }
+
+  // Add an item to the filter.
+  Status Add(const ItemType &item);
+  
+  Status AddAll(const vector<ItemType> &data, const size_t start, const size_t end) {
+    return AddAll(data.data(), start, end);
+  }
+
+  // Add an item to the filter.
+  Status AddAll(const ItemType *data, const size_t start, const size_t end);
+  
+  // Report if the item is inserted, with false positive rate.
+  Status Contain(const ItemType &item) const;
+
+  /* methods for providing stats  */
+  // summary infomation
+  std::string Info() const;
+
+  // number of current inserted items;
+  size_t Size() const { return 0; }
+
+  // size of the filter in bytes.
+  size_t SizeInBytes() const { return bytesUsed; }
+};
+
+template <typename ItemType, typename HashFamily>
+Status VQFilter<ItemType, HashFamily>::Add(
+    const ItemType &key) {
+    uint64_t hash = hasher(key);
+    bool ret = vqf_insert(filter, hash);
+    if (!ret) {
+        std::cout << "failed insertion for key.\n";
+        abort();
+    }
+    return Ok;
+}
+
+template <typename ItemType, typename HashFamily>
+Status VQFilter<ItemType, HashFamily>::Contain(
+    const ItemType &key) const {
+    uint64_t hash = hasher(key);
+    bool ret = vqf_is_present(filter, hash);
+    return ret ? Ok : NotFound;
+}
+
+const int blockShift = 15;
+const int blockLen = 1 << blockShift;
+
+template <typename ItemType, typename HashFamily>
+void VQFilter<ItemType, HashFamily>::ApplyBlock(uint64_t *tmp, int block, int len) {
+  // std::cout << "addAll ApplyBlock block " << block << " len " << len << "\n";
+  for (int i = 0; i < len; i++) {
+    uint64_t hash = tmp[(block << blockShift) + i];
+    // std::cout << "inserting " << hash << "\n";
+    bool ret = vqf_insert(filter, hash);
+    if (!ret) {
+        std::cout << "failed insertion for key.\n";
+        abort();
+    }
+  }
+}
+
+template <typename ItemType, typename HashFamily>
+Status VQFilter<ItemType, HashFamily>::AddAll(
+    const ItemType* keys, const size_t start, const size_t end) {
+    /*
+   for (size_t i = start; i < end; i++) {
+     uint64_t key = keys[i];
+     uint64_t hash = hasher(key);
+     std::cout << "adding " << hash << "\n";
+  }   
+  */ 
+  int blocks = 1 + (end - start) / blockLen;
+  uint64_t *tmp = new uint64_t[blocks * blockLen];
+  int *tmpLen = new int[blocks]();
+  // std::cout << "addAll blocks " << blocks << "\n";
+  for (size_t i = start; i < end; i++) {
+    uint64_t key = keys[i];
+    uint64_t hash = hasher(key);
+    // __uint128_t x = (__uint128_t)key * (__uint128_t)blocks;
+    __uint128_t x = (__uint128_t)hash * (__uint128_t)blocks;
+    int block = (uint64_t)(x >> 64);
+    int len = tmpLen[block];
+    tmp[(block << blockShift) + len] = hash;
+    tmpLen[block] = len + 1;
+    if (len + 1 == blockLen) {
+      ApplyBlock(tmp, block, len + 1);
+      tmpLen[block] = 0;
+    }
+  }
+  for (int block = 0; block < blocks; block++) {
+    ApplyBlock(tmp, block, tmpLen[block]);
+    tmpLen[block] = 0;
+  }
+  delete[] tmp;
+  delete[] tmpLen;
+  return Ok;
+}
+
+template <typename ItemType, typename HashFamily>
+std::string VQFilter<ItemType, HashFamily>::Info() const {
+  std::stringstream ss;
+  ss << "VQFilter Status:\n"
+     << "\t\tKeys stored: " << Size() << "\n";
+  if (Size() > 0) {
+    ss << "\t\tk:   " << BitsPerItem() << "\n";
+  } else {
+    ss << "\t\tk:   N/A\n";
+  }
+  return ss.str();
+}
+}  // namespace vqfilter
+#endif  // VQ_FILTER_VQ_FILTER_H_