Solving issue 2 #2

Author: lemire

Makefile

@@ -7,7 +7,7 @@ dependencies/xor_singleheader/include/xorfilter.h:
 	git submodule update --init --recursive
 
 query_filter: src/query_filter.cpp src/hexutil.h          dependencies/xor_singleheader/include/xorfilter.h
-	c++ -O3 -o query_filter src/query_filter.cpp -Wall -std=c++11 -Idependencies
+	c++ -O3 -o query_filter src/query_filter.cpp -Wall -std=c++11 -Idependencies/fastfilter_cpp/src  -Idependencies
 
 build_filter: src/build_filter.cpp dependencies/fastfilter_cpp/src/xorfilter.h dependencies/fastfilter_cpp/src/xorfilter_plus.h src/hexutil.h             dependencies/xor_singleheader/include/xorfilter.h
 	c++ -O3 -o build_filter src/build_filter.cpp -std=c++11 -Wall -Idependencies/fastfilter_cpp/src -Idependencies

src/build_filter.cpp

@@ -13,11 +13,15 @@
 #include "hexutil.h"
 #include "xorfilter.h"
 #include "xor_singleheader/include/xorfilter.h"
+#include "mappeablebloomfilter.h"
+
 
 static void printusage(char *command) {
   printf(" Try %s -f xor8 -o filter.bin mydatabase \n", command);
   ;
   printf("The supported filters are xor8 and bloom12.\n");
+
+  printf("The -V flag verifies the resulting filter.\n");
 }
 
 int main(int argc, char **argv) {
@@ -26,15 +30,19 @@ int main(int argc, char **argv) {
       1000 * 1000 * 1000; // one billion lines ought to be more than enough?
   const char *filtername = "xor8";
   bool printall = false;
+  bool verify = false;
   const char *outputfilename = "filter.bin";
-  while ((c = getopt(argc, argv, "af:ho:m:")) != -1)
+  while ((c = getopt(argc, argv, "af:ho:m:V")) != -1)
     switch (c) {
     case 'f':
       filtername = optarg;
       break;
     case 'o':
       outputfilename = optarg;
       break;
+    case 'V':
+      verify = true;
+      break;
     case 'm':
       maxline = atoll(optarg);
       printf("setting the max. number of entries to %zu \n", maxline);
@@ -130,6 +138,16 @@ int main(int argc, char **argv) {
     xor8_buffered_populate(array, array_size, &filter);
     end = clock();
     printf("Done in %.3f seconds.\n", (float)(end - start) / CLOCKS_PER_SEC);
+    if(verify) {
+      printf("Checking for false negatives\n");
+      for(size_t i = 0; i < array_size; i++) {
+        if(!xor8_contain(array[i],&filter)) {
+          printf("Detected a false negative. You probably have a bug. Aborting.\n");
+          return EXIT_FAILURE;
+        }
+      }
+      printf("Verified with success: no false negatives\n");
+    }
     free(array);
 
     FILE *write_ptr;
@@ -165,6 +183,16 @@ int main(int argc, char **argv) {
     table.AddAll(array, 0, array_size);
     end = clock();
     printf("Done in %.3f seconds.\n", (float)(end - start) / CLOCKS_PER_SEC);
+    if(verify) {
+      printf("Checking for false negatives\n");
+      for(size_t i = 0; i < array_size; i++) {
+        if(table.Contain(array[i]) != xorfilter::Ok) {
+          printf("Detected a false negative. You probably have a bug. Aborting.\n");
+          return EXIT_FAILURE;
+        }
+      }
+      printf("Verified with success: no false negatives\n");
+    }
     free(array);
 
     FILE *write_ptr;
@@ -199,6 +227,24 @@ int main(int argc, char **argv) {
     table.AddAll(array, 0, array_size);
     end = clock();
     printf("Done in %.3f seconds.\n", (float)(end - start) / CLOCKS_PER_SEC);
+    if(verify) {
+      printf("Checking for false negatives\n");
+      for(size_t i = 0; i < array_size; i++) {
+        if(table.Contain(array[i]) != bloomfilter::Ok) {
+          printf("Detected a false negative. You probably have a bug. Aborting.\n");
+          return EXIT_FAILURE;
+        }
+      }
+      MappeableBloomFilter<12> filter(
+        table.SizeInBytes() / 8, table.hasher.seed, table.data);
+      for(size_t i = 0; i < array_size; i++) {
+        if(!filter.Contain(array[i])) {
+          printf("Detected a false negative. You probably have a bug. Aborting.\n");
+          return EXIT_FAILURE;
+        }
+      }
+      printf("Verified with success: no false negatives\n");
+    }
     free(array);
     FILE *write_ptr;
     write_ptr = fopen(outputfilename, "wb");

src/mappeablebloomfilter.h

@@ -3,17 +3,26 @@
 #include <inttypes.h>
 #include <limits.h>
 #include <sstream>
+#include <cmath>
+
+namespace {
+static inline size_t getBestK(size_t bitsPerItem) {
+  return std::max(1, (int)round((double)bitsPerItem * log(2)));
+}
+}
+
 inline uint64_t getBit(uint32_t index) { return 1L << (index & 63); }
 
-template <int k> class MappeableBloomFilter {
+template <int bitsPerItem> class MappeableBloomFilter {
 public:
   size_t arrayLength;
   const uint64_t *data;
   MixSplit hasher;
+  int k;
 
   explicit MappeableBloomFilter(const size_t arrayLength, const uint64_t seed,
                                 const uint64_t *fps)
-      : arrayLength(arrayLength), data(fps), hasher(seed) {}
+      : arrayLength(arrayLength), data(fps), hasher(seed), k(getBestK(bitsPerItem)) {}
 
   // Report if the item is inserted, with false positive rate.
   bool Contain(const uint64_t key) const {