Merge pull request #1 from lemire/master

Simplifying the code, removing unnecessary (obsolete) dependencies.

Author: thomasmueller

README.md

@@ -1,2 +1,26 @@
 # xorfilter_cpp
-Bloom filter alternative (C++)
+Xor Filters: Faster and Smaller Than Bloom and Cuckoo Filters (C++)
+
+## Prerequisites
+
+- A  C++11 compiler such as GNU G++ or LLVM Clang++
+- Make 
+
+
+
+## Usage
+
+```
+cd benchmarks
+make
+./bulk-insert-and-query.exe 10000000
+```
+
+
+## Where is your code?
+
+See src/xorfilter.h. This single header depends on src/hashutil.h.
+
+## Credit
+
+The code is derived from https://github.com/efficient/cuckoofilter by Bin Fan et al.

benchmarks/Makefile

@@ -2,13 +2,13 @@
 OPT = -O3 -DNDEBUG
 #OPT = -g -ggdb
 
-CXXFLAGS += -fno-strict-aliasing -Wall -std=c++11 -I. -I../src/ $(OPT) -march=core-avx2
+CXXFLAGS += -fno-strict-aliasing -Wall -std=c++11 -I. -I../src/ $(OPT) -march=native
 
-LDFLAGS+= -Wall -lpthread -lssl -lcrypto
+LDFLAGS = -Wall 
 
 HEADERS = $(wildcard ../src/*.h) *.h
 
-SRC = ../src/hashutil.cc
+
 
 .PHONY: all
 
@@ -19,5 +19,5 @@ all: $(BINS)
 clean:
 	/bin/rm -f $(BINS)
 
-%.exe: %.cc ${HEADERS} ${SRC} Makefile
-	$(CXX) $(CXXFLAGS) $< -o $@ $(SRC) $(LDFLAGS)
+%.exe: %.cc ${HEADERS}  Makefile
+	$(CXX) $(CXXFLAGS) $< -o $@ $(LDFLAGS)

benchmarks/bulk-insert-and-query.cc

@@ -41,7 +41,7 @@
 #endif
 
 using namespace std;
-
+using namespace hashing;
 using namespace cuckoofilter;
 using namespace xorfilter;
 using namespace xorfilter2;
@@ -66,6 +66,15 @@ struct Statistics {
   double bits_per_item;
 };
 
+//
+// Inlining the "contains" which are executed within a tight loop can be both
+// very detrimental or very beneficial, and which ways it goes depends on the
+// compiler. It is unclear whether we want to benchmark the inlining of Contains,
+// as it depends very much on how "contains" is used. So it is maybe reasonable
+// to benchmark it without inlining.
+//
+#define CONTAIN_ATTRIBUTES  __attribute__ ((noinline))
+
 // Output for the first row of the table of results. type_width is the maximum number of
 // characters of the description of any table type, and find_percent_count is the number
 // of different lookup statistics gathered for each table. This function assumes the
@@ -133,6 +142,8 @@ struct FilterAPI<CuckooFilter<ItemType, bits_per_item, TableType, HashFamily>> {
   }
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -167,6 +178,8 @@ struct FilterAPI<SimdBlockFilter<HashFamily>> {
   }
   static void AddAll(const vector<uint64_t> keys, const size_t start, const size_t end, Table* table) {
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return table->Find(key);
   }
@@ -184,6 +197,8 @@ struct FilterAPI<SimdBlockFilterFixed<HashFamily>> {
   }
   static void AddAll(const vector<uint64_t> keys, const size_t start, const size_t end, Table* table) {
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return table->Find(key);
   }
@@ -200,6 +215,8 @@ struct FilterAPI<XorFilter<ItemType, FingerprintType>> {
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
     table->AddAll(keys, start, end);
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -214,6 +231,8 @@ struct FilterAPI<XorFilter<ItemType, FingerprintType, HashFamily>> {
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
     table->AddAll(keys, start, end);
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -228,6 +247,8 @@ struct FilterAPI<XorFilter2<ItemType, FingerprintType, FingerprintStorageType, H
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
     table->AddAll(keys, start, end);
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -242,6 +263,8 @@ struct FilterAPI<XorFilter2n<ItemType, FingerprintType, FingerprintStorageType,
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
     table->AddAll(keys, start, end);
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -256,6 +279,8 @@ struct FilterAPI<XorFilterPlus<ItemType, FingerprintType, HashFamily>> {
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
     table->AddAll(keys, start, end);
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -270,6 +295,8 @@ struct FilterAPI<GcsFilter<ItemType, bits_per_item, HashFamily>> {
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
     table->AddAll(keys, start, end);
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -285,6 +312,8 @@ struct FilterAPI<GQFilter<ItemType, bits_per_item, HashFamily>> {
   }
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }
@@ -300,6 +329,8 @@ struct FilterAPI<BloomFilter<ItemType, bits_per_item, HashFamily>> {
   }
   static void AddAll(const vector<ItemType> keys, const size_t start, const size_t end, Table* table) {
   }
+
+  CONTAIN_ATTRIBUTES
   static bool Contain(uint64_t key, const Table * table) {
     return (0 == table->Contain(key));
   }

src/bloom.h

@@ -4,12 +4,10 @@
 #include <assert.h>
 #include <algorithm>
 
-#include "debug.h"
 #include "hashutil.h"
-#include "printutil.h"
 
 using namespace std;
-using namespace cuckoofilter;
+using namespace hashing;
 
 namespace bloomfilter {
 // status returned by a Bloom filter operation

src/cuckoofilter.h

@@ -31,7 +31,7 @@ const size_t kMaxCuckooCount = 500;
 // PackedTable to enable semi-sorting
 template <typename ItemType, size_t bits_per_item,
           template <size_t> class TableType = SingleTable,
-          typename HashFamily = TwoIndependentMultiplyShift>
+          typename HashFamily = hashing::TwoIndependentMultiplyShift>
 class CuckooFilter {
   // Storage of items
   TableType<bits_per_item> *table_;

src/cuckoofilter_stable.h

@@ -26,7 +26,7 @@ inline uint32_t reduce(uint64_t hash, uint32_t n) {
 // PackedTable to enable semi-sorting
 template <typename ItemType, size_t bits_per_item,
           template <size_t> class TableType = SingleTable,
-          typename HashFamily = TwoIndependentMultiplyShift>
+          typename HashFamily = hashing::TwoIndependentMultiplyShift>
 class CuckooFilterStable {
   // Storage of items
   TableType<bits_per_item> *table_;

src/gcs.h

@@ -4,12 +4,10 @@
 #include <assert.h>
 #include <algorithm>
 
-#include "debug.h"
 #include "hashutil.h"
-#include "printutil.h"
 
 using namespace std;
-using namespace cuckoofilter;
+using namespace hashing;
 
 namespace gcsfilter {
 // status returned by a gcs filter operation

src/gqf_cpp.h

@@ -4,9 +4,7 @@
 #include <assert.h>
 #include <algorithm>
 
-#include "debug.h"
 #include "hashutil.h"
-#include "printutil.h"
 
 #include "gqf_hashutil.h"
 #include "gqf_hashutil.c"
@@ -15,7 +13,7 @@
 #include "gqf.c"
 
 using namespace std;
-using namespace cuckoofilter;
+using namespace hashing;
 
 namespace gqfilter {
 // status returned by a GQ filter operation