Merge pull request #5 from lemire/master

64-byte Blocked approach has *slightly* better space usage.

Author: thomasmueller

benchmarks/bulk-insert-and-query.cc

@@ -190,6 +190,27 @@ struct FilterAPI<SimdBlockFilter<HashFamily>> {
   }
 };
 
+template <typename HashFamily>
+struct FilterAPI<SimdBlockFilterFixed64<HashFamily>> {
+  using Table = SimdBlockFilterFixed64<HashFamily>;
+  static Table ConstructFromAddCount(size_t add_count) {
+    Table ans(ceil(add_count * 8.0 / CHAR_BIT));
+    return ans;
+  }
+  static void Add(uint64_t key, Table* table) {
+    table->Add(key);
+  }
+  static void AddAll(const vector<uint64_t> keys, const size_t start, const size_t end, Table* table) {
+     throw std::runtime_error("Unsupported");
+  }
+
+  CONTAIN_ATTRIBUTES
+  static bool Contain(uint64_t key, const Table * table) {
+    return table->Find(key);
+  }
+};
+
+
 template <typename HashFamily>
 struct FilterAPI<SimdBlockFilterFixed<HashFamily>> {
   using Table = SimdBlockFilterFixed<HashFamily>;
@@ -209,7 +230,6 @@ struct FilterAPI<SimdBlockFilterFixed<HashFamily>> {
     return table->Find(key);
   }
 };
-
 #endif
 
 template <typename ItemType, typename FingerprintType>
@@ -663,7 +683,7 @@ int main(int argc, char * argv[]) {
    {14,"GCS"}, {15,"CQF"}, {22, "Xor10 (NBitArray)"}, {23, "Xor14 (NBitArray)"}, 
    {25, "Xor10"},{26, "Xor10.666"}, {37,"Bloom8 (addall)"},
    {38,"Bloom12 (addall)"},{39,"Bloom16 (addall)"},
-   {40,"BlockedBloom (addall)"}
+   {40,"BlockedBloom (addall)"}, {64,"BlockedBloom64"}
   };
 
   if (argc < 2) {
@@ -876,10 +896,15 @@ int main(int argc, char * argv[]) {
 
 #ifdef __AVX2__
   if (algorithmId == 10 || algorithmId < 0 || (algos.find(10) != algos.end())) {
-      auto cf = FilterBenchmark<SimdBlockFilterFixed<SimpleMixSplit>>(
+      auto cf = FilterBenchmark<SimdBlockFilterFixed<>>(
           add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed);
       cout << setw(NAME_WIDTH) << names[10] << cf << endl;
   }
+  if (algorithmId == 64 || algorithmId < 0 || (algos.find(64) != algos.end())) {
+      auto cf = FilterBenchmark<SimdBlockFilterFixed64<SimpleMixSplit>>(
+          add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed);
+      cout << setw(NAME_WIDTH) << names[64] << cf << endl;
+  }
 #endif
 
   if (algorithmId == 11 || algorithmId < 0 || (algos.find(11) != algos.end())) {

src/simd-block-fixed-fpp.h

@@ -184,3 +184,101 @@ SimdBlockFilterFixed<HashFamily>::Find(const uint64_t key) const noexcept {
   return _mm256_testc_si256(bucket, mask);
 }
 
+/// 64-byte version
+
+struct mask64bytes {
+    __m256i first;
+    __m256i second;
+};
+
+typedef struct mask64bytes mask64bytes_t;
+
+template<typename HashFamily = ::hashing::TwoIndependentMultiplyShift>
+class SimdBlockFilterFixed64 {
+ private:
+  // The filter is divided up into Buckets:
+  using Bucket = mask64bytes_t;
+
+  const int bucketCount;
+
+  Bucket* directory_;
+
+  HashFamily hasher_;
+
+ public:
+  // Consumes at most (1 << log_heap_space) bytes on the heap:
+  explicit SimdBlockFilterFixed64(const int bits);
+  ~SimdBlockFilterFixed64() noexcept;
+  void Add(const uint64_t key) noexcept;
+
+  bool Find(const uint64_t key) const noexcept;
+  uint64_t SizeInBytes() const { return sizeof(Bucket) * bucketCount; }
+
+ private:
+  static mask64bytes_t MakeMask(const uint64_t hash) noexcept;
+
+
+};
+
+template<typename HashFamily>
+SimdBlockFilterFixed64<HashFamily>::SimdBlockFilterFixed64(const int bits)
+
+  : bucketCount(::std::max(1, bits / 50)),
+    directory_(nullptr),
+    hasher_() {
+  if (!__builtin_cpu_supports("avx2")) {
+    throw ::std::runtime_error("SimdBlockFilterFixed64 does not work without AVX2 instructions");
+  }
+  const size_t alloc_size = bucketCount * sizeof(Bucket);
+  const int malloc_failed =
+      posix_memalign(reinterpret_cast<void**>(&directory_), 64, alloc_size);
+  if (malloc_failed) throw ::std::bad_alloc();
+  memset(directory_, 0, alloc_size);
+}
+
+template<typename HashFamily>
+SimdBlockFilterFixed64<HashFamily>::~SimdBlockFilterFixed64() noexcept {
+  free(directory_);
+  directory_ = nullptr;
+}
+
+
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline mask64bytes_t
+SimdBlockFilterFixed64<HashFamily>::MakeMask(const uint64_t hash) noexcept {
+  const __m256i ones = _mm256_set1_epi64x(1);
+  const __m256i rehash1 = _mm256_setr_epi32(0x47b6137bU, 0x44974d91U, 0x8824ad5bU,
+      0xa2b7289dU, 0x705495c7U, 0x2df1424bU, 0x9efc4947U, 0x5c6bfb31U);
+  mask64bytes_t answer;
+  __m256i hash_data = _mm256_set1_epi64x(hash);
+  __m256i h = _mm256_mullo_epi32(rehash1, hash_data);
+  h = _mm256_srli_epi32(h, 26);
+  answer.first = _mm256_and_si256(_mm256_set1_epi64x(0x3f),h);
+  answer.first = _mm256_sllv_epi64(ones, answer.first);
+  answer.second = _mm256_srli_epi64(h,32);
+  answer.second = _mm256_sllv_epi64(ones, answer.second);
+  return answer;
+}
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline void
+SimdBlockFilterFixed64<HashFamily>::Add(const uint64_t key) noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  mask64bytes_t mask = MakeMask(hash);
+  mask64bytes_t* const bucket = &reinterpret_cast<mask64bytes_t*>(directory_)[bucket_idx];
+  bucket->first = _mm256_or_si256(mask.first, bucket->first);
+  bucket->second= _mm256_or_si256(mask.second, bucket->second);
+  assert(Find(key));
+}
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline bool
+SimdBlockFilterFixed64<HashFamily>::Find(const uint64_t key) const noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  const mask64bytes_t mask = MakeMask(hash);
+  const mask64bytes_t  bucket = reinterpret_cast<mask64bytes_t*>(directory_)[bucket_idx];
+  return _mm256_testc_si256(bucket.first, mask.first) & _mm256_testc_si256(bucket.second, mask.second);
+}

src/simd-block.h

@@ -141,168 +141,3 @@ SimdBlockFilter<HashFamily>::Find(const uint64_t key) const noexcept {
 }
 
 
-
-
-/// Rest is copied and pasted to work over 64-byte blocks
-
-template <typename HashFamily = ::hashing::TwoIndependentMultiplyShift>
-class SimdBlockFilter64 {
-private:
-  // The filter is divided up into Buckets:
-  using Bucket = uint32_t[16];
-
-  // log2(number of bytes in a bucket):
-  static constexpr int LOG_BUCKET_BYTE_SIZE = 6;
-
-  static_assert((1 << LOG_BUCKET_BYTE_SIZE) == sizeof(Bucket) &&
-                    sizeof(Bucket) == 2 * sizeof(__m256i),
-                "Bucket sizing has gone awry.");
-
-  // log_num_buckets_ is the log (base 2) of the number of buckets in the
-  // directory:
-  const int log_num_buckets_;
-
-  // directory_mask_ is (1 << log_num_buckets_) - 1. It is precomputed in the
-  // contructor for efficiency reasons:
-  const uint32_t directory_mask_;
-
-  Bucket *directory_;
-
-  HashFamily hasher_;
-
-public:
-  // Consumes at most (1 << log_heap_space) bytes on the heap:
-  explicit SimdBlockFilter64(const int log_heap_space);
-  SimdBlockFilter64(SimdBlockFilter64 &&that)
-      : log_num_buckets_(that.log_num_buckets_),
-        directory_mask_(that.directory_mask_), directory_(that.directory_),
-        hasher_(that.hasher_) {}
-  ~SimdBlockFilter64() noexcept;
-  void Add(const uint64_t key) noexcept;
-  bool Find(const uint64_t key) const noexcept;
-  uint64_t SizeInBytes() const {
-    return sizeof(Bucket) * (1ull << log_num_buckets_);
-  }
-
-private:
-  static void MakeMask(const uint32_t hash, __m256i *out1,
-                       __m256i *out2) noexcept;
-
-  SimdBlockFilter64(const SimdBlockFilter64 &) = delete;
-  void operator=(const SimdBlockFilter64 &) = delete;
-};
-
-template <typename HashFamily>
-SimdBlockFilter64<HashFamily>::SimdBlockFilter64(const int log_heap_space)
-    : // Since log_heap_space is in bytes, we need to convert it to the number
-      // of Buckets we will use.
-      log_num_buckets_(::std::max(1, log_heap_space - LOG_BUCKET_BYTE_SIZE)),
-      // Don't use log_num_buckets_ if it will lead to undefined behavior by a
-      // shift that is too large.
-      directory_mask_((1ull << ::std::min(63, log_num_buckets_)) - 1),
-      directory_(nullptr), hasher_() {
-  if (!__builtin_cpu_supports("avx2")) {
-    throw ::std::runtime_error(
-        "SimdBlockFilter64 does not work without AVX2 instructions");
-  }
-  const size_t alloc_size = 1ull << (log_num_buckets_ + LOG_BUCKET_BYTE_SIZE);
-  const int malloc_failed =
-      posix_memalign(reinterpret_cast<void **>(&directory_), 64, alloc_size);
-  if (malloc_failed)
-    throw ::std::bad_alloc();
-  memset(directory_, 0, alloc_size);
-}
-
-template <typename HashFamily>
-SimdBlockFilter64<HashFamily>::~SimdBlockFilter64() noexcept {
-  free(directory_);
-  directory_ = nullptr;
-}
-
-// with AVX-512, this becomes a single instruction
-static inline __m256i hacked_mm256_mullo_epi64(__m256i x, __m256i ml,
-                                               __m256i mh) {
-  __m256i xl = x;
-  //      _mm256_and_si256(x, _mm256_set1_epi64x(UINT64_C(0x00000000ffffffff)));
-  //  __m256i xh = _mm256_srli_epi64(x, 32);
-  //__m256i hl = _mm256_slli_epi64(_mm256_mul_epu32(xh, ml), 32);
-  __m256i lh = _mm256_slli_epi64(_mm256_mul_epu32(xl, mh), 32);
-  __m256i ll = _mm256_mul_epu32(xl, ml);
-  // return _mm256_add_epi64(ll, _mm256_add_epi64(hl, lh));
-  return _mm256_add_epi64(lh, ll);
-}
-
-// The SIMD reinterpret_casts technically violate C++'s strict aliasing rules.
-// However, we compile with -fno-strict-aliasing.
-template <typename HashFamily>
-[[gnu::always_inline]] inline void
-SimdBlockFilter64<HashFamily>::MakeMask(const uint32_t hash, __m256i *out1,
-                                        __m256i *out2) noexcept {
-  const __m256i ones = _mm256_set1_epi64x(1);
-  // Odd contants for hashing:
-  const __m256i rehash1_l = _mm256_setr_epi64x(
-      0x53214365047b6137 & 0xffffffff, 0x2c5635344974d91 & 0xffffffff,
-      0x7fe299d78824ad5b & 0xffffffff, 0xc01ac48e4d29f115 & 0xffffffff);
-  const __m256i rehash1_h = _mm256_setr_epi64x(
-      UINT64_C(0x53214365047b6137) >> 32, UINT64_C(0x2c5635344974d91) >> 32,
-      UINT64_C(0x7fe299d78824ad5b) >> 32, UINT64_C(0xc01ac48e4d29f115) >> 32);
-
-  const __m256i rehash2_l = _mm256_setr_epi64x(
-      0x7bdeb6734f95e2e3 & 0xffffffff, 0x2ec75a90a4e6ad3d & 0xffffffff,
-      0x3d485cae00ae48fd & 0xffffffff, 0xe7d0f0c09b59d29b & 0xffffffff);
-  const __m256i rehash2_h = _mm256_setr_epi64x(
-      UINT64_C(0x7bdeb6734f95e2e3) >> 32, UINT64_C(0x2ec75a90a4e6ad3d) >> 32,
-      UINT64_C(0x3d485cae00ae48fd) >> 32, UINT64_C(0xe7d0f0c09b59d29b) >> 32);
-
-  __m256i hash_data = _mm256_set1_epi64x(hash);
-
-  // Multiply-shift hashing ala Dietzfelbinger et al.: multiply 'hash' by eight
-  // different odd constants, then keep the 6 most significant bits from each
-  // product.
-  __m256i hash_data1 = hacked_mm256_mullo_epi64(
-      hash_data, rehash1_l,
-      rehash1_h); //_mm256_mullo_epi64(rehash1, hash_data);
-  __m256i hash_data2 = hacked_mm256_mullo_epi64(
-      hash_data, rehash2_l,
-      rehash2_h); //_mm256_mullo_epi64(rehash2, hash_data);
-
-  hash_data1 = _mm256_and_si256(_mm256_srli_epi64(hash_data1, 32),
-                                _mm256_set1_epi64x(63));
-  hash_data2 = _mm256_and_si256(_mm256_srli_epi64(hash_data2, 32),
-                                _mm256_set1_epi64x(63));
-  // Use these 6 bits to shift a single bit to a location in each 32-bit lane
-  *out1 = _mm256_sllv_epi64(ones, hash_data1);
-  *out2 = _mm256_sllv_epi64(ones, hash_data2);
-}
-
-
-template <typename HashFamily>
-[[gnu::always_inline]] inline void
-SimdBlockFilter64<HashFamily>::Add(const uint64_t key) noexcept {
-  const auto hash = hasher_(key);
-  const uint32_t bucket_idx = hash & directory_mask_;
-  __m256i mask1, mask2;
-  MakeMask(hash >> log_num_buckets_, &mask1, &mask2);
-  __m256i *const bucket1 =
-      &reinterpret_cast<__m256i *>(directory_)[2 * bucket_idx];
-  __m256i *const bucket2 =
-      &reinterpret_cast<__m256i *>(directory_)[2 * bucket_idx + 1];
-
-  _mm256_store_si256(bucket1, _mm256_or_si256(*bucket1, mask1));
-  _mm256_store_si256(bucket2, _mm256_or_si256(*bucket2, mask2));
-}
-
-template <typename HashFamily>
-[[gnu::always_inline]] inline bool
-SimdBlockFilter64<HashFamily>::Find(const uint64_t key) const noexcept {
-  const auto hash = hasher_(key);
-  const uint32_t bucket_idx = hash & directory_mask_;
-  __m256i mask1, mask2;
-  MakeMask(hash >> log_num_buckets_, &mask1, &mask2);
-  const __m256i bucket1 =
-      reinterpret_cast<__m256i *>(directory_)[2 * bucket_idx];
-  const __m256i bucket2 =
-      reinterpret_cast<__m256i *>(directory_)[2 * bucket_idx + 1];
-  return _mm256_testc_si256(bucket1, mask1) &
-         _mm256_testc_si256(bucket2, mask2);
-}

src/xorfilter.h

@@ -242,42 +242,6 @@ Status XorFilter<ItemType, FingerprintType, HashFamily>::AddAll(
         delete[] tmpc;
         delete[] alone;
 
-/*
-
-        int* alone = new int[arrayLength];
-        int alonePos = 0;
-        for (size_t i = 0; i < arrayLength; i++) {
-            if (t2vals[i].t2count == 1) {
-                alone[alonePos++] = i;
-            }
-        }
-        reverseOrderPos = 0;
-        while (alonePos > 0 && reverseOrderPos < size) {
-            int i = alone[--alonePos];
-            if (t2vals[i].t2count == 0) {
-                continue;
-            }
-            long hash = t2vals[i].t2;
-            uint8_t found = -1;
-            for (int hi = 0; hi < 3; hi++) {
-                int h = getHashFromHash(hash, hi, blockLength);
-                int newCount =  --t2vals[h].t2count;
-                if (newCount == 0) {
-                    found = (uint8_t) hi;
-                } else {
-                    if (newCount == 1) {
-                        alone[alonePos++] = h;
-                    }
-                    t2vals[h].t2 ^= hash;
-                }
-            }
-            reverseOrder[reverseOrderPos] = hash;
-            reverseH[reverseOrderPos] = found;
-            reverseOrderPos++;
-        }
-        delete [] alone;
-*/
-
         if (reverseOrderPos == size) {
             break;
         }