adding 16-byte vectorized bloom version (not very good).

Author: lemire

benchmarks/bulk-insert-and-query.cc

@@ -211,6 +211,26 @@ struct FilterAPI<SimdBlockFilterFixed64<HashFamily>> {
 };
 
 
+template <typename HashFamily>
+struct FilterAPI<SimdBlockFilterFixed16<HashFamily>> {
+  using Table = SimdBlockFilterFixed16<HashFamily>;
+  static Table ConstructFromAddCount(size_t add_count) {
+    Table ans(ceil(add_count * 8.0 / CHAR_BIT));
+    return ans;
+  }
+  static void Add(uint64_t key, Table* table) {
+    table->Add(key);
+  }
+  static void AddAll(const vector<uint64_t> keys, const size_t start, const size_t end, Table* table) {
+     throw std::runtime_error("Unsupported");
+  }
+
+  CONTAIN_ATTRIBUTES
+  static bool Contain(uint64_t key, const Table * table) {
+    return table->Find(key);
+  }
+};
+
 template <typename HashFamily>
 struct FilterAPI<SimdBlockFilterFixed<HashFamily>> {
   using Table = SimdBlockFilterFixed<HashFamily>;
@@ -683,7 +703,7 @@ int main(int argc, char * argv[]) {
    {14,"GCS"}, {15,"CQF"}, {22, "Xor10 (NBitArray)"}, {23, "Xor14 (NBitArray)"}, 
    {25, "Xor10"},{26, "Xor10.666"}, {37,"Bloom8 (addall)"},
    {38,"Bloom12 (addall)"},{39,"Bloom16 (addall)"},
-   {40,"BlockedBloom (addall)"}, {64,"BlockedBloom64"}
+   {40,"BlockedBloom (addall)"}, {63,"BlockedBloom16"}, {64,"BlockedBloom64"}
   };
 
   if (argc < 2) {
@@ -905,6 +925,15 @@ int main(int argc, char * argv[]) {
           add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed);
       cout << setw(NAME_WIDTH) << names[64] << cf << endl;
   }
+
+#endif
+#ifdef __SSSE3__
+
+  if (algorithmId == 63 || algorithmId < 0 || (algos.find(63) != algos.end())) {
+      auto cf = FilterBenchmark<SimdBlockFilterFixed16<SimpleMixSplit>>(
+          add_count, to_add, distinct_add, to_lookup, distinct_lookup, intersectionsize, hasduplicates, mixed_sets, seed);
+      cout << setw(NAME_WIDTH) << names[63] << cf << endl;
+  }
 #endif
 
   if (algorithmId == 11 || algorithmId < 0 || (algos.find(11) != algos.end())) {

src/simd-block-fixed-fpp.h

@@ -18,7 +18,7 @@
 #include <algorithm>
 #include <new>
 
-#include <immintrin.h>
+#include <x86intrin.h>
 
 #include "hashutil.h"
 
@@ -39,6 +39,8 @@ inline uint64_t rotl64(uint64_t n, unsigned int c) {
     return (n << c) | ( n >> ((-c) & mask));
 }
 
+#ifdef __AVX2__
+
 template<typename HashFamily = ::hashing::TwoIndependentMultiplyShift>
 class SimdBlockFilterFixed {
  private:
@@ -184,7 +186,10 @@ SimdBlockFilterFixed<HashFamily>::Find(const uint64_t key) const noexcept {
   return _mm256_testc_si256(bucket, mask);
 }
 
+///////////////////////////////////////////////////////////////////
 /// 64-byte version
+///////////////////////////////////////////////////////////////////
+
 
 struct mask64bytes {
     __m256i first;
@@ -270,7 +275,6 @@ SimdBlockFilterFixed64<HashFamily>::Add(const uint64_t key) noexcept {
   mask64bytes_t* const bucket = &reinterpret_cast<mask64bytes_t*>(directory_)[bucket_idx];
   bucket->first = _mm256_or_si256(mask.first, bucket->first);
   bucket->second= _mm256_or_si256(mask.second, bucket->second);
-  assert(Find(key));
 }
 
 template <typename HashFamily>
@@ -282,3 +286,269 @@ SimdBlockFilterFixed64<HashFamily>::Find(const uint64_t key) const noexcept {
   const mask64bytes_t  bucket = reinterpret_cast<mask64bytes_t*>(directory_)[bucket_idx];
   return _mm256_testc_si256(bucket.first, mask.first) & _mm256_testc_si256(bucket.second, mask.second);
 }
+
+#endif //__AVX2__
+
+///////////////////
+// 32-bit version ARM
+//////////////////
+#ifdef __aarch64__
+
+struct mask32bytes {
+    uint32x4_t first;
+    uint32x4_t second;
+};
+
+typedef struct mask32bytes mask32bytes_t;
+
+
+template<typename HashFamily = ::hashing::TwoIndependentMultiplyShift>
+class SimdBlockFilterFixed {
+ private:
+  // The filter is divided up into Buckets:
+  using Bucket = mask32bytes_t;
+
+  const int bucketCount;
+
+  Bucket* directory_;
+
+  HashFamily hasher_;
+
+ public:
+  // Consumes at most (1 << log_heap_space) bytes on the heap:
+  explicit SimdBlockFilterFixed(const int bits);
+  ~SimdBlockFilterFixed() noexcept;
+  void Add(const uint64_t key) noexcept;
+
+  // Add multiple items to the filter.
+  void AddAll(const vector<uint64_t> data, const size_t start, const size_t end);
+
+  bool Find(const uint64_t key) const noexcept;
+  uint64_t SizeInBytes() const { return sizeof(Bucket) * bucketCount; }
+
+ private:
+  // A helper function for Insert()/Find(). Turns a 32-bit hash into a 256-bit Bucket
+  // with 1 single 1-bit set in each 32-bit lane.
+  static mask32bytes_t MakeMask(const uint32_t hash) noexcept;
+
+  void ApplyBlock(uint64_t* tmp, int block, int len);
+
+};
+
+template<typename HashFamily>
+SimdBlockFilterFixed<HashFamily>::SimdBlockFilterFixed(const int bits)
+    // bits / 16: fpp 0.1777%, 75.1%
+    // bits / 20: fpp 0.4384%, 63.4%
+    // bits / 22: fpp 0.6692%, 61.1%
+    // bits / 24: fpp 0.9765%, 59.7% <<== seems to be best (1% fpp seems important)
+    // bits / 26: fpp 1.3769%, 59.3%
+    // bits / 28: fpp 1.9197%, 60.3%
+    // bits / 32: fpp 3.3280%, 63.0%
+  : bucketCount(::std::max(1, bits / 24)),
+    directory_(nullptr),
+    hasher_() {
+  const size_t alloc_size = bucketCount * sizeof(Bucket);
+  const int malloc_failed =
+      posix_memalign(reinterpret_cast<void**>(&directory_), 64, alloc_size);
+  if (malloc_failed) throw ::std::bad_alloc();
+  memset(directory_, 0, alloc_size);
+}
+
+template<typename HashFamily>
+SimdBlockFilterFixed<HashFamily>::~SimdBlockFilterFixed() noexcept {
+  free(directory_);
+  directory_ = nullptr;
+}
+
+// The SIMD reinterpret_casts technically violate C++'s strict aliasing rules. However, we
+// compile with -fno-strict-aliasing.
+template <typename HashFamily>
+[[gnu::always_inline]] inline mask32bytes_t
+SimdBlockFilterFixed<HashFamily>::MakeMask(const uint32_t hash) noexcept {
+  const uint32x4_t ones = {1,1,1,1};
+  // Odd contants for hashing:
+  const uint32x4_t rehash1 = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU,
+      0xa2b7289dU};
+  const uint32x4_t rehash2 = {0x705495c7U, 0x2df1424bU, 0x9efc4947U, 0x5c6bfb31U};
+  uint32x4_t hash_data = {hash,hash,hash,hash};
+  // Multiply-shift hashing ala Dietzfelbinger et al.: multiply 'hash' by eight different
+  // odd constants, then keep the 5 most significant bits from each product.
+  uint32x4_t part1 = vmulq_u32(hash_data,rehash1);
+  uint32x4_t part2 = vmulq_u32(hash_data,rehash2);
+  part1 = vshrq_n_u32(part1, 27);
+  part2 = vshrq_n_u32(part2, 27);
+  vshlq_u32(ones, part1);
+  vshlq_u32(ones, part2);
+  mask32bytes_t answer;
+  answer.first = part1;
+  answer.second = part2;
+  return answer;
+}
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline void
+SimdBlockFilterFixed<HashFamily>::Add(const uint64_t key) noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  const mask32bytes_t mask = MakeMask(hash);
+  mask32bytes_t bucket = directory_[bucket_idx];
+  bucket.first = vorrq_u32(mask.first, bucket.first);
+  bucket.second = vorrq_u32(mask.second, bucket.second);
+  directory_[bucket_idx] = bucket;
+}
+
+const int blockShift = 14;
+const int blockLen = 1 << blockShift;
+
+template<typename HashFamily>
+void SimdBlockFilterFixed<HashFamily>::ApplyBlock(uint64_t* tmp, int block, int len) {
+    for (int i = 0; i < len; i += 2) {
+        uint64_t hash = tmp[(block << blockShift) + i];
+        uint32_t bucket_idx = tmp[(block << blockShift) + i + 1];
+        const mask32bytes_t mask = MakeMask(hash);
+
+        mask32bytes_t bucket = directory_[bucket_idx];
+        bucket.first = vorrq_u32(mask.first, bucket.first);
+        bucket.second = vorrq_u32(mask.second, bucket.second);
+        directory_[bucket_idx] = bucket;
+    }
+}
+
+template<typename HashFamily>
+void SimdBlockFilterFixed<HashFamily>::AddAll(
+    const vector<uint64_t> keys, const size_t start, const size_t end) {
+    int blocks = 1 + bucketCount / blockLen;
+    uint64_t* tmp = new uint64_t[blocks * blockLen];
+    int* tmpLen = new int[blocks]();
+    for(size_t i = start; i < end; i++) {
+        uint64_t key = keys[i];
+        uint64_t hash = hasher_(key);
+        uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+        int block = bucket_idx >> blockShift;
+        int len = tmpLen[block];
+        tmp[(block << blockShift) + len] = hash;
+        tmp[(block << blockShift) + len + 1] = bucket_idx;
+        tmpLen[block] = len + 2;
+        if (len + 2 == blockLen) {
+            ApplyBlock(tmp, block, len + 1);
+            tmpLen[block] = 0;
+        }
+    }
+    for (int block = 0; block < blocks; block++) {
+        ApplyBlock(tmp, block, tmpLen[block]);
+    }
+    delete[] tmp;
+    delete[] tmpLen;
+}
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline bool
+SimdBlockFilterFixed<HashFamily>::Find(const uint64_t key) const noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  const mask32bytes_t mask = MakeMask(hash);
+  const mask32bytes_t bucket = directory_[bucket_idx];
+  uint32x4_t an1 = vbicq_u32(bucket.first,mask.first);
+  uint32x4_t an2 = vbicq_u32(bucket.second,mask.second);
+  uint32x4_t an = vorrq_u32(an1, an2);
+  uint64x2_t v64 = vreinterpretq_u64_u32(an);
+  uint32x2_t v32 = vqmovn_u64(v64);
+  uint64x1_t result = vreinterpret_u64_u32(v32);
+  return vget_lane_u64(result, 0);
+}
+
+
+
+#endif // __aarch64__
+
+
+///////////////////////////////////////////////////////////////////
+/// 16-byte version (not very good)
+///////////////////////////////////////////////////////////////////
+
+#ifdef __SSSE3__
+
+template<typename HashFamily = ::hashing::TwoIndependentMultiplyShift>
+class SimdBlockFilterFixed16 {
+ private:
+  // The filter is divided up into Buckets:
+  using Bucket = __m128i;
+
+  const int bucketCount;
+
+  Bucket* directory_;
+
+  HashFamily hasher_;
+
+ public:
+  // Consumes at most (1 << log_heap_space) bytes on the heap:
+  explicit SimdBlockFilterFixed16(const int bits);
+  ~SimdBlockFilterFixed16() noexcept;
+  void Add(const uint64_t key) noexcept;
+
+  bool Find(const uint64_t key) const noexcept;
+  uint64_t SizeInBytes() const { return sizeof(Bucket) * bucketCount; }
+
+ private:
+  static __m128i MakeMask(const uint64_t hash) noexcept;
+
+
+};
+
+template<typename HashFamily>
+SimdBlockFilterFixed16<HashFamily>::SimdBlockFilterFixed16(const int bits)
+
+  : bucketCount(::std::max(1, bits / 10)),
+    directory_(nullptr),
+    hasher_() {
+  const size_t alloc_size = bucketCount * sizeof(Bucket);
+  const int malloc_failed =
+      posix_memalign(reinterpret_cast<void**>(&directory_), 64, alloc_size);
+  if (malloc_failed) throw ::std::bad_alloc();
+  memset(directory_, 0, alloc_size);
+}
+
+template<typename HashFamily>
+SimdBlockFilterFixed16<HashFamily>::~SimdBlockFilterFixed16() noexcept {
+  free(directory_);
+  directory_ = nullptr;
+}
+
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline __m128i
+SimdBlockFilterFixed16<HashFamily>::MakeMask(const uint64_t hash) noexcept {
+  const __m128i rehash1 = _mm_setr_epi16(0x47b5, 0x4497, 0x8823,
+      0xa2b7, 0x7053, 0x2df1, 0x9efc, 0x5c6b);
+  __m128i hash_data = _mm_set1_epi32(hash );
+  __m128i h = _mm_mulhi_epi16(rehash1, hash_data);
+return _mm_shuffle_epi8(_mm_set_epi8(1,2,4,8,16,32,64,-128,1,2,4,8,16,32,64,-128),h); 
+}
+
+
+
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline void
+SimdBlockFilterFixed16<HashFamily>::Add(const uint64_t key) noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  __m128i mask = MakeMask(hash);
+  __m128i* const bucket = reinterpret_cast<__m128i*>(directory_) + bucket_idx;
+  __m128i bucketvalue = _mm_loadu_si128(bucket);
+  bucketvalue = _mm_or_si128(bucketvalue, mask);
+  _mm_storeu_si128(bucket,bucketvalue);
+}
+
+template <typename HashFamily>
+[[gnu::always_inline]] inline bool
+SimdBlockFilterFixed16<HashFamily>::Find(const uint64_t key) const noexcept {
+  const auto hash = hasher_(key);
+  const uint32_t bucket_idx = reduce(rotl64(hash, 32), bucketCount);
+  const __m128i mask = MakeMask(hash);
+  __m128i* const bucket = reinterpret_cast<__m128i*>(directory_) + bucket_idx;
+  __m128i bucketvalue = _mm_loadu_si128(bucket);
+  return _mm_testc_si128(bucketvalue,mask);
+}
+
+#endif // #ifdef __SSSE3__