Improved succinct counting blocked Bloom filter

Author: thomasmueller

src/main/java/org/fastfilter/FilterType.java

@@ -5,6 +5,7 @@
 import org.fastfilter.bloom.Bloom;
 import org.fastfilter.bloom.CountingBloom;
 import org.fastfilter.bloom.SuccinctCountingBlockedBloom;
+import org.fastfilter.bloom.SuccinctCountingBlockedBloomV2;
 import org.fastfilter.bloom.SuccinctCountingBloom;
 import org.fastfilter.cuckoo.Cuckoo16;
 import org.fastfilter.cuckoo.Cuckoo8;
@@ -59,6 +60,12 @@ public Filter construct(long[] keys, int setting) {
             return SuccinctCountingBlockedBloom.construct(keys, setting);
         }
     },
+    SUCCINCT_COUNTING_BLOCKED_BLOOM_V2 {
+        @Override
+        public Filter construct(long[] keys, int setting) {
+            return SuccinctCountingBlockedBloomV2.construct(keys, setting);
+        }
+    },
     XOR_SIMPLE {
         @Override
         public Filter construct(long[] keys, int setting) {

src/main/java/org/fastfilter/bloom/BlockedBloomV2.java

@@ -4,10 +4,10 @@
 import org.fastfilter.utils.Hash;
 
 /**
- * A special kind of blocked Bloom filter. It sets 6 bits in 3 consecutive
- * 64-bit words, and exactly 2 bits per word. It is faster than a regular Bloom
- * filter, but needs slightly more space / has a slightly worse false positive
- * rate.
+ * A special kind of blocked Bloom filter. It sets 2 to 4 (usually 4) bits in
+ * two 64-bit words; 1 or 2 (usually 2) per word. It is faster than a regular
+ * Bloom filter, but needs slightly more space / has a slightly worse false
+ * positive rate.
  */
 public class BlockedBloomV2 implements Filter {
 
@@ -20,7 +20,7 @@ public static BlockedBloomV2 construct(long[] keys, int bitsPerKey) {
         return f;
     }
 
-    private final int blocks;
+    private final int buckets;
     private final long seed;
     private final long[] data;
 
@@ -29,11 +29,12 @@ public long getBitCount() {
     }
 
     BlockedBloomV2(int entryCount, int bitsPerKey) {
+        // bitsPerKey = 11;
         entryCount = Math.max(1, entryCount);
         this.seed = Hash.randomSeed();
         long bits = (long) entryCount * bitsPerKey;
-        this.blocks = (int) bits / 64;
-        data = new long[(int) (blocks + 8)];
+        this.buckets = (int) bits / 64;
+        data = new long[(int) (buckets + 16)];
     }
 
     @Override
@@ -44,31 +45,24 @@ public boolean supportsAdd() {
     @Override
     public void add(long key) {
         long hash = Hash.hash64(key, seed);
-        int start = Hash.reduce((int) hash, blocks);
-        int a = (int) hash;
-        int b = (int) (hash >>> 32);
-        for (int i = 0; i < 3; i++) {
-            a += b;
-            data[start] |= (1L << a) | (1L << (a >> 8));
-            start++;
-        }
+        int start = Hash.reduce((int) hash, buckets);
+        hash = hash ^ Long.rotateLeft(hash, 32);
+        long m1 = (1L << hash) | (1L << (hash >> 6));
+        long m2 = (1L << (hash >> 12)) | (1L << (hash >> 18));
+        data[start] |= m1;
+        data[start + 1 + (int) (hash >>> 60)] |= m2;
     }
 
     @Override
     public boolean mayContain(long key) {
         long hash = Hash.hash64(key, seed);
-        int start = Hash.reduce((int) hash, blocks);
-        int a = (int) hash;
-        int b = (int) (hash >>> 32);
-        for (int i = 0; i < 3; i++) {
-            a += b;
-            long x = data[start];
-            if (((x >> a) & (x >> (a >> 8)) & 1) == 0) {
-                return false;
-            }
-            start++;
-        }
-        return true;
+        int start = Hash.reduce((int) hash, buckets);
+        hash = hash ^ Long.rotateLeft(hash, 32);
+        long a = data[start];
+        long b = data[start + 1 + (int) (hash >>> 60)];
+        long m1 = (1L << hash) | (1L << (hash >> 6));
+        long m2 = (1L << (hash >> 12)) | (1L << (hash >> 18));
+        return ((m1 & a) == m1) && ((m2 & b) == m2);
     }
 
 }

src/main/java/org/fastfilter/bloom/SuccinctCountingBlockedBloom.java

@@ -142,7 +142,7 @@ public long cardinality() {
         }
         long sum = 0;
         for (int i = 0; i < data.capacity(); i++) {
-            sum += data.get(i);
+            sum += Long.bitCount(data.get(i));
         }
         for(long x : counts) {
             sum += Long.bitCount(x);