Infinite loop in XOR filter construction #29

Author: thomasmueller

fastfilter/src/main/java/org/fastfilter/xor/Xor16.java

@@ -16,6 +16,7 @@ public class Xor16 implements Filter {
 
     private static final int BITS_PER_FINGERPRINT = 16;
     private static final int HASHES = 3;
+    private static final int OFFSET = 32;
     private static final int FACTOR_TIMES_100 = 123;
     private final int blockLength;
     private long seed;
@@ -27,7 +28,7 @@ public long getBitCount() {
     }
 
     private static int getArrayLength(int size) {
-        return (int) (HASHES + (long) FACTOR_TIMES_100 * size / 100);
+        return (int) (OFFSET + (long) FACTOR_TIMES_100 * size / 100);
     }
 
     public static Xor16 construct(long[] keys) {

fastfilter/src/main/java/org/fastfilter/xor/Xor8.java

@@ -18,6 +18,7 @@ public class Xor8 implements Filter {
 
     private static final int BITS_PER_FINGERPRINT = 8;
     private static final int HASHES = 3;
+    private static final int OFFSET = 32;
     private static final int FACTOR_TIMES_100 = 123;
     private final int size;
     private final int arrayLength;
@@ -31,7 +32,7 @@ public long getBitCount() {
     }
 
     private static int getArrayLength(int size) {
-        return (int) (HASHES + (long) FACTOR_TIMES_100 * size / 100);
+        return (int) (OFFSET + (long) FACTOR_TIMES_100 * size / 100);
     }
 
     public static Xor8 construct(long[] keys) {

fastfilter/src/main/java/org/fastfilter/xorplus/XorPlus8.java

@@ -27,6 +27,8 @@ public class XorPlus8 implements Filter {
 
     private static final int HASHES = 3;
 
+    private static final int OFFSET = 32;
+
     // the table needs to be 1.23 times the number of keys to store
     // with 2 hashes, we would need 232 (factor 2.32) for a 50% chance,
     // 240 for 55%, 250 for a 60%, 264 for 65%, 282 for 67%, as for
@@ -63,13 +65,13 @@ public long getBitCount() {
     }
 
     /**
-     * Calculate the table (array) length. This is 1.23 times the size.
+     * Calculate the table (array) length. This is 1.23 times the size, plus an offset of 32 (see paper, Fig. 1)
      *
      * @param size the number of entries
      * @return the table length
      */
     private static int getArrayLength(int size) {
-        return (int) (HASHES + (long) FACTOR_TIMES_100 * size / 100);
+        return (int) (OFFSET + (long) FACTOR_TIMES_100 * size / 100);
     }
 
     public static XorPlus8 construct(long[] keys) {

fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java

@@ -0,0 +1,19 @@
+package org.fastfilter.xor;
+
+import org.junit.Test;
+
+/**
+ * Test small sets.
+ */
+
+public class SmallSetTest {
+
+    @Test
+    public void small() {
+        Xor8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
+        Xor16.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
+        XorBinaryFuse8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
+        XorSimple.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
+        XorSimple2.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
+    }
+}

fastfilter/src/test/java/org/fastfilter/xorplus/SmallSetTest.java

@@ -0,0 +1,15 @@
+package org.fastfilter.xorplus;
+
+import org.junit.Test;
+
+/**
+ * Test small sets.
+ */
+
+public class SmallSetTest {
+
+    @Test
+    public void small() {
+        XorPlus8.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
+    }
+}