Speed up password filter generation

thomasmueller · thomasmueller · commit 5ddd3276d0dd · 2019-03-22T09:10:40.000+01:00
diff --git a/src/main/java/org/fastfilter/tools/BuildFilterFile.java b/src/main/java/org/fastfilter/tools/BuildFilterFile.java
@@ -15,13 +15,14 @@
 
 public class BuildFilterFile {
 
-    public static final int SEGMENT_BITS = 4;
+    public static final int SEGMENT_BITS = 10;
 
     public static void main(String... args) throws IOException {
         if (args.length != 1) {
             System.out.println("Usage: java " + BuildFilterFile.class.getName() + " <textFile>\n"
-                    + "Builds a .filter file from a text file that contains SHA-1 hashes and counts.");
-            // see also https://haveibeenpwned.com/passwords
+                    + "Builds a .filter file from a text file that contains SHA-1 hashes and counts.\n"
+                    + "You can get the hash file from https://haveibeenpwned.com/passwords\n"
+                    + "It needs to be a list of SHA-1 hashes, ordered by hash, line format <hash>:<count>.");
             return;
         }
         String textFile = args[0];
@@ -32,6 +33,7 @@ public static void main(String... args) throws IOException {
         new File(filterFileName).delete();
         RandomAccessFile out = new RandomAccessFile(filterFileName, "rw");
         int lines = 0;
+        long[] segmentStarts = new long[1 << SEGMENT_BITS];
         // header
         out.write(new byte[8 << SEGMENT_BITS]);
         int currentSegment = 0;
@@ -45,49 +47,54 @@ public static void main(String... args) throws IOException {
             lines++;
             long hash = 0;
             for (int i = 0; i < 16; i++) {
-                hash <<= 4;
-                hash |= StringUtils.getHex(line.charAt(i));
+                hash = (hash << 4) | StringUtils.getHex(line.charAt(i));
             }
             if (lastHash == hash) {
-                System.out.println("Warning: duplicate hash detected, ignoring: " + line);
+                System.out.println("Warning: duplicate 64-bit key detected, ignoring: " + line);
                 continue;
+            } else if (Long.compareUnsigned(hash, lastHash) < 0) {
+                throw new IllegalArgumentException("The file is not sorted by hash");
             }
             lastHash = hash;
             int dot = line.lastIndexOf(':');
             int count = Integer.parseInt(line.substring(dot + 1), 10);
-            // set the lowest bit to 0
+            // clear the lowest bit
             long key = hash ^ (hash & 1);
             // if common, set the lowest bit
             if (count > 9) {
                 key |= 1;
             }
             int segment = (int) (key >>> (64 - SEGMENT_BITS));
             if (segment != currentSegment) {
-                writeSegment(keys, currentSegment, out);
-                long time = System.nanoTime() - start;
-                System.out.println("Lines processed: " + lines + " " + (time / lines) + " ns/line");
+                segmentStarts[currentSegment] = out.getFilePointer();
+                out.write(getSegment(keys));
+                keys.clear();
                 currentSegment = segment;
             }
+            if (lines % 10000000 == 0) {
+                long time = System.nanoTime() - start;
+                System.out.println(lines / 1000000 + " million lines processed, " + (time / lines) + " ns/line");
+            }
             keys.add(key);
         }
-        writeSegment(keys, currentSegment, out);
+        segmentStarts[currentSegment] = out.getFilePointer();
+        out.write(getSegment(keys));
         lineReader.close();
+        out.seek(0);
+        for(long s : segmentStarts) {
+            out.writeLong(s);
+        }
         out.close();
+        long time = System.nanoTime() - start;
+        System.out.println(lines + " lines processed, " + (time / 1000000 / 1000) + " seconds");
     }
 
-    private static void writeSegment(ArrayList<Long> keys, int segment,
-            RandomAccessFile out) throws IOException {
+    private static byte[] getSegment(ArrayList<Long> keys) {
         long[] array = new long[keys.size()];
-        for(int i=0; i<keys.size(); i++) {
+        for (int i = 0; i < keys.size(); i++) {
             array[i] = keys.get(i);
         }
-        long start = out.length();
-        out.seek(segment * 8);
-        out.writeLong(start);
-        out.seek(start);
-        XorPlus8 filter = XorPlus8.construct(array);
-        out.write(filter.getData());
-        keys.clear();
+        return XorPlus8.construct(array).getData();
     }
 
 }
diff --git a/src/main/java/org/fastfilter/tools/PasswordLookup.java b/src/main/java/org/fastfilter/tools/PasswordLookup.java
@@ -15,7 +15,7 @@ public class PasswordLookup {
 
     public static void main(String... args) throws Exception {
         if (args.length != 1) {
-            System.out.println("Usage: java " + PasswordLookup.class.getName() + " <filterFileName>\n"
+            System.out.println("Usage: java " + PasswordLookup.class.getName() + " <filterFileName> \n"
                     + "Requires a filter file generated by " + BuildFilterFile.class.getName());
             return;
         }
@@ -39,7 +39,8 @@ public static void main(String... args) throws Exception {
     }
 
     private static void testPassword(String filterFileName, String password) throws Exception {
-        byte[] passwordBytes = password.getBytes(Charset.forName("ISO-8859-1"));
+        // it's unclear which character set was used; ASCII gave good results, as umlauts are converted to '?'
+        byte[] passwordBytes = password.getBytes(Charset.forName("ASCII"));
         MessageDigest md = MessageDigest.getInstance("SHA-1");
         byte[] sha1 = md.digest(passwordBytes);
         long hash = 0;
@@ -63,16 +64,12 @@ private static void testPassword(String filterFileName, String password) throws
         }
         XorPlus8 filter = new XorPlus8(in);
         in.close();
-        boolean found = filter.mayContain(key);
-        if (found) {
+        if (filter.mayContain(key)) {
             System.out.println("Found");
+        } else if (filter.mayContain(key | 1)) {
+            System.out.println("Found; common");
         } else {
-            found = filter.mayContain(key | 1);
-            if (found) {
-                System.out.println("Found; common");
-            } else {
-                System.out.println("Not found");
-            }
+            System.out.println("Not found");
         }
     }
 
diff --git a/src/main/java/org/fastfilter/xor/Xor8.java b/src/main/java/org/fastfilter/xor/Xor8.java
@@ -171,11 +171,7 @@ public byte[] getData() {
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             DataOutputStream d = new DataOutputStream(out);
             d.writeInt(size);
-            d.writeInt(arrayLength);
-            d.writeInt(blockLength);
             d.writeLong(seed);
-            d.writeInt(bitCount);
-            d.writeInt(fingerprints.length);
             d.write(fingerprints);
             return out.toByteArray();
         } catch (IOException e) {
@@ -187,12 +183,11 @@ public Xor8(InputStream in) {
         try {
             DataInputStream din = new DataInputStream(in);
             size = din.readInt();
-            arrayLength = din.readInt();
-            blockLength = din.readInt();
+            arrayLength = getArrayLength(size);
+            bitCount = arrayLength * BITS_PER_FINGERPRINT;
+            blockLength = arrayLength / HASHES;
             seed = din.readLong();
-            bitCount = din.readInt();
-            int fingerprintLength = din.readInt();
-            fingerprints = new byte[fingerprintLength];
+            fingerprints = new byte[arrayLength];
             din.readFully(fingerprints);
         } catch (IOException e) {
             throw new RuntimeException(e);
diff --git a/src/main/java/org/fastfilter/xorplus/Rank9.java b/src/main/java/org/fastfilter/xorplus/Rank9.java
@@ -45,7 +45,7 @@ public Rank9(BitSet set, long bitCount) {
         this.bits = bits;
         long length = bits.length * 64;
         int numWords = (int) ((length + 63) / 64);
-        final int numCounts = (int) ((length + 8 * 64 - 1) / (8 * 64)) * 2;
+        int numCounts = (int) ((length + 8 * 64 - 1) / (8 * 64)) * 2;
         counts = new long[numCounts + 1];
         long c = 0;
         int pos = 0;
diff --git a/src/main/java/org/fastfilter/xorplus/XorPlus8.java b/src/main/java/org/fastfilter/xorplus/XorPlus8.java
@@ -349,10 +349,7 @@ public byte[] getData() {
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             DataOutputStream d = new DataOutputStream(out);
             d.writeInt(size);
-            d.writeInt(arrayLength);
-            d.writeInt(blockLength);
             d.writeLong(seed);
-            d.writeInt(bitCount);
             d.writeInt(fingerprints.length);
             d.write(fingerprints);
             rank.write(d);
@@ -366,10 +363,10 @@ public XorPlus8(InputStream in) {
         try {
             DataInputStream din = new DataInputStream(in);
             size = din.readInt();
-            arrayLength = din.readInt();
-            blockLength = din.readInt();
+            arrayLength = getArrayLength(size);
+            bitCount = arrayLength * BITS_PER_FINGERPRINT;
+            blockLength = arrayLength / HASHES;
             seed = din.readLong();
-            bitCount = din.readInt();
             int fingerprintLength = din.readInt();
             fingerprints = new byte[fingerprintLength];
             din.readFully(fingerprints);
