Password filter generator / lookup improvements

Author: thomasmueller

src/main/java/org/fastfilter/tools/BuildFilterFile.java

@@ -1,12 +1,12 @@
 package org.fastfilter.tools;
 
 import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataOutputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
-import java.io.RandomAccessFile;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 
@@ -19,23 +19,19 @@ public class BuildFilterFile {
 
     public static void main(String... args) throws IOException {
         if (args.length != 1) {
-            System.out.println("Usage: java " + BuildFilterFile.class.getName() + " <textFile>\n"
-                    + "Builds a .filter file from a text file that contains SHA-1 hashes and counts.\n"
-                    + "You can get the hash file from https://haveibeenpwned.com/passwords\n"
-                    + "It needs to be a list of SHA-1 hashes, ordered by hash, line format <hash>:<count>.");
+            System.out.println("Usage: java " + BuildFilterFile.class.getName() + " <filterFile>\n"
+                    + "Reads a text file from standard in and writes a filter file.\n"
+                    + "The input is a text file that contains sorted SHA-1 hashes and counts.\n"
+                    + "You may get the file from https://haveibeenpwned.com/passwords.");
             return;
         }
-        String textFile = args[0];
-        String filterFileName = textFile + ".filter";
+        LineNumberReader lineReader = new LineNumberReader(
+                new InputStreamReader(new BufferedInputStream(System.in), Charset.forName("LATIN1")));
         long start = System.nanoTime();
-        LineNumberReader lineReader = new LineNumberReader(new InputStreamReader(
-                new BufferedInputStream(new FileInputStream(textFile)), Charset.forName("LATIN1")));
-        new File(filterFileName).delete();
-        RandomAccessFile out = new RandomAccessFile(filterFileName, "rw");
+        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(args[0])));
+        long outPos = 0;
         int lines = 0;
         long[] segmentStarts = new long[1 << SEGMENT_BITS];
-        // header
-        out.write(new byte[8 << SEGMENT_BITS]);
         int currentSegment = 0;
         long lastHash = 0;
         ArrayList<Long> keys = new ArrayList<Long>();
@@ -66,8 +62,10 @@ public static void main(String... args) throws IOException {
             }
             int segment = (int) (key >>> (64 - SEGMENT_BITS));
             if (segment != currentSegment) {
-                segmentStarts[currentSegment] = out.getFilePointer();
-                out.write(getSegment(keys));
+                segmentStarts[currentSegment] = outPos;
+                byte[] data = getSegment(keys);
+                out.write(data);
+                outPos += data.length;
                 keys.clear();
                 currentSegment = segment;
             }
@@ -77,11 +75,10 @@ public static void main(String... args) throws IOException {
             }
             keys.add(key);
         }
-        segmentStarts[currentSegment] = out.getFilePointer();
+        segmentStarts[currentSegment] = outPos;
         out.write(getSegment(keys));
         lineReader.close();
-        out.seek(0);
-        for(long s : segmentStarts) {
+        for (long s : segmentStarts) {
             out.writeLong(s);
         }
         out.close();

src/main/java/org/fastfilter/tools/PasswordLookup.java

@@ -3,8 +3,8 @@
 import java.io.BufferedInputStream;
 import java.io.Console;
 import java.io.DataInputStream;
+import java.io.File;
 import java.io.FileInputStream;
-import java.io.RandomAccessFile;
 import java.nio.charset.Charset;
 import java.security.MessageDigest;
 import java.util.Scanner;
@@ -15,11 +15,17 @@ public class PasswordLookup {
 
     public static void main(String... args) throws Exception {
         if (args.length != 1) {
-            System.out.println("Usage: java " + PasswordLookup.class.getName() + " <filterFileName> \n"
+            System.out.println("Usage: java " + PasswordLookup.class.getName() + " <filterFile> \n"
                     + "Requires a filter file generated by " + BuildFilterFile.class.getName());
             return;
         }
-        String filterFileName = args[0];
+        String filterFile = args[0];
+        DataInputStream in = openFile(filterFile, new File(filterFile).length() - (8 << BuildFilterFile.SEGMENT_BITS));
+        long[] segmentStarts = new long[1 << BuildFilterFile.SEGMENT_BITS];
+        for (int i = 0; i < segmentStarts.length; i++) {
+            segmentStarts[i] = in.readLong();
+        }
+        in.close();
         Scanner scanner = new Scanner(System.in);
         while (true) {
             Console console = System.console();
@@ -33,13 +39,14 @@ public static void main(String... args) throws Exception {
             if (password.length() == 0) {
                 break;
             }
-            testPassword(filterFileName, password);
+            testPassword(filterFile, segmentStarts, password);
         }
         scanner.close();
     }
 
-    private static void testPassword(String filterFileName, String password) throws Exception {
-        // it's unclear which character set was used; ASCII gave good results, as umlauts are converted to '?'
+    private static void testPassword(String filterFile, long[] segmentStarts, String password) throws Exception {
+        // it's unclear which character set was used; ASCII gave good results, as
+        // umlauts are converted to '?'
         byte[] passwordBytes = password.getBytes(Charset.forName("ASCII"));
         MessageDigest md = MessageDigest.getInstance("SHA-1");
         byte[] sha1 = md.digest(passwordBytes);
@@ -49,19 +56,8 @@ private static void testPassword(String filterFileName, String password) throws
         }
         // set the lowest bit to 0
         long key = hash ^ (hash & 1);
-        RandomAccessFile f = new RandomAccessFile(filterFileName, "r");
         int segment = (int) (key >>> (64 - BuildFilterFile.SEGMENT_BITS));
-        f.seek(segment * 8);
-        long skip = f.readLong();
-        f.close();
-        DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(filterFileName)));
-        while (skip > 0) {
-            long skipped = in.skip(skip);
-            if (skipped <= 0) {
-                break;
-            }
-            skip -= skipped;
-        }
+        DataInputStream in = openFile(filterFile, segmentStarts[segment]);
         XorPlus8 filter = new XorPlus8(in);
         in.close();
         if (filter.mayContain(key)) {
@@ -73,4 +69,16 @@ private static void testPassword(String filterFileName, String password) throws
         }
     }
 
+    static DataInputStream openFile(String fileName, long skip) throws Exception {
+        DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName)));
+        while (skip > 0) {
+            long skipped = in.skip(skip);
+            if (skipped <= 0) {
+                break;
+            }
+            skip -= skipped;
+        }
+        return in;
+    }
+
 }