Skip to content

Commit 469d603

Browse files
thomasmuellerthomasmueller
committed
Support the password hash database from https://haveibeenpwned.com/passwords 
1 parent 0c72e37 commit 469d603

File tree

7 files changed

+299
-1
lines changed

7 files changed

+299
-1
lines changed

‎src/main/java/org/fastfilter/tools/BuildFilterFile.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package org.fastfilter.tools;
2+
3+
import java.io.File;
4+
import java.io.FileReader;
5+
import java.io.IOException;
6+
import java.io.LineNumberReader;
7+
import java.io.RandomAccessFile;
8+
import java.nio.charset.Charset;
9+
import java.util.ArrayList;
10+
11+
import org.fastfilter.utils.StringUtils;
12+
import org.fastfilter.xorplus.XorPlus8;
13+
14+
public class BuildFilterFile {
15+
16+
public static final int SEGMENT_BITS = 4;
17+
18+
public static void main(String... args) throws IOException {
19+
if (args.length != 1) {
20+
System.out.println("Usage: java " + BuildFilterFile.class.getName() + " <textFile>\n"
21+
+ "Builds a .filter file from a text file that contains SHA-1 hashes and counts.");
22+
return;
23+
}
24+
String textFile = args[0];
25+
String filterFileName = textFile + ".filter";
26+
long start = System.nanoTime();
27+
LineNumberReader lineReader = new LineNumberReader(new FileReader(textFile, Charset.forName("LATIN1")));
28+
new File(filterFileName).delete();
29+
RandomAccessFile out = new RandomAccessFile(filterFileName, "rw");
30+
int lines = 0;
31+
// header
32+
out.write(new byte[8 << SEGMENT_BITS]);
33+
int currentSegment = 0;
34+
long lastHash = 0;
35+
ArrayList<Long> keys = new ArrayList<Long>();
36+
while (true) {
37+
String line = lineReader.readLine();
38+
if (line == null) {
39+
break;
40+
}
41+
lines++;
42+
long hash = 0;
43+
for (int i = 0; i < 16; i++) {
44+
hash <<= 4;
45+
hash |= StringUtils.getHex(line.charAt(i));
46+
}
47+
if (lastHash == hash) {
48+
System.out.println("Warning: duplicate hash detected, ignoring: " + line);
49+
continue;
50+
}
51+
lastHash = hash;
52+
int dot = line.lastIndexOf(':');
53+
int count = Integer.parseInt(line, dot + 1, line.length(), 10);
54+
// set the lowest bit to 0
55+
long key = hash ^ (hash & 1);
56+
// if common, set the lowest bit
57+
if (count > 9) {
58+
key |= 1;
59+
}
60+
int segment = (int) (key >>> (64 - SEGMENT_BITS));
61+
if (segment != currentSegment) {
62+
writeSegment(keys, currentSegment, out);
63+
long time = System.nanoTime() - start;
64+
System.out.println("Lines processed: " + lines + " " + (time / lines) + " ns/line");
65+
currentSegment = segment;
66+
}
67+
keys.add(key);
68+
}
69+
writeSegment(keys, currentSegment, out);
70+
lineReader.close();
71+
out.close();
72+
}
73+
74+
private static void writeSegment(ArrayList<Long> keys, int segment, RandomAccessFile out) throws IOException {
75+
long[] array = new long[keys.size()];
76+
for(int i=0; i<keys.size(); i++) {
77+
array[i] = keys.get(i);
78+
}
79+
long start = out.length();
80+
out.seek(segment * 8);
81+
out.writeLong(start);
82+
out.seek(start);
83+
XorPlus8 filter = XorPlus8.construct(array);
84+
out.write(filter.getData());
85+
keys.clear();
86+
}
87+
88+
}

‎src/main/java/org/fastfilter/tools/PasswordLookup.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package org.fastfilter.tools;
2+
3+
import java.io.BufferedInputStream;
4+
import java.io.Console;
5+
import java.io.DataInputStream;
6+
import java.io.FileInputStream;
7+
import java.io.RandomAccessFile;
8+
import java.nio.charset.Charset;
9+
import java.security.MessageDigest;
10+
import java.util.Scanner;
11+
12+
import org.fastfilter.xorplus.XorPlus8;
13+
14+
public class PasswordLookup {
15+
16+
public static void main(String... args) throws Exception {
17+
if (args.length != 1) {
18+
System.out.println("Usage: java " + PasswordLookup.class.getName() + " <filterFileName>\n"
19+
+ "Requires a filter file generated by " + BuildFilterFile.class.getName());
20+
return;
21+
}
22+
String filterFileName = args[0];
23+
Scanner scanner = new Scanner(System.in);
24+
while (true) {
25+
Console console = System.console();
26+
String password;
27+
if (console != null) {
28+
password = new String(console.readPassword("Password? "));
29+
} else {
30+
System.out.println("Password? ");
31+
password = scanner.nextLine();
32+
}
33+
if (password.length() == 0) {
34+
break;
35+
}
36+
testPassword(filterFileName, password);
37+
}
38+
scanner.close();
39+
}
40+
41+
private static void testPassword(String filterFileName, String password) throws Exception {
42+
byte[] passwordBytes = password.getBytes(Charset.forName("ISO-8859-1"));
43+
MessageDigest md = MessageDigest.getInstance("SHA-1");
44+
byte[] sha1 = md.digest(passwordBytes);
45+
long hash = 0;
46+
for (int i = 0; i < 8; i++) {
47+
hash = (hash << 8) | (sha1[i] & 0xff);
48+
}
49+
// set the lowest bit to 0
50+
long key = hash ^ (hash & 1);
51+
RandomAccessFile f = new RandomAccessFile(filterFileName, "r");
52+
int segment = (int) (key >>> (64 - BuildFilterFile.SEGMENT_BITS));
53+
f.seek(segment * 8);
54+
long skip = f.readLong();
55+
f.close();
56+
DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(filterFileName)));
57+
while (skip > 0) {
58+
long skipped = in.skip(skip);
59+
if (skipped <= 0) {
60+
break;
61+
}
62+
skip -= skipped;
63+
}
64+
XorPlus8 filter = new XorPlus8(in);
65+
in.close();
66+
boolean found = filter.mayContain(key);
67+
if (found) {
68+
System.out.println("Found");
69+
} else {
70+
found = filter.mayContain(key | 1);
71+
if (found) {
72+
System.out.println("Found; common");
73+
} else {
74+
System.out.println("Not found");
75+
}
76+
}
77+
}
78+
79+
}

‎src/main/java/org/fastfilter/utils/Hash.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ public class Hash {
66

77
private static Random random = new Random();
88

9+
public static void setSeed(long seed) {
10+
random.setSeed(seed);
11+
}
12+
913
public static long hash64(long x, long seed) {
1014
x += seed;
1115
x = (x ^ (x >>> 33)) * 0xff51afd7ed558ccdL;
@@ -20,7 +24,7 @@ public static long randomSeed() {
2024

2125
/**
2226
* Shrink the hash to a value 0..n. Kind of like modulo, but using
23-
* multiplication.
27+
* multiplication and shift, which are faster to compute.
2428
*
2529
* @param hash the hash
2630
* @param n the maximum of the result

‎src/main/java/org/fastfilter/utils/StringUtils.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package org.fastfilter.utils;
2+
3+
public class StringUtils {
4+
private static final int[] HEX_DECODE = new int['f' + 1];
5+
6+
static {
7+
for (int i = 0; i < HEX_DECODE.length; i++) {
8+
HEX_DECODE[i] = -1;
9+
}
10+
for (int i = 0; i <= 9; i++) {
11+
HEX_DECODE[i + '0'] = i;
12+
}
13+
for (int i = 0; i <= 5; i++) {
14+
HEX_DECODE[i + 'a'] = HEX_DECODE[i + 'A'] = i + 10;
15+
}
16+
}
17+
18+
public static int getHex(char c) {
19+
return HEX_DECODE[c];
20+
}
21+
22+
}

‎src/main/java/org/fastfilter/xor/Xor8.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
package org.fastfilter.xor;
22

3+
import java.io.ByteArrayOutputStream;
4+
import java.io.DataInputStream;
5+
import java.io.DataOutputStream;
6+
import java.io.IOException;
7+
import java.io.InputStream;
8+
39
import org.fastfilter.Filter;
410
import org.fastfilter.utils.Hash;
511

@@ -160,4 +166,37 @@ private int fingerprint(long hash) {
160166
return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1));
161167
}
162168

169+
public byte[] getData() {
170+
try {
171+
ByteArrayOutputStream out = new ByteArrayOutputStream();
172+
DataOutputStream d = new DataOutputStream(out);
173+
d.writeInt(size);
174+
d.writeInt(arrayLength);
175+
d.writeInt(blockLength);
176+
d.writeLong(seed);
177+
d.writeInt(bitCount);
178+
d.writeInt(fingerprints.length);
179+
d.write(fingerprints);
180+
return out.toByteArray();
181+
} catch (IOException e) {
182+
throw new RuntimeException(e);
183+
}
184+
}
185+
186+
public Xor8(InputStream in) {
187+
try {
188+
DataInputStream din = new DataInputStream(in);
189+
size = din.readInt();
190+
arrayLength = din.readInt();
191+
blockLength = din.readInt();
192+
seed = din.readLong();
193+
bitCount = din.readInt();
194+
int fingerprintLength = din.readInt();
195+
fingerprints = new byte[fingerprintLength];
196+
din.readFully(fingerprints);
197+
} catch (IOException e) {
198+
throw new RuntimeException(e);
199+
}
200+
}
201+
163202
}

‎src/main/java/org/fastfilter/xorplus/Rank9.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818
*/
1919
package org.fastfilter.xorplus;
2020

21+
import java.io.DataInputStream;
22+
import java.io.DataOutputStream;
23+
import java.io.IOException;
2124
import java.util.Arrays;
2225
import java.util.BitSet;
2326

@@ -114,4 +117,26 @@ public int getBitCount() {
114117
return bits.length * 64 + counts.length * 64;
115118
}
116119

120+
public void write(DataOutputStream d) throws IOException {
121+
d.writeInt(bits.length);
122+
for (int i = 0; i < bits.length; i++) {
123+
d.writeLong(bits[i]);
124+
}
125+
d.writeInt(counts.length);
126+
for (int i = 0; i < counts.length; i++) {
127+
d.writeLong(counts[i]);
128+
}
129+
}
130+
131+
public Rank9(DataInputStream in) throws IOException {
132+
bits = new long[in.readInt()];
133+
for (int i = 0; i < bits.length; i++) {
134+
bits[i] = in.readLong();
135+
}
136+
counts = new long[in.readInt()];
137+
for (int i = 0; i < counts.length; i++) {
138+
counts[i] = in.readLong();
139+
}
140+
}
141+
117142
}

‎src/main/java/org/fastfilter/xorplus/XorPlus8.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
package org.fastfilter.xorplus;
22

3+
import java.io.ByteArrayOutputStream;
4+
import java.io.DataInputStream;
5+
import java.io.DataOutputStream;
6+
import java.io.IOException;
7+
import java.io.InputStream;
38
import java.util.BitSet;
49

510
import org.fastfilter.Filter;
@@ -339,4 +344,40 @@ private int fingerprint(long hash) {
339344
return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1));
340345
}
341346

347+
public byte[] getData() {
348+
try {
349+
ByteArrayOutputStream out = new ByteArrayOutputStream();
350+
DataOutputStream d = new DataOutputStream(out);
351+
d.writeInt(size);
352+
d.writeInt(arrayLength);
353+
d.writeInt(blockLength);
354+
d.writeLong(seed);
355+
d.writeInt(bitCount);
356+
d.writeInt(fingerprints.length);
357+
d.write(fingerprints);
358+
rank.write(d);
359+
return out.toByteArray();
360+
} catch (IOException e) {
361+
throw new RuntimeException(e);
362+
}
363+
}
364+
365+
public XorPlus8(InputStream in) {
366+
try {
367+
DataInputStream din = new DataInputStream(in);
368+
size = din.readInt();
369+
arrayLength = din.readInt();
370+
blockLength = din.readInt();
371+
seed = din.readLong();
372+
bitCount = din.readInt();
373+
int fingerprintLength = din.readInt();
374+
fingerprints = new byte[fingerprintLength];
375+
din.readFully(fingerprints);
376+
rank = new Rank9(din);
377+
} catch (IOException e) {
378+
throw new RuntimeException(e);
379+
}
380+
}
381+
382+
342383
}

0 commit comments

Comments
 (0)