Skip to content

Commit ae7c63d

Browse files
longbailongbai
committed
upgrade to datx
1 parent c4f1e08 commit ae7c63d

File tree

3 files changed

+133
-36
lines changed

3 files changed

+133
-36
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ hs_err_pid*
1717

1818
build/
1919
.gradle/
20+
21+
*.dat
22+
*.datx

‎src/main/java/qiniu/ip17mon/Locator.java‎

Lines changed: 109 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -7,36 +7,65 @@
77
import java.util.InputMismatchException;
88

99
public final class Locator implements ILocator {
10-
public static final String VERSION = "0.1.2";
10+
public static final String VERSION = "0.2.0";
1111
private static final Charset Utf8 = Charset.forName("UTF-8");
1212
private final byte[] ipData;
1313
private final int textOffset;
1414
private final int[] index;
15-
private final int[] indexData1;
16-
private final int[] indexData2;
17-
private final byte[] indexData3;
15+
private final int[] indexData;
16+
private final int[] textStartIndex;
17+
private final short[] textLengthIndex;
18+
public final boolean x;
1819

1920

2021
private Locator(byte[] data) {
21-
this.ipData = data;
22-
this.textOffset = bigEndian(data, 0);
22+
this(data, (data[4] == 0 && data[5] == 0 && data[6] == 0 && data[7] == 0) && data[8] == 0 && data[9] == 0);
23+
}
2324

25+
private Locator(byte[] data, boolean x) {
26+
this.x = x;
27+
this.ipData = data;
28+
int offset = bigEndian(data, 0);
2429
this.index = new int[256];
25-
for (int i = 0; i < 256; i++) {
26-
index[i] = littleEndian(data, 4 + i * 4);
27-
}
30+
if (x) {
31+
32+
textOffset = offset - 256 * 256 * 4;
2833

29-
int nidx = (textOffset - 4 - 1024 - 1024) / 8;
30-
indexData1 = new int[nidx];
31-
indexData2 = new int[nidx];
32-
indexData3 = new byte[nidx];
34+
for (int i = 0; i < 256; i++) {
35+
index[i] = littleEndian(data, 4 + i * 4 * 256);
36+
}
37+
38+
int nidx = (textOffset - 4 - 256 * 256 * 4) / 9;
39+
indexData = new int[nidx];
40+
textStartIndex = new int[nidx];
41+
textLengthIndex = new short[nidx];
42+
43+
for (int i = 0, off = 0; i < nidx; i++) {
44+
off = 4 + 256 * 256 * 4 + i * 9;
45+
indexData[i] = bigEndian(ipData, off);
46+
textStartIndex[i] = ((int) ipData[off + 6] & 0xff) << 16 | ((int) ipData[off + 5] & 0xff) << 8
47+
| ((int) ipData[off + 4] & 0xff);
48+
textLengthIndex[i] = (short) (((int) ipData[off + 7] & 0xff) << 8 | ((int) ipData[off + 8] & 0xff));
49+
}
50+
} else {
51+
52+
for (int i = 0; i < 256; i++) {
53+
index[i] = littleEndian(data, 4 + i * 4);
54+
}
55+
textOffset = offset - 1024;
3356

34-
for (int i = 0, off = 0; i < nidx; i++) {
35-
off = 4 + 1024 + i * 8;
36-
indexData1[i] = bigEndian(ipData, off);
37-
indexData2[i] = ((int) ipData[off + 6] & 0xff) << 16 | ((int) ipData[off + 5] & 0xff) << 8
38-
| ((int) ipData[off + 4] & 0xff);
39-
indexData3[i] = ipData[off + 7];
57+
int nIdx = (textOffset - 4 - 1024) / 8;
58+
indexData = new int[nIdx];
59+
textStartIndex = new int[nIdx];
60+
textLengthIndex = new short[nIdx];
61+
62+
for (int i = 0, off = 0; i < nIdx; i++) {
63+
off = 4 + 1024 + i * 8;
64+
indexData[i] = bigEndian(ipData, off);
65+
textStartIndex[i] = ((int) ipData[off + 6] & 0xff) << 16 | ((int) ipData[off + 5] & 0xff) << 8
66+
| ((int) ipData[off + 4] & 0xff);
67+
textLengthIndex[i] = ipData[off + 7];
68+
}
4069
}
4170
}
4271

@@ -85,7 +114,7 @@ static LocationInfo buildInfo(byte[] bytes, int offset, int len) {
85114
String[] ss = str.split("\t", -1);
86115
if (ss.length == 4) {
87116
return new LocationInfo(ss[0], ss[1], ss[2], "");
88-
} else if (ss.length == 5) {
117+
} else if (ss.length >= 5) {
89118
return new LocationInfo(ss[0], ss[1], ss[2], ss[4]);
90119
} else if (ss.length == 3) {
91120
return new LocationInfo(ss[0], ss[1], ss[2], "");
@@ -106,7 +135,7 @@ public static Locator loadFromNet(String netPath) throws IOException {
106135
}
107136

108137
int length = httpConn.getContentLength();
109-
if (length <= 0 || length > 20 * 1024 * 1024) {
138+
if (length <= 0 || length > 64 * 1024 * 1024) {
110139
throw new InputMismatchException("invalid ip data");
111140
}
112141
InputStream is = httpConn.getInputStream();
@@ -129,7 +158,14 @@ public static Locator loadFromNet(String netPath) throws IOException {
129158

130159
is.close();
131160

132-
return loadBinary(data);
161+
String path = url.getPath();
162+
if (path.toLowerCase().endsWith("datx")) {
163+
return loadBinary(data, true);
164+
} else if (path.toLowerCase().endsWith("dat")) {
165+
return loadBinary(data, false);
166+
} else {
167+
return loadBinaryUnkown(data);
168+
}
133169
}
134170

135171
public static Locator loadFromLocal(String filePath) throws IOException {
@@ -149,10 +185,16 @@ public static Locator loadFromLocal(String filePath) throws IOException {
149185
}
150186
fi.close();
151187

152-
return loadBinary(b);
188+
if (filePath.toLowerCase().endsWith("datx")) {
189+
return loadBinary(b, true);
190+
} else if (filePath.toLowerCase().endsWith("dat")) {
191+
return loadBinary(b, false);
192+
} else {
193+
return loadBinaryUnkown(b);
194+
}
153195
}
154196

155-
public static Locator loadFromStream(InputStream in) throws Exception {
197+
public static Locator loadFromStream(InputStream in, boolean x) throws Exception {
156198
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
157199
byte[] buffer = new byte[16 * 1024];
158200
int n;
@@ -161,13 +203,49 @@ public static Locator loadFromStream(InputStream in) throws Exception {
161203
byteArrayOutputStream.write(buffer, 0, n);
162204
}
163205

164-
return loadBinary(byteArrayOutputStream.toByteArray());
206+
return loadBinary(byteArrayOutputStream.toByteArray(), x);
165207
}
166208

167-
public static Locator loadBinary(byte[] ipdb) {
209+
public static Locator loadFromStreamUnkown(InputStream in) throws Exception {
210+
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
211+
byte[] buffer = new byte[16 * 1024];
212+
int n;
213+
214+
while ((n = in.read(buffer)) != -1) {
215+
byteArrayOutputStream.write(buffer, 0, n);
216+
}
217+
218+
return loadBinaryUnkown(byteArrayOutputStream.toByteArray());
219+
}
220+
221+
public static Locator loadFromStreamOld(InputStream in, boolean x) throws Exception {
222+
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
223+
byte[] buffer = new byte[16 * 1024];
224+
int n;
225+
226+
while ((n = in.read(buffer)) != -1) {
227+
byteArrayOutputStream.write(buffer, 0, n);
228+
}
229+
230+
return loadBinary(byteArrayOutputStream.toByteArray(), x);
231+
}
232+
233+
public static Locator loadBinary(byte[] ipdb, boolean x) {
234+
return new Locator(ipdb, x);
235+
}
236+
237+
public static Locator loadBinaryUnkown(byte[] ipdb) {
168238
return new Locator(ipdb);
169239
}
170240

241+
public static Locator loadBinaryOld(byte[] ipdb) {
242+
return loadBinary(ipdb, false);
243+
}
244+
245+
public static Locator loadBinaryX(byte[] ipdb) {
246+
return loadBinary(ipdb, true);
247+
}
248+
171249
public static void main(String[] args) {
172250
if (args == null || args.length < 2) {
173251
System.out.println("locator ipfile ip");
@@ -186,14 +264,14 @@ private int findIndexOffset(long ip, int start, int end) {
186264
int mid = 0;
187265
while (start < end) {
188266
mid = (start + end) / 2;
189-
long l = 0xffffffffL & ((long) indexData1[mid]);
267+
long l = 0xffffffffL & ((long) indexData[mid]);
190268
if (ip > l) {
191269
start = mid + 1;
192270
} else {
193271
end = mid;
194272
}
195273
}
196-
long l = ((long) indexData1[end]) & 0xffffffffL;
274+
long l = ((long) indexData[end]) & 0xffffffffL;
197275
if (l >= ip) {
198276
return end;
199277
}
@@ -211,15 +289,15 @@ public LocationInfo find(String ip) {
211289
}
212290

213291
public LocationInfo find(byte[] ipBin) {
214-
int end = indexData1.length - 1;
292+
int end = indexData.length - 1;
215293
int a = 0xff & ((int) ipBin[0]);
216294
if (a != 0xff) {
217295
end = index[a + 1];
218296
}
219297
long ip = (long) bigEndian(ipBin, 0) & 0xffffffffL;
220298
int idx = findIndexOffset(ip, index[a], end);
221-
int off = indexData2[idx];
222-
return buildInfo(ipData, textOffset - 1024 + off, 0xff & (int) indexData3[idx]);
299+
int off = textStartIndex[idx];
300+
return buildInfo(ipData, textOffset + off, 0xffff & (int) textLengthIndex[idx]);
223301
}
224302

225303
public LocationInfo find(int address) {

‎src/test/java/qiniu/ip17mon/LocationTest.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public void testBuildLocationInfo() {
6969
Assert.assertEquals(new LocationInfo("中国", "上海", "上海", "电信"), l);
7070
}
7171

72-
private void testLocator(Locator locator) {
72+
private void checkFreeLocator(Locator locator) {
7373
LocationInfo info = locator.find("8.8.8.8");
7474
Assert.assertEquals(new LocationInfo("GOOGLE", "GOOGLE", "", ""), info);
7575
info = locator.find("183.131.7.18");
@@ -78,21 +78,37 @@ private void testLocator(Locator locator) {
7878
Assert.assertEquals(new LocationInfo("中国", "浙江", "杭州", ""), info);
7979
info = locator.find(Locator.bigEndian(new byte[]{(byte) 183, (byte) 131, 7, 18}, 0));
8080
Assert.assertEquals(new LocationInfo("中国", "浙江", "杭州", ""), info);
81+
}
8182

83+
private void checkFullLocator(Locator locator) {
84+
LocationInfo info = locator.find("8.8.8.8");
85+
Assert.assertEquals(new LocationInfo("GOOGLE.COM", "GOOGLE.COM", "", "level3.com"), info);
86+
info = locator.find("183.131.7.18");
87+
Assert.assertEquals(new LocationInfo("中国", "浙江", "杭州", "电信"), info);
88+
info = locator.find(new byte[]{(byte) 183, (byte) 131, 7, 18});
89+
Assert.assertEquals(new LocationInfo("中国", "浙江", "杭州", "电信"), info);
90+
info = locator.find(Locator.bigEndian(new byte[]{(byte) 183, (byte) 131, 7, 18}, 0));
91+
Assert.assertEquals(new LocationInfo("中国", "浙江", "杭州", "电信"), info);
8292
}
8393

8494
@Test
85-
public void testNetLoad() throws IOException {
95+
public void testNetLoadFree() throws IOException {
8696
Locator l = Locator.loadFromNet("http://7j1xnu.com1.z0.glb.clouddn.com/17monipdb.dat");
87-
testLocator(l);
97+
checkFreeLocator(l);
8898
}
8999

90100
@Test
91-
public void testLocalLoad() throws IOException {
101+
public void testLocalLoadFree() throws IOException {
92102
Locator l = Locator.loadFromLocal("17monipdb.dat");
93-
testLocator(l);
103+
checkFreeLocator(l);
94104
}
95105

106+
// @Test
107+
// public void testLocalLoadFull() throws IOException {
108+
// Locator l = Locator.loadFromLocal("mydata4vipday2.datx");
109+
// checkFullLocator(l);
110+
// }
111+
96112

97113
public void bench() throws IOException {
98114
Locator l = Locator.loadFromLocal("17monipdb.dat");

0 commit comments

Comments
 (0)