Skip to content

Commit 8658370

Browse files
ip2locationip2location
committed
Reduced file I/O
1 parent 458efc6 commit 8658370

File tree

1 file changed

+83
-98
lines changed

1 file changed

+83
-98
lines changed

‎ip2location.go‎

Lines changed: 83 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ type ip2locationmeta struct {
3131
ipv4databaseaddr uint32
3232
ipv6databasecount uint32
3333
ipv6databaseaddr uint32
34+
ipv4indexed bool
35+
ipv6indexed bool
3436
ipv4indexbaseaddr uint32
3537
ipv6indexbaseaddr uint32
3638
ipv4columnsize uint32
@@ -142,7 +144,7 @@ var usagetype_position = [26]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
142144
var addresstype_position = [26]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21}
143145
var category_position = [26]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22}
144146

145-
const api_version string = "9.3.0"
147+
const api_version string = "9.4.0"
146148

147149
var max_ipv4_range = big.NewInt(4294967295)
148150
var max_ipv6_range = big.NewInt(0)
@@ -224,13 +226,13 @@ func (d *DB) checkip(ip string) (iptype uint32, ipnum *big.Int, ipindex uint32)
224226
}
225227
}
226228
if iptype == 4 {
227-
if d.meta.ipv4indexbaseaddr > 0 {
229+
if d.meta.ipv4indexed {
228230
ipnumtmp.Rsh(ipnum, 16)
229231
ipnumtmp.Lsh(ipnumtmp, 3)
230232
ipindex = uint32(ipnumtmp.Add(ipnumtmp, big.NewInt(int64(d.meta.ipv4indexbaseaddr))).Uint64())
231233
}
232234
} else if iptype == 6 {
233-
if d.meta.ipv6indexbaseaddr > 0 {
235+
if d.meta.ipv6indexed {
234236
ipnumtmp.Rsh(ipnum, 112)
235237
ipnumtmp.Lsh(ipnumtmp, 3)
236238
ipindex = uint32(ipnumtmp.Add(ipnumtmp, big.NewInt(int64(d.meta.ipv6indexbaseaddr))).Uint64())
@@ -251,6 +253,17 @@ func (d *DB) readuint8(pos int64) (uint8, error) {
251253
return retval, nil
252254
}
253255

256+
// read row
257+
func (d *DB) read_row(pos uint32, size uint32) ([]byte, error) {
258+
pos2 := int64(pos)
259+
data := make([]byte, size)
260+
_, err := d.f.ReadAt(data, pos2-1)
261+
if err != nil {
262+
return nil, err
263+
}
264+
return data, nil
265+
}
266+
254267
// read unsigned 32-bit integer from slices
255268
func (d *DB) readuint32_row(row []byte, pos uint32) uint32 {
256269
var retval uint32
@@ -276,6 +289,19 @@ func (d *DB) readuint32(pos uint32) (uint32, error) {
276289
return retval, nil
277290
}
278291

292+
// read unsigned 128-bit integer from slices
293+
func (d *DB) readuint128_row(row []byte, pos uint32) *big.Int {
294+
retval := big.NewInt(0)
295+
data := row[pos : pos+16]
296+
297+
// little endian to big endian
298+
for i, j := 0, len(data)-1; i < j; i, j = i+1, j-1 {
299+
data[i], data[j] = data[j], data[i]
300+
}
301+
retval.SetBytes(data)
302+
return retval
303+
}
304+
279305
// read unsigned 128-bit integer
280306
func (d *DB) readuint128(pos uint32) (*big.Int, error) {
281307
pos2 := int64(pos)
@@ -297,19 +323,15 @@ func (d *DB) readuint128(pos uint32) (*big.Int, error) {
297323
// read string
298324
func (d *DB) readstr(pos uint32) (string, error) {
299325
pos2 := int64(pos)
326+
readlen := 256 // max size of string field + 1 byte for the length
300327
var retval string
301-
lenbyte := make([]byte, 1)
302-
_, err := d.f.ReadAt(lenbyte, pos2)
303-
if err != nil {
304-
return "", err
305-
}
306-
strlen := lenbyte[0]
307-
data := make([]byte, strlen)
308-
_, err = d.f.ReadAt(data, pos2+1)
328+
data := make([]byte, readlen)
329+
_, err := d.f.ReadAt(data, pos2)
309330
if err != nil {
310331
return "", err
311332
}
312-
retval = string(data[:strlen])
333+
strlen := data[0]
334+
retval = string(data[1:(strlen + 1)])
313335
return retval, nil
314336
}
315337

@@ -351,67 +373,42 @@ func OpenDBWithReader(reader DBReader) (*DB, error) {
351373

352374
db.f = reader
353375

376+
var row []byte
354377
var err error
355-
db.meta.databasetype, err = db.readuint8(1)
356-
if err != nil {
357-
return fatal(db, err)
358-
}
359-
db.meta.databasecolumn, err = db.readuint8(2)
360-
if err != nil {
361-
return fatal(db, err)
362-
}
363-
db.meta.databaseyear, err = db.readuint8(3)
364-
if err != nil {
365-
return fatal(db, err)
366-
}
367-
db.meta.databasemonth, err = db.readuint8(4)
368-
if err != nil {
369-
return fatal(db, err)
370-
}
371-
db.meta.databaseday, err = db.readuint8(5)
372-
if err != nil {
373-
return fatal(db, err)
374-
}
375-
db.meta.ipv4databasecount, err = db.readuint32(6)
376-
if err != nil {
377-
return fatal(db, err)
378-
}
379-
db.meta.ipv4databaseaddr, err = db.readuint32(10)
380-
if err != nil {
381-
return fatal(db, err)
382-
}
383-
db.meta.ipv6databasecount, err = db.readuint32(14)
384-
if err != nil {
385-
return fatal(db, err)
386-
}
387-
db.meta.ipv6databaseaddr, err = db.readuint32(18)
388-
if err != nil {
389-
return fatal(db, err)
390-
}
391-
db.meta.ipv4indexbaseaddr, err = db.readuint32(22)
392-
if err != nil {
393-
return fatal(db, err)
394-
}
395-
db.meta.ipv6indexbaseaddr, err = db.readuint32(26)
396-
if err != nil {
397-
return fatal(db, err)
398-
}
399-
db.meta.productcode, err = db.readuint8(30)
400-
if err != nil {
401-
return fatal(db, err)
402-
}
403-
db.meta.producttype, err = db.readuint8(31)
404-
if err != nil {
405-
return fatal(db, err)
406-
}
407-
db.meta.filesize, err = db.readuint32(32)
378+
readlen := uint32(64) // 64-byte header
379+
380+
row, err = db.read_row(1, readlen)
408381
if err != nil {
409382
return fatal(db, err)
410383
}
384+
db.meta.databasetype = row[0]
385+
db.meta.databasecolumn = row[1]
386+
db.meta.databaseyear = row[2]
387+
db.meta.databasemonth = row[3]
388+
db.meta.databaseday = row[4]
389+
db.meta.ipv4databasecount = db.readuint32_row(row, 5)
390+
db.meta.ipv4databaseaddr = db.readuint32_row(row, 9)
391+
db.meta.ipv6databasecount = db.readuint32_row(row, 13)
392+
db.meta.ipv6databaseaddr = db.readuint32_row(row, 17)
393+
db.meta.ipv4indexbaseaddr = db.readuint32_row(row, 21)
394+
db.meta.ipv6indexbaseaddr = db.readuint32_row(row, 25)
395+
db.meta.productcode = row[29]
396+
db.meta.producttype = row[30]
397+
db.meta.filesize = db.readuint32_row(row, 31)
398+
411399
// check if is correct BIN (should be 1 for IP2Location BIN file), also checking for zipped file (PK being the first 2 chars)
412400
if (db.meta.productcode != 1 && db.meta.databaseyear >= 21) || (db.meta.databasetype == 80 && db.meta.databasecolumn == 75) { // only BINs from Jan 2021 onwards have this byte set
413401
return fatal(db, errors.New(invalid_bin))
414402
}
403+
404+
if db.meta.ipv4indexbaseaddr > 0 {
405+
db.meta.ipv4indexed = true
406+
}
407+
408+
if db.meta.ipv6databasecount > 0 && db.meta.ipv6indexbaseaddr > 0 {
409+
db.meta.ipv6indexed = true
410+
}
411+
415412
db.meta.ipv4columnsize = uint32(db.meta.databasecolumn << 2) // 4 bytes each column
416413
db.meta.ipv6columnsize = uint32(16 + ((db.meta.databasecolumn - 1) << 2)) // 4 bytes each column, except IPFrom column which is 16 bytes
417414

@@ -853,7 +850,10 @@ func (d *DB) query(ipaddress string, mode uint32) (IP2Locationrecord, error) {
853850
var high uint32
854851
var mid uint32
855852
var rowoffset uint32
856-
var rowoffset2 uint32
853+
var firstcol uint32 = 4 // 4 bytes for ip from
854+
var row []byte
855+
var fullrow []byte
856+
var readlen uint32
857857
ipfrom := big.NewInt(0)
858858
ipto := big.NewInt(0)
859859
maxip := big.NewInt(0)
@@ -864,6 +864,7 @@ func (d *DB) query(ipaddress string, mode uint32) (IP2Locationrecord, error) {
864864
maxip = max_ipv4_range
865865
colsize = d.meta.ipv4columnsize
866866
} else {
867+
firstcol = 16 // 16 bytes for ip from
867868
baseaddr = d.meta.ipv6databaseaddr
868869
high = d.meta.ipv6databasecount
869870
maxip = max_ipv6_range
@@ -872,14 +873,12 @@ func (d *DB) query(ipaddress string, mode uint32) (IP2Locationrecord, error) {
872873

873874
// reading index
874875
if ipindex > 0 {
875-
low, err = d.readuint32(ipindex)
876-
if err != nil {
877-
return x, err
878-
}
879-
high, err = d.readuint32(ipindex + 4)
876+
row, err = d.read_row(ipindex, 8) // 4 bytes each for IP From and IP To
880877
if err != nil {
881878
return x, err
882879
}
880+
low = d.readuint32_row(row, 0)
881+
high = d.readuint32_row(row, 4)
883882
}
884883

885884
if ipno.Cmp(maxip) >= 0 {
@@ -889,44 +888,30 @@ func (d *DB) query(ipaddress string, mode uint32) (IP2Locationrecord, error) {
889888
for low <= high {
890889
mid = ((low + high) >> 1)
891890
rowoffset = baseaddr + (mid * colsize)
892-
rowoffset2 = rowoffset + colsize
891+
892+
// reading IP From + whole row + next IP From
893+
readlen = colsize + firstcol
894+
fullrow, err = d.read_row(rowoffset, readlen)
895+
if err != nil {
896+
return x, err
897+
}
893898

894899
if iptype == 4 {
895-
ipfrom32, err := d.readuint32(rowoffset)
896-
if err != nil {
897-
return x, err
898-
}
900+
ipfrom32 := d.readuint32_row(fullrow, 0)
899901
ipfrom = big.NewInt(int64(ipfrom32))
900902

901-
ipto32, err := d.readuint32(rowoffset2)
902-
if err != nil {
903-
return x, err
904-
}
903+
ipto32 := d.readuint32_row(fullrow, colsize)
905904
ipto = big.NewInt(int64(ipto32))
906905

907906
} else {
908-
ipfrom, err = d.readuint128(rowoffset)
909-
if err != nil {
910-
return x, err
911-
}
907+
ipfrom = d.readuint128_row(fullrow, 0)
912908

913-
ipto, err = d.readuint128(rowoffset2)
914-
if err != nil {
915-
return x, err
916-
}
909+
ipto = d.readuint128_row(fullrow, colsize)
917910
}
918911

919912
if ipno.Cmp(ipfrom) >= 0 && ipno.Cmp(ipto) < 0 {
920-
var firstcol uint32 = 4 // 4 bytes for ip from
921-
if iptype == 6 {
922-
firstcol = 16 // 16 bytes for ipv6
923-
}
924-
925-
row := make([]byte, colsize-firstcol) // exclude the ip from field
926-
_, err := d.f.ReadAt(row, int64(rowoffset+firstcol-1))
927-
if err != nil {
928-
return x, err
929-
}
913+
rowlen := colsize - firstcol
914+
row = fullrow[firstcol:(firstcol + rowlen)] // extract the actual row data
930915

931916
if mode&countryshort == 1 && d.country_enabled {
932917
if x.Country_short, err = d.readstr(d.readuint32_row(row, d.country_position_offset)); err != nil {

0 commit comments

Comments
 (0)