Skip to content

Commit 4db3608

Browse files
ip2locationip2location
committed
Reduce I/O
1 parent b7c9293 commit 4db3608

File tree

2 files changed

+116
-27
lines changed

2 files changed

+116
-27
lines changed

‎IP2ProxyComponent/IP2Proxy.vb‎

Lines changed: 114 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -224,12 +224,48 @@ Public Class Component
224224
If _MMF Is Nothing Then
225225
Try
226226
_MMF = MemoryMappedFile.OpenExisting(_MapFileName, MemoryMappedFileRights.Read)
227-
Catch Ex As Exception
227+
Catch ex As Exception
228228
Try
229229
_MMF = MemoryMappedFile.CreateFromFile(_DBFilePath, FileMode.Open, _MapFileName, New FileInfo(_DBFilePath).Length, MemoryMappedFileAccess.Read)
230-
Catch Ex2 As Exception
231-
ErrLog(Ex2.Message)
232-
Throw Ex2
230+
Catch ex2 As Exception
231+
Try
232+
Dim len As Long = New FileInfo(_DBFilePath).Length
233+
_MMF = MemoryMappedFile.CreateNew(_MapFileName, len, MemoryMappedFileAccess.ReadWrite)
234+
Using stream As MemoryMappedViewStream = _MMF.CreateViewStream()
235+
Using writer As BinaryWriter = New BinaryWriter(stream)
236+
Using fs As FileStream = New FileStream(_DBFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)
237+
Dim buff(len) As Byte
238+
fs.Read(buff, 0, buff.Length)
239+
writer.Write(buff, 0, buff.Length)
240+
End Using
241+
End Using
242+
End Using
243+
Catch ex3 As Exception
244+
' this part onwards trying Linux specific stuff (no named map)
245+
Try
246+
_MMF = MemoryMappedFile.OpenExisting(Nothing, MemoryMappedFileRights.Read)
247+
Catch ex4 As Exception
248+
Try
249+
_MMF = MemoryMappedFile.CreateFromFile(_DBFilePath, FileMode.Open, Nothing, New FileInfo(_DBFilePath).Length, MemoryMappedFileAccess.Read)
250+
Catch ex5 As Exception
251+
Try
252+
Dim len As Long = New FileInfo(_DBFilePath).Length
253+
_MMF = MemoryMappedFile.CreateNew(Nothing, len, MemoryMappedFileAccess.ReadWrite)
254+
Using stream As MemoryMappedViewStream = _MMF.CreateViewStream()
255+
Using writer As BinaryWriter = New BinaryWriter(stream)
256+
Using fs As FileStream = New FileStream(_DBFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)
257+
Dim buff(len) As Byte
258+
fs.Read(buff, 0, buff.Length)
259+
writer.Write(buff, 0, buff.Length)
260+
End Using
261+
End Using
262+
End Using
263+
Catch ex6 As Exception
264+
ErrLog(ex6.Message & "---" & ex6.StackTrace)
265+
End Try
266+
End Try
267+
End Try
268+
End Try
233269
End Try
234270
End Try
235271
End If
@@ -309,16 +345,28 @@ Public Class Component
309345
_IPv6ColumnSize = 16 + ((_DBColumn - 1) << 2) ' 4 bytes each column, except IPFrom column which is 16 bytes
310346

311347
' since both IPv4 and IPv6 use 4 bytes for the below columns, can just do it once here
312-
COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 1) << 2, 0)
313-
REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 1) << 2, 0)
314-
CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 1) << 2, 0)
315-
ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 1) << 2, 0)
316-
PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 1) << 2, 0)
317-
DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 1) << 2, 0)
318-
USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 1) << 2, 0)
319-
ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 1) << 2, 0)
320-
AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 1) << 2, 0)
321-
LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 1) << 2, 0)
348+
'COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 1) << 2, 0)
349+
'REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 1) << 2, 0)
350+
'CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 1) << 2, 0)
351+
'ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 1) << 2, 0)
352+
'PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 1) << 2, 0)
353+
'DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 1) << 2, 0)
354+
'USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 1) << 2, 0)
355+
'ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 1) << 2, 0)
356+
'AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 1) << 2, 0)
357+
'LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 1) << 2, 0)
358+
359+
' slightly different offset for reading by row
360+
COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 2) << 2, 0)
361+
REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 2) << 2, 0)
362+
CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 2) << 2, 0)
363+
ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 2) << 2, 0)
364+
PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 2) << 2, 0)
365+
DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 2) << 2, 0)
366+
USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 2) << 2, 0)
367+
ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 2) << 2, 0)
368+
AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 2) << 2, 0)
369+
LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 2) << 2, 0)
322370

323371
COUNTRY_ENABLED = If(COUNTRY_POSITION(_DBType) <> 0, True, False)
324372
REGION_ENABLED = If(REGION_POSITION(_DBType) <> 0, True, False)
@@ -572,18 +620,25 @@ Public Class Component
572620
Dim [AS] As String = MSG_NOT_SUPPORTED
573621
Dim Last_Seen As String = MSG_NOT_SUPPORTED
574622

623+
Dim FirstCol As Integer = 4 ' for IPv4, IP From is 4 bytes
575624
If IPType = 6 Then ' IPv6
576-
RowOffset = RowOffset + 12 ' coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total
625+
FirstCol = 16 ' 16 bytes for IPv6
626+
'RowOffset = RowOffset + 12 ' coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total
577627
End If
578628

629+
' read the row here after the IP From column (remaining columns are all 4 bytes)
630+
Dim Row() As Byte = ReadRow(RowOffset + FirstCol, ColumnSize - FirstCol, Accessor, FS)
631+
579632
If PROXYTYPE_ENABLED Then
580633
If Mode = Modes.ALL OrElse Mode = Modes.PROXY_TYPE OrElse Mode = Modes.IS_PROXY Then
581-
Proxy_Type = ReadStr(Read32(RowOffset + PROXYTYPE_POSITION_OFFSET, Accessor, FS), FS)
634+
'Proxy_Type = ReadStr(Read32(RowOffset + PROXYTYPE_POSITION_OFFSET, Accessor, FS), FS)
635+
Proxy_Type = ReadStr(Read32_Row(Row, PROXYTYPE_POSITION_OFFSET), FS)
582636
End If
583637
End If
584638
If COUNTRY_ENABLED Then
585639
If Mode = Modes.ALL OrElse Mode = Modes.COUNTRY_SHORT OrElse Mode = Modes.COUNTRY_LONG OrElse Mode = Modes.IS_PROXY Then
586-
CountryPos = Read32(RowOffset + COUNTRY_POSITION_OFFSET, Accessor, FS)
640+
'CountryPos = Read32(RowOffset + COUNTRY_POSITION_OFFSET, Accessor, FS)
641+
CountryPos = Read32_Row(Row, COUNTRY_POSITION_OFFSET)
587642
End If
588643
If Mode = Modes.ALL OrElse Mode = Modes.COUNTRY_SHORT OrElse Mode = Modes.IS_PROXY Then
589644
Country_Short = ReadStr(CountryPos, FS)
@@ -594,42 +649,50 @@ Public Class Component
594649
End If
595650
If REGION_ENABLED Then
596651
If Mode = Modes.ALL OrElse Mode = Modes.REGION Then
597-
Region = ReadStr(Read32(RowOffset + REGION_POSITION_OFFSET, Accessor, FS), FS)
652+
'Region = ReadStr(Read32(RowOffset + REGION_POSITION_OFFSET, Accessor, FS), FS)
653+
Region = ReadStr(Read32_Row(Row, REGION_POSITION_OFFSET), FS)
598654
End If
599655
End If
600656
If CITY_ENABLED Then
601657
If Mode = Modes.ALL OrElse Mode = Modes.CITY Then
602-
City = ReadStr(Read32(RowOffset + CITY_POSITION_OFFSET, Accessor, FS), FS)
658+
'City = ReadStr(Read32(RowOffset + CITY_POSITION_OFFSET, Accessor, FS), FS)
659+
City = ReadStr(Read32_Row(Row, CITY_POSITION_OFFSET), FS)
603660
End If
604661
End If
605662
If ISP_ENABLED Then
606663
If Mode = Modes.ALL OrElse Mode = Modes.ISP Then
607-
ISP = ReadStr(Read32(RowOffset + ISP_POSITION_OFFSET, Accessor, FS), FS)
664+
'ISP = ReadStr(Read32(RowOffset + ISP_POSITION_OFFSET, Accessor, FS), FS)
665+
ISP = ReadStr(Read32_Row(Row, ISP_POSITION_OFFSET), FS)
608666
End If
609667
End If
610668
If DOMAIN_ENABLED Then
611669
If Mode = Modes.ALL OrElse Mode = Modes.DOMAIN Then
612-
Domain = ReadStr(Read32(RowOffset + DOMAIN_POSITION_OFFSET, Accessor, FS), FS)
670+
'Domain = ReadStr(Read32(RowOffset + DOMAIN_POSITION_OFFSET, Accessor, FS), FS)
671+
Domain = ReadStr(Read32_Row(Row, DOMAIN_POSITION_OFFSET), FS)
613672
End If
614673
End If
615674
If USAGETYPE_ENABLED Then
616675
If Mode = Modes.ALL OrElse Mode = Modes.USAGE_TYPE Then
617-
Usage_Type = ReadStr(Read32(RowOffset + USAGETYPE_POSITION_OFFSET, Accessor, FS), FS)
676+
'Usage_Type = ReadStr(Read32(RowOffset + USAGETYPE_POSITION_OFFSET, Accessor, FS), FS)
677+
Usage_Type = ReadStr(Read32_Row(Row, USAGETYPE_POSITION_OFFSET), FS)
618678
End If
619679
End If
620680
If ASN_ENABLED Then
621681
If Mode = Modes.ALL OrElse Mode = Modes.ASN Then
622-
ASN = ReadStr(Read32(RowOffset + ASN_POSITION_OFFSET, Accessor, FS), FS)
682+
'ASN = ReadStr(Read32(RowOffset + ASN_POSITION_OFFSET, Accessor, FS), FS)
683+
ASN = ReadStr(Read32_Row(Row, ASN_POSITION_OFFSET), FS)
623684
End If
624685
End If
625686
If AS_ENABLED Then
626687
If Mode = Modes.ALL OrElse Mode = Modes.AS Then
627-
[AS] = ReadStr(Read32(RowOffset + AS_POSITION_OFFSET, Accessor, FS), FS)
688+
'[AS] = ReadStr(Read32(RowOffset + AS_POSITION_OFFSET, Accessor, FS), FS)
689+
[AS] = ReadStr(Read32_Row(Row, AS_POSITION_OFFSET), FS)
628690
End If
629691
End If
630692
If LASTSEEN_ENABLED Then
631693
If Mode = Modes.ALL OrElse Mode = Modes.LAST_SEEN Then
632-
Last_Seen = ReadStr(Read32(RowOffset + LASTSEEN_POSITION_OFFSET, Accessor, FS), FS)
694+
'Last_Seen = ReadStr(Read32(RowOffset + LASTSEEN_POSITION_OFFSET, Accessor, FS), FS)
695+
Last_Seen = ReadStr(Read32_Row(Row, LASTSEEN_POSITION_OFFSET), FS)
633696
End If
634697
End If
635698

@@ -694,6 +757,19 @@ Public Class Component
694757
End Try
695758
End Function
696759

760+
' Read whole row into array of bytes
761+
Private Function ReadRow(ByVal _Pos As Long, ByVal MyLen As UInt32, ByRef MyAccessor As MemoryMappedViewAccessor, ByRef MyFilestream As FileStream) As Byte()
762+
Dim row(MyLen - 1) As Byte
763+
764+
If _UseMemoryMappedFile Then
765+
MyAccessor.ReadArray(Of Byte)(_Pos, row, 0, MyLen)
766+
Else
767+
MyFilestream.Seek(_Pos - 1, SeekOrigin.Begin)
768+
MyFilestream.Read(row, 0, MyLen)
769+
End If
770+
Return row
771+
End Function
772+
697773
Private Function Read32Or128(ByVal _Pos As Long, ByVal _MyIPType As Integer, ByRef MyAccessor As MemoryMappedViewAccessor, ByRef MyFilestream As FileStream) As BigInteger
698774
If _MyIPType = 4 Then
699775
Return Read32(_Pos, MyAccessor, MyFilestream)
@@ -729,6 +805,19 @@ Public Class Component
729805
End Try
730806
End Function
731807

808+
' Read 32 bits in byte array
809+
Private Function Read32_Row(ByRef Row() As Byte, ByVal ByteOffset As Integer) As BigInteger
810+
Try
811+
Dim _Byte(3) As Byte ' 4 bytes
812+
Array.Copy(Row, ByteOffset, _Byte, 0, 4)
813+
814+
Return System.BitConverter.ToUInt32(_Byte, 0)
815+
Catch ex As Exception
816+
Throw
817+
'ErrLog("Read32_Row-" & ex.Message)
818+
End Try
819+
End Function
820+
732821
' Read 32 bits in the encrypted database
733822
Private Function Read32(ByVal _Pos As Long, ByRef MyAccessor As MemoryMappedViewAccessor, ByRef MyFilestream As FileStream) As BigInteger
734823
Try

‎IP2ProxyComponent/My Project/AssemblyInfo.vb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,5 @@ Imports System.Runtime.InteropServices
3131
' by using the '*' as shown below:
3232
' <Assembly: AssemblyVersion("1.0.*")>
3333

34-
<Assembly: AssemblyVersion("2.1.0.0")>
35-
<Assembly: AssemblyFileVersion("2.1.0.0")>
34+
<Assembly: AssemblyVersion("2.2.0.0")>
35+
<Assembly: AssemblyFileVersion("2.2.0.0")>

0 commit comments

Comments
 (0)