Skip to content

Commit b9bd65c

Browse files
ip2locationip2location
committed
Reduced file I/O
1 parent ba61705 commit b9bd65c

File tree

3 files changed

+152
-102
lines changed

3 files changed

+152
-102
lines changed

‎IP2ProxyComponent/IP2ProxyComponent/Http.vb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ Public Class Http
1616
Return ("Failed : HTTP error code :" & response.StatusCode)
1717
End If
1818
End Function
19+
1920
Public Function PostMethod(ByVal url As String, post As String) As String
2021
Dim request As HttpWebRequest
2122
Dim response As HttpWebResponse

‎IP2ProxyComponent/IP2ProxyComponent/IP2Proxy.vb‎

Lines changed: 148 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ Public Structure ProxyResult
2424
End Structure
2525

2626
Public Class Component
27+
Private ReadOnly _LockLoadBIN As New Object
2728
Private _DBFilePath As String = ""
2829
Private _MMF As MemoryMappedFile = Nothing
2930
Private ReadOnly _IndexArrayIPv4(65535, 1) As Integer
@@ -339,91 +340,104 @@ Public Class Component
339340
' Description: Read BIN file into memory mapped file and create accessors
340341
Private Function LoadBIN() As Boolean
341342
Dim LoadOK As Boolean = False
342-
If _DBFilePath <> "" Then
343-
CreateMemoryMappedFile()
344-
345-
If _MMF IsNot Nothing Then
346-
' below use temp accessor as we only need once to read meta data (use this even when in filestream mode)
347-
Using _MetaAccessor As MemoryMappedViewAccessor = _MMF.CreateViewAccessor(0, 64, MemoryMappedFileAccess.Read) ' 64 bytes header
348-
_DBType = _MetaAccessor.ReadByte(0)
349-
_DBColumn = _MetaAccessor.ReadByte(1)
350-
_DBYear = _MetaAccessor.ReadByte(2)
351-
_DBMonth = _MetaAccessor.ReadByte(3)
352-
_DBDay = _MetaAccessor.ReadByte(4)
353-
_DBCount = _MetaAccessor.ReadInt32(5) '4 bytes
354-
_BaseAddr = _MetaAccessor.ReadInt32(9) '4 bytes
355-
_DBCountIPv6 = _MetaAccessor.ReadInt32(13) '4 bytes
356-
_BaseAddrIPv6 = _MetaAccessor.ReadInt32(17) '4 bytes
357-
_IndexBaseAddr = _MetaAccessor.ReadInt32(21) '4 bytes
358-
_IndexBaseAddrIPv6 = _MetaAccessor.ReadInt32(25) '4 bytes
359-
_ProductCode = _MetaAccessor.ReadByte(29)
360-
_ProductType = _MetaAccessor.ReadByte(30)
361-
_FileSize = _MetaAccessor.ReadInt32(31) '4 bytes
362-
363-
' check if is correct BIN (should be 2 for IP2Proxy BIN file), also checking for zipped file (PK being the first 2 chars)
364-
If (_ProductCode <> 2 AndAlso _DBYear >= 21) OrElse (_DBType = 80 AndAlso _DBColumn = 75) Then ' only BINs from Jan 2021 onwards have this byte set
365-
Throw New Exception(MSG_INVALID_BIN)
366-
End If
343+
SyncLock _LockLoadBIN
344+
If _DBFilePath <> "" Then
345+
If _DBType = 0 Then
346+
Using _Filestream = New FileStream(_DBFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)
347+
Dim len = 64 ' 64-byte header
348+
Dim row(len - 1) As Byte
349+
350+
_Filestream.Seek(0, SeekOrigin.Begin)
351+
_Filestream.Read(row, 0, len)
352+
353+
_DBType = Read8Header(row, 0)
354+
_DBColumn = Read8Header(row, 1)
355+
_DBYear = Read8Header(row, 2)
356+
_DBMonth = Read8Header(row, 3)
357+
_DBDay = Read8Header(row, 4)
358+
_DBCount = Read32Header(row, 5) '4 bytes
359+
_BaseAddr = Read32Header(row, 9) '4 bytes
360+
_DBCountIPv6 = Read32Header(row, 13) '4 bytes
361+
_BaseAddrIPv6 = Read32Header(row, 17) '4 bytes
362+
_IndexBaseAddr = Read32Header(row, 21) '4 bytes
363+
_IndexBaseAddrIPv6 = Read32Header(row, 25) '4 bytes
364+
_ProductCode = Read8Header(row, 29)
365+
' below 2 fields just read for now, not being used yet
366+
_ProductType = Read8Header(row, 30)
367+
_FileSize = Read32Header(row, 31) '4 bytes
368+
369+
' check if is correct BIN (should be 2 for IP2Proxy BIN file), also checking for zipped file (PK being the first 2 chars)
370+
If (_ProductCode <> 2 AndAlso _DBYear >= 21) OrElse (_DBType = 80 AndAlso _DBColumn = 75) Then ' only BINs from Jan 2021 onwards have this byte set
371+
Throw New Exception(MSG_INVALID_BIN)
372+
End If
367373

368-
_IPv4ColumnSize = _DBColumn << 2 ' 4 bytes each column
369-
_IPv6ColumnSize = 16 + ((_DBColumn - 1) << 2) ' 4 bytes each column, except IPFrom column which is 16 bytes
370-
371-
COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 2) << 2, 0)
372-
REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 2) << 2, 0)
373-
CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 2) << 2, 0)
374-
ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 2) << 2, 0)
375-
PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 2) << 2, 0)
376-
DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 2) << 2, 0)
377-
USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 2) << 2, 0)
378-
ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 2) << 2, 0)
379-
AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 2) << 2, 0)
380-
LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 2) << 2, 0)
381-
THREAT_POSITION_OFFSET = If(THREAT_POSITION(_DBType) <> 0, (THREAT_POSITION(_DBType) - 2) << 2, 0)
382-
PROVIDER_POSITION_OFFSET = If(PROVIDER_POSITION(_DBType) <> 0, (PROVIDER_POSITION(_DBType) - 2) << 2, 0)
383-
384-
COUNTRY_ENABLED = COUNTRY_POSITION(_DBType) <> 0
385-
REGION_ENABLED = REGION_POSITION(_DBType) <> 0
386-
CITY_ENABLED = CITY_POSITION(_DBType) <> 0
387-
ISP_ENABLED = ISP_POSITION(_DBType) <> 0
388-
PROXYTYPE_ENABLED = PROXYTYPE_POSITION(_DBType) <> 0
389-
DOMAIN_ENABLED = DOMAIN_POSITION(_DBType) <> 0
390-
USAGETYPE_ENABLED = USAGETYPE_POSITION(_DBType) <> 0
391-
ASN_ENABLED = ASN_POSITION(_DBType) <> 0
392-
AS_ENABLED = AS_POSITION(_DBType) <> 0
393-
LASTSEEN_ENABLED = LASTSEEN_POSITION(_DBType) <> 0
394-
THREAT_ENABLED = THREAT_POSITION(_DBType) <> 0
395-
PROVIDER_ENABLED = PROVIDER_POSITION(_DBType) <> 0
396-
End Using
397-
398-
Using _IndexAccessor As MemoryMappedViewAccessor = _MMF.CreateViewAccessor(_IndexBaseAddr - 1, _BaseAddr - _IndexBaseAddr, MemoryMappedFileAccess.Read) ' reading indexes
399-
Dim Pointer As Integer = 0
400-
401-
' read IPv4 index
402-
For x As Integer = _IndexArrayIPv4.GetLowerBound(0) To _IndexArrayIPv4.GetUpperBound(0)
403-
_IndexArrayIPv4(x, 0) = _IndexAccessor.ReadInt32(Pointer) '4 bytes for from row
404-
_IndexArrayIPv4(x, 1) = _IndexAccessor.ReadInt32(Pointer + 4) '4 bytes for to row
405-
Pointer += 8
406-
Next
374+
_IPv4ColumnSize = _DBColumn << 2 ' 4 bytes each column
375+
_IPv6ColumnSize = 16 + ((_DBColumn - 1) << 2) ' 4 bytes each column, except IPFrom column which is 16 bytes
376+
377+
COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 2) << 2, 0)
378+
REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 2) << 2, 0)
379+
CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 2) << 2, 0)
380+
ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 2) << 2, 0)
381+
PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 2) << 2, 0)
382+
DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 2) << 2, 0)
383+
USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 2) << 2, 0)
384+
ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 2) << 2, 0)
385+
AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 2) << 2, 0)
386+
LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 2) << 2, 0)
387+
THREAT_POSITION_OFFSET = If(THREAT_POSITION(_DBType) <> 0, (THREAT_POSITION(_DBType) - 2) << 2, 0)
388+
PROVIDER_POSITION_OFFSET = If(PROVIDER_POSITION(_DBType) <> 0, (PROVIDER_POSITION(_DBType) - 2) << 2, 0)
389+
390+
COUNTRY_ENABLED = COUNTRY_POSITION(_DBType) <> 0
391+
REGION_ENABLED = REGION_POSITION(_DBType) <> 0
392+
CITY_ENABLED = CITY_POSITION(_DBType) <> 0
393+
ISP_ENABLED = ISP_POSITION(_DBType) <> 0
394+
PROXYTYPE_ENABLED = PROXYTYPE_POSITION(_DBType) <> 0
395+
DOMAIN_ENABLED = DOMAIN_POSITION(_DBType) <> 0
396+
USAGETYPE_ENABLED = USAGETYPE_POSITION(_DBType) <> 0
397+
ASN_ENABLED = ASN_POSITION(_DBType) <> 0
398+
AS_ENABLED = AS_POSITION(_DBType) <> 0
399+
LASTSEEN_ENABLED = LASTSEEN_POSITION(_DBType) <> 0
400+
THREAT_ENABLED = THREAT_POSITION(_DBType) <> 0
401+
PROVIDER_ENABLED = PROVIDER_POSITION(_DBType) <> 0
402+
403+
Dim readLen = _IndexArrayIPv4.GetLength(0)
404+
If _IndexBaseAddrIPv6 > 0 Then
405+
readLen += _IndexArrayIPv6.GetLength(0)
406+
End If
407407

408-
If _IndexBaseAddrIPv6 > 0 Then
409-
' read IPv6 index
410-
For x As Integer = _IndexArrayIPv6.GetLowerBound(0) To _IndexArrayIPv6.GetUpperBound(0)
411-
_IndexArrayIPv6(x, 0) = _IndexAccessor.ReadInt32(Pointer) '4 bytes for from row
412-
_IndexArrayIPv6(x, 1) = _IndexAccessor.ReadInt32(Pointer + 4) '4 bytes for to row
413-
Pointer += 8
408+
readLen *= 8 ' 4 bytes for both From/To
409+
Dim indexData(readLen - 1) As Byte
410+
411+
_Filestream.Seek(_IndexBaseAddr - 1, SeekOrigin.Begin)
412+
_Filestream.Read(indexData, 0, readLen)
413+
414+
Dim pointer As Integer = 0
415+
416+
' read IPv4 index
417+
For x As Integer = _IndexArrayIPv4.GetLowerBound(0) To _IndexArrayIPv4.GetUpperBound(0)
418+
_IndexArrayIPv4(x, 0) = Read32Header(indexData, pointer) '4 bytes for row
419+
_IndexArrayIPv4(x, 1) = Read32Header(indexData, pointer + 4) '4 bytes for to row
420+
pointer += 8
414421
Next
415-
End If
416-
End Using
417422

418-
If _UseMemoryMappedFile Then
419-
CreateAccessors()
420-
Else
421-
DestroyMemoryMappedFile()
423+
If _IndexBaseAddrIPv6 > 0 Then
424+
' read IPv6 index
425+
For x As Integer = _IndexArrayIPv6.GetLowerBound(0) To _IndexArrayIPv6.GetUpperBound(0)
426+
_IndexArrayIPv6(x, 0) = Read32Header(indexData, pointer) '4 bytes for row
427+
_IndexArrayIPv6(x, 1) = Read32Header(indexData, pointer + 4) '4 bytes for to row
428+
pointer += 8
429+
Next
430+
End If
431+
End Using
432+
433+
If _UseMemoryMappedFile Then
434+
CreateMemoryMappedFile()
435+
CreateAccessors()
436+
End If
437+
LoadOK = True
422438
End If
423-
LoadOK = True
424439
End If
425-
End If
426-
440+
End SyncLock
427441
Return LoadOK
428442
End Function
429443

@@ -477,6 +491,9 @@ Public Class Component
477491
Dim RowOffset2 As Long = 0
478492
Dim ColumnSize As Integer = 0
479493
Dim OverCapacity As Boolean = False
494+
Dim FullRow As Byte() = Nothing
495+
Dim Row As Byte()
496+
Dim FirstCol As Integer = 4 ' IP From is 4 bytes
480497

481498
Try
482499
If IPAddress = "" OrElse IPAddress Is Nothing Then
@@ -572,6 +589,7 @@ Public Class Component
572589
High = _IndexArrayIPv4(IndexAddr, 1)
573590
Case 6
574591
' IPv6
592+
FirstCol = 16 ' IPv6 is 16 bytes
575593
If _DBCountIPv6 = 0 Then
576594
With Result
577595
.Is_Proxy = -1
@@ -619,12 +637,17 @@ Public Class Component
619637
RowOffset2 = RowOffset + ColumnSize
620638

621639
If _UseMemoryMappedFile Then
640+
' only reading the IP From fields
622641
OverCapacity = (RowOffset2 >= Accessor.Capacity)
642+
IPFrom = Read32Or128(RowOffset, IPType, Accessor, FS)
643+
IPTo = If(OverCapacity, BigInteger.Zero, Read32Or128(RowOffset2, IPType, Accessor, FS))
644+
Else
645+
' reading IP From + whole row + next IP From
646+
FullRow = ReadRow(RowOffset, ColumnSize + FirstCol, Accessor, FS)
647+
IPFrom = Read32Or128Row(FullRow, 0, FirstCol)
648+
IPTo = If(OverCapacity, BigInteger.Zero, Read32Or128Row(FullRow, ColumnSize, FirstCol))
623649
End If
624650

625-
IPFrom = Read32Or128(RowOffset, IPType, Accessor, FS)
626-
IPTo = If(OverCapacity, BigInteger.Zero, Read32Or128(RowOffset2, IPType, Accessor, FS))
627-
628651
If IPNum >= IPFrom AndAlso IPNum < IPTo Then
629652
Dim Is_Proxy As Integer = -1
630653
Dim Proxy_Type As String = MSG_NOT_SUPPORTED
@@ -641,13 +664,14 @@ Public Class Component
641664
Dim Threat As String = MSG_NOT_SUPPORTED
642665
Dim Provider As String = MSG_NOT_SUPPORTED
643666

644-
Dim FirstCol As Integer = 4 ' for IPv4, IP From is 4 bytes
645-
If IPType = 6 Then ' IPv6
646-
FirstCol = 16 ' 16 bytes for IPv6
647-
End If
667+
Dim RowLen = ColumnSize - FirstCol
648668

649-
' read the row here after the IP From column (remaining columns are all 4 bytes)
650-
Dim Row() As Byte = ReadRow(RowOffset + FirstCol, ColumnSize - FirstCol, Accessor, FS)
669+
If _UseMemoryMappedFile Then
670+
Row = ReadRow(RowOffset + FirstCol, RowLen, Accessor, FS)
671+
Else
672+
ReDim Row(RowLen - 1)
673+
Array.Copy(FullRow, FirstCol, Row, 0, RowLen) ' extract the actual row data
674+
End If
651675

652676
If PROXYTYPE_ENABLED Then
653677
If Mode = Modes.ALL OrElse Mode = Modes.PROXY_TYPE OrElse Mode = Modes.IS_PROXY Then
@@ -791,6 +815,12 @@ Public Class Component
791815
Return row
792816
End Function
793817

818+
Private Function Read32Or128Row(ByRef Row() As Byte, ByVal ByteOffset As Integer, ByVal Len As Integer) As BigInteger
819+
Dim _Byte(Len - 1) As Byte
820+
Array.Copy(Row, ByteOffset, _Byte, 0, Len)
821+
Return New BigInteger(_Byte)
822+
End Function
823+
794824
Private Function Read32Or128(ByVal _Pos As Long, ByVal _MyIPType As Integer, ByRef MyAccessor As MemoryMappedViewAccessor, ByRef MyFilestream As FileStream) As BigInteger
795825
If _MyIPType = 4 Then
796826
Return Read32(_Pos, MyAccessor, MyFilestream)
@@ -821,6 +851,20 @@ Public Class Component
821851
Return BigRetVal
822852
End Function
823853

854+
' Read 8 bits in header
855+
Private Function Read8Header(ByRef Row() As Byte, ByVal ByteOffset As Integer) As Integer
856+
Dim _Byte(0) As Byte ' 1 byte
857+
Array.Copy(Row, ByteOffset, _Byte, 0, 1)
858+
Return _Byte(0)
859+
End Function
860+
861+
' Read 32 bits in header
862+
Private Function Read32Header(ByRef Row() As Byte, ByVal ByteOffset As Integer) As Integer
863+
Dim _Byte(3) As Byte ' 4 bytes
864+
Array.Copy(Row, ByteOffset, _Byte, 0, 4)
865+
Return BitConverter.ToUInt32(_Byte, 0)
866+
End Function
867+
824868
' Read 32 bits in byte array
825869
Private Function Read32Row(ByRef Row() As Byte, ByVal ByteOffset As Integer) As BigInteger
826870
Dim _Byte(3) As Byte ' 4 bytes
@@ -843,22 +887,27 @@ Public Class Component
843887

844888
' Read strings in the database
845889
Private Function ReadStr(ByVal _Pos As Long, ByRef Myfilestream As FileStream) As String
890+
Dim _Size = 256 ' max size of string field + 1 byte for the length
891+
Dim _Data(_Size - 1) As Byte
892+
846893
If _UseMemoryMappedFile Then
847-
Dim _Byte1 As Byte
894+
Dim _Len As Byte
848895
Dim _Bytes() As Byte
849-
_Pos -= _MapDataOffset
850-
_Byte1 = _MapDataAccessor.ReadByte(_Pos)
851-
ReDim _Bytes(_Byte1 - 1)
852-
_MapDataAccessor.ReadArray(_Pos + 1, _Bytes, 0, _Byte1)
896+
_Pos -= _MapDataOffset ' position stored in BIN file is for full file, not just the mapped data segment, so need to minus
897+
_MapDataAccessor.ReadArray(Of Byte)(_Pos, _Data, 0, _Size)
898+
_Len = _Data(0)
899+
ReDim _Bytes(_Len - 1)
900+
Array.Copy(_Data, 1, _Bytes, 0, _Len)
853901
Return Encoding.Default.GetString(_Bytes)
854902
Else
855-
Dim _Bytes(0) As Byte
856-
Dim _Bytes2() As Byte
903+
Dim _Len As Byte
904+
Dim _Bytes() As Byte
857905
Myfilestream.Seek(_Pos, SeekOrigin.Begin)
858-
Myfilestream.Read(_Bytes, 0, 1)
859-
ReDim _Bytes2(_Bytes(0) - 1)
860-
Myfilestream.Read(_Bytes2, 0, _Bytes(0))
861-
Return Encoding.Default.GetString(_Bytes2)
906+
Myfilestream.Read(_Data, 0, _Size)
907+
_Len = _Data(0)
908+
ReDim _Bytes(_Len - 1)
909+
Array.Copy(_Data, 1, _Bytes, 0, _Len)
910+
Return Encoding.Default.GetString(_Bytes)
862911
End If
863912
End Function
864913

‎IP2ProxyComponent/IP2ProxyComponent/IP2ProxyComponent.vbproj‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@
44
<RootNamespace>IP2Proxy</RootNamespace>
55
<TargetFrameworks>netstandard2.0;net472</TargetFrameworks>
66
<AssemblyName>IP2Proxy</AssemblyName>
7-
<Version>3.2.0</Version>
7+
<Version>3.3.0</Version>
88
<Authors>IP2Proxy.com</Authors>
99
<Company>IP2Proxy.com</Company>
1010
<Product>IP2Proxy .NET Component</Product>
1111
<Title>IP2Proxy .NET Component</Title>
1212
<Description>IP2Proxy .NET Component</Description>
1313
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
14-
<Copyright>Copyright 2021</Copyright>
14+
<Copyright>Copyright 2022</Copyright>
1515
<PackageLicenseExpression>MIT</PackageLicenseExpression>
1616
<PackageProjectUrl>https://www.ip2location.com/development-libraries/ip2proxy/dot-net</PackageProjectUrl>
1717
<PackageIcon>ip2proxy-logo-square-128.png</PackageIcon>
18-
<PackageReleaseNotes>Added support for IP2Proxy Web Service.</PackageReleaseNotes>
18+
<PackageReleaseNotes>Reduced file I/O.</PackageReleaseNotes>
1919
<RepositoryUrl>https://github.com/ip2location/ip2proxy-dotnet.git</RepositoryUrl>
2020
<PackageTags>ip2proxy proxy detection</PackageTags>
2121
<RepositoryType>git</RepositoryType>

0 commit comments

Comments
 (0)