11Imports System.Net.Http.Headers
22Imports System.Net.Sockets
33Imports System.Runtime.CompilerServices
4- Imports System.Security.Policy
54Imports System.Threading.Tasks
6- Imports CacheCow.Client
75
86Public Module ModNet
97
@@ -75,6 +73,11 @@ Retry:
7573 Catch ex As ThreadInterruptedException
7674 Throw
7775 Catch ex As Exception
76+ If TypeOf ex Is ResponsedWebException Then
77+ If CType (ex, ResponsedWebException).StatusCode = HttpStatusCode.Forbidden Then Throw
78+ If CType (ex, ResponsedWebException).StatusCode = 429 Then Thread.Sleep( 10000 ) 'Too Many Requests
79+ End If
80+ '重试
7881 Select Case RetryCount
7982 Case 0
8083 RetryException = ex
@@ -245,8 +248,20 @@ RequestFinished:
245248 SecretHeadersSign(Url, Request, SimulateBrowserHeaders)
246249 'DNS 解析
247250 CancelToken = New CancellationTokenSource(Timeout)
248- HostIp = DnsLookup(Request, CancelToken)
251+ HostIp = DNSLookup(Request, CancelToken)
252+ If HostIp IsNot Nothing AndAlso Not IPReliability.ContainsKey(HostIp) Then
253+ IPReliability(HostIp) = - 0.01 '预先降低一点,这样快速的重复请求会使用不同的 IP 以提高成功率
254+ End If
249255 '发送请求
256+ SyncLock RequestClientLock
257+ '延迟初始化,以避免在程序启动前加载 CacheCow 导致 DLL 加载失败
258+ If RequestClient Is Nothing Then
259+ RequestClient = CacheCow.Client.ClientExtensions.CreateClient( New CacheCow.Client.FileCacheStore.FileStore(PathTemp & "Cache/Http/" ), New HttpClientHandler With {
260+ .AutomaticDecompression = DecompressionMethods.Deflate Or DecompressionMethods.GZip Or DecompressionMethods.None,
261+ .UseCookies = False '不设为 False 就不能从 Header 手动传入 Cookies
262+ })
263+ End If
264+ End SyncLock
250265 Response = RequestClient.SendAsync(Request, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token).GetAwaiter().GetResult()
251266 Dim ResponseStream = Response.Content.ReadAsStreamAsync().GetAwaiter().GetResult()
252267 Dim ResponseBytes As Byte ()
@@ -263,18 +278,18 @@ RequestFinished:
263278 Dim ResponseMessage = If (Encoding, Encoding.UTF8).GetString(ResponseBytes)
264279 Throw New ResponsedWebException(
265280 $"错误码 {Response.StatusCode} ({CInt(Response.StatusCode)}),{Method},{Url},{HostIp}" &
266- If ( String .IsNullOrEmpty(ResponseMessage), "" , vbCrLf & ResponseMessage), ResponseMessage)
281+ If ( String .IsNullOrEmpty(ResponseMessage), "" , vbCrLf & ResponseMessage), Response.StatusCode, ResponseMessage)
267282 End If
268283 Catch ex As ThreadInterruptedException
269284 Throw
270285 Catch ex As ResponsedWebException
271286 Throw
272- Catch ex As DnsLookupException
273- Throw
274287 Catch ex As Exception
275288 RecordIPReliability(HostIp, - 1 )
276289 If TypeOf ex Is OperationCanceledException Then 'CancellationToken 超时
277- Throw New WebException( $"连接服务器超时,请稍后再试,或使用 VPN 改善网络环境({Method}, {Url},{HostIp})" , WebExceptionStatus.Timeout)
290+ Throw New WebException( $"连接服务器超时,请稍后再试,或使用 VPN 改善网络环境({Method}, {Url},IP:{HostIp})" , WebExceptionStatus.Timeout)
291+ ElseIf ex.IsNetworkRelated Then
292+ Throw New WebException( $"网络请求失败,请稍后再试,或使用 VPN 改善网络环境({Method}, {Url},IP:{HostIp})" , WebExceptionStatus.Timeout)
278293 Else
279294 Throw New Exception( $"网络请求出现意外异常({Method}, {Url},{HostIp})" , ex)
280295 End If
@@ -284,17 +299,15 @@ RequestFinished:
284299 Response?.Dispose()
285300 End Try
286301 End Function
287- Private RequestClient As HttpClient = ClientExtensions.CreateClient( New FileCacheStore.FileStore(PathTemp & "Cache/Http/" ), New HttpClientHandler With {
288- .AutomaticDecompression = DecompressionMethods.Deflate Or DecompressionMethods.GZip Or DecompressionMethods.None,
289- .UseCookies = False '不设为 False 就不能从 Header 手动传入 Cookies
290- })
302+ Private RequestClient As HttpClient = Nothing
303+ Private RequestClientLock As New Object
291304
292305 ''' <summary>
293306 ''' 进行 DNS 解析。它仅在选择的 IP 与系统默认的不一致时才对 URL 中的 Host 进行替换。
294307 ''' 返回请求时应使用的 IP;若为 IPv6,则加上了方括号。
308+ ''' 若解析失败,则返回 Nothing。
295309 ''' </summary>
296- ''' <exception cref="DnsLookupException"></exception>
297- Private Function DnsLookup(Request As HttpRequestMessage, CancelToken As CancellationTokenSource) As String
310+ Private Function DNSLookup(Request As HttpRequestMessage, CancelToken As CancellationTokenSource) As String
298311 Dim GetIPReliability = Function ( Key ) IPReliability.GetOrDefault( Key .ToString, 0 )
299312 '初步 DNS 解析
300313 Dim Host = Request.RequestUri.Host
@@ -303,10 +316,14 @@ RequestFinished:
303316 DnsTask = Dns.GetHostAddressesAsync(Host)
304317 DnsTask.Wait(CancelToken.Token)
305318 Catch ex As Exception
306- Throw New DnsLookupException( $"DNS 解析失败,请检查你的网络连接({Host})" , ex)
319+ Log(ex, $"DNS 解析失败({Host})" )
320+ Return Nothing
307321 End Try
308322 Dim Candidates As IPAddress() = DnsTask.Result.ToArray
309- If Not Candidates.Any Then Throw New DnsLookupException( $"DNS 解析无结果,请检查你的网络连接({Host})" )
323+ If Not Candidates.Any Then
324+ Log( $"[Net] DNS 解析无结果({Host})" )
325+ Return Nothing
326+ End If
310327 '若同时存在 IPv4 和 IPv6 地址,仅选择其中一类(因为 GFW 可能只屏蔽了 IPv4 或 IPv6)
311328 Dim IPv4Targets = Candidates.Where( Function (i) i.AddressFamily = AddressFamily.InterNetwork).ToArray
312329 Dim IPv6Targets = Candidates.Where( Function (i) i.AddressFamily = AddressFamily.InterNetworkV6).ToArray
@@ -322,20 +339,19 @@ RequestFinished:
322339 Dim TargetIp As String = If (Target.AddressFamily = AddressFamily.InterNetworkV6, $"[{Target}]" , Target.ToString)
323340 If Target IsNot Candidates.First Then
324341 '优选结果与系统默认 IP 不一致,替换域名并设置 Host 头
325- If ModeDebug Then Log( $"[Net] DNS 结果 :{Host} → {Target},所有可能的 IP 与可靠度:{DnsTask.Result.Select(Function(i) $" {i} → {GetIPReliability(i): 0.000 } ").Join(" ;")}" )
342+ If ModeDebug Then Log( $"[Net] DNS 解析结果 :{Host} → {Target},所有可能的 IP 与可靠度:{DnsTask.Result.Select(Function(i) $" {i} → {GetIPReliability(i): 0.000 } ").Join(" ;")}" )
326343 Request.Headers.Host = Host
327344 Dim Builder As New UriBuilder(Request.RequestUri)
328345 Builder.Host = TargetIp
329346 Request.RequestUri = Builder.Uri
330347 End If
331- If Not IPReliability.ContainsKey(TargetIp) Then IPReliability(TargetIp) = - 0.01 '预先降低一点,这样快速的重复请求会使用不同的 IP 以提高成功率
332348 Return TargetIp
333349 End Function
334350 ''' <summary>
335351 ''' 记录每个 IP 地址的请求可靠度。
336352 ''' 通常取值 -1 ~ +0.5,越高越好。未尝试过的 IP 应视为 0。
337353 ''' </summary>
338- Private IPReliability As New Dictionary (Of String , Double )
354+ Private IPReliability As New SafeDictionary (Of String , Double )
339355 ''' <summary>
340356 ''' 根据请求结果,记录 IP 地址的可靠度。
341357 ''' </summary>
@@ -346,29 +362,22 @@ RequestFinished:
346362 End Sub
347363
348364 ''' <summary>
349- ''' DNS 解析失败引发的异常。
350- ''' </summary>
351- Private Class DnsLookupException
352- Inherits Exception
353- Public Sub New (Message As String , Optional InnerException As Exception = Nothing )
354- MyBase .New(Message, InnerException)
355- End Sub
356- Public Sub New (Message As String )
357- MyBase .New(Message)
358- End Sub
359- End Class
360- ''' <summary>
361- ''' 当网络请求失败时引发的异常。
362- ''' 附带 Response 属性,可用于获取远程服务器给予的回复。
365+ ''' 当 HTTP 状态码不指示成功时引发的异常。
366+ ''' 附带额外属性,可用于获取远程服务器给予的回复以及 HTTP 状态码。
363367 ''' </summary>
364368 Public Class ResponsedWebException
365369 Inherits WebException
366370 ''' <summary>
371+ ''' HTTP 状态码。
372+ ''' </summary>
373+ Public StatusCode As HttpStatusCode
374+ ''' <summary>
367375 ''' 远程服务器给予的回复。
368376 ''' </summary>
369377 Public Overloads Property Response As String
370- Public Sub New (Message As String , Response As String )
378+ Public Sub New (Message As String , StatusCode As HttpStatusCode, Response As String )
371379 MyBase .New(Message)
380+ Me .StatusCode = StatusCode
372381 Me .Response = Response
373382 End Sub
374383 End Class
@@ -750,7 +759,7 @@ RequestFinished:
750759 ''' </summary>
751760 Public Check As FileChecker
752761 ''' <summary>
753- ''' 下载时是否添加浏览器 UA 。
762+ ''' 是否模拟浏览器的 UserAgent 和 Referer 。
754763 ''' </summary>
755764 Public SimulateBrowserHeaders As Boolean
756765
@@ -969,7 +978,7 @@ StartThread:
969978 If ModeDebug OrElse Th.DownloadStart = 0 Then Log( $"[Download] {LocalName}:开始,起始点 {Th.DownloadStart},{Th.Source.Url}" )
970979 Dim ResultStream As Stream = Nothing , HttpRequest As HttpRequestMessage = Nothing ,
971980 Response As HttpResponseMessage = Nothing , ResponseStream As Stream = Nothing ,
972- CancelToken As CancellationTokenSource = Nothing , HostIp As String = ""
981+ CancelToken As CancellationTokenSource = Nothing , HostIp As String = Nothing
973982 '部分下载源真的特别慢,并且只需要一个请求,例如 Ping 为 20s,如果增长太慢,就会造成类似 2.5s 5s 7.5s 10s 12.5s... 的极大延迟
974983 '延迟过长会导致某些特别慢的链接迟迟不被掐死
975984 Dim Timeout As Integer = Math.Min(Math.Max(ConnectAverage, 6000 ) * ( 1 + Th.Source.FailCount), 30000 )
@@ -980,7 +989,7 @@ StartThread:
980989 HttpRequest = New HttpRequestMessage(HttpMethod.Get, Th.Source.Url)
981990 SecretHeadersSign(Th.Source.Url, HttpRequest, SimulateBrowserHeaders)
982991 CancelToken = New CancellationTokenSource(Timeout)
983- HostIp = DnsLookup (HttpRequest, CancelToken) 'DNS 预解析
992+ HostIp = DNSLookup (HttpRequest, CancelToken) 'DNS 预解析
984993 If Not Th.IsFirstThread Then HttpRequest.Headers.Range = New RangeHeaderValue(Th.DownloadStart, Nothing )
985994 Dim ContentLength As Long = 0
986995 Response = ThreadClient.SendAsync(HttpRequest, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token).GetAwaiter().GetResult()
@@ -1126,7 +1135,7 @@ SourceBreak:
11261135 RecordIPReliability(HostIp, 0.5 )
11271136 End If
11281137 Catch ex As Exception
1129- Log( $"[Download] {LocalName}:出错,{If(TypeOf ex Is OperationCanceledException, $" 已超时 ({Timeout}ms )", ex.GetDetail())},{HostIp}" )
1138+ Log( $"[Download] {LocalName}:出错,{If(TypeOf ex Is OperationCanceledException, $" 已超时 ({Timeout}ms )", ex.GetDetail())},IP: {HostIp}" )
11301139 RecordIPReliability(HostIp, - 0.7 )
11311140 SourceFail(Th, ex, False )
11321141 Finally
@@ -1864,23 +1873,27 @@ Retry:
18641873 End Sub
18651874 Private IsManagerStarted As Boolean = False
18661875
1867- 'Public FileRemainList As New List(Of String)
1876+ Private DownloadCacheLock As New Object
18681877 Private IsDownloadCacheCleared As Boolean = False
18691878 ''' <summary>
18701879 ''' 开始一个下载任务。
18711880 ''' </summary>
18721881 Public Sub Start(Task As LoaderDownload)
18731882 StartManager()
18741883 '清理缓存
1875- If Not IsDownloadCacheCleared Then
1876- Try
1877- DeleteDirectory(PathTemp & "Download" )
1878- Catch ex As Exception
1879- Log(ex, "清理下载缓存失败" )
1880- End Try
1884+ SyncLock DownloadCacheLock '防止同时开启多个下载任务时重复清理
1885+ If Not IsDownloadCacheCleared Then
1886+ Try
1887+ Log( "[Net] 开始清理下载缓存" )
1888+ DeleteDirectory(PathTemp & "Download" )
1889+ Log( "[Net] 下载缓存已清理" )
1890+ Catch ex As Exception
1891+ Log(ex, "清理下载缓存失败" )
1892+ End Try
1893+ Directory.CreateDirectory(PathTemp & "Download" )
1894+ End If
18811895 IsDownloadCacheCleared = True
1882- End If
1883- Directory.CreateDirectory(PathTemp & "Download" )
1896+ End SyncLock
18841897 '文件处理
18851898 SyncLock LockFiles
18861899 '添加每个文件
@@ -1959,11 +1972,12 @@ Retry:
19591972 ''' 判断某个 Exception 是否为网络问题所导致。
19601973 ''' </summary>
19611974 <Extension> Public Function IsNetworkRelated(Ex As Exception) As Boolean
1962- If TypeOf Ex Is DnsLookupException Then Return True
19631975 Dim Detail = Ex.GetDetail()
19641976 If Detail.Contains( "(403)" ) Then Return False
1965- Return { "(408)" , "超时" , "timeout" , "网络请求失败" , "连接尝试失败" , "远程主机强迫关闭了" , "远程方已关闭传输流" , "未能解析此远程名称" , "由于目标计算机积极拒绝" }.
1966- Any( Function (k) Detail.ContainsF(k, True ))
1977+ Return {
1978+ "(408)" , "超时" , "timeout" , "网络请求失败" , "连接尝试失败" , "远程主机强迫关闭了" , "远程方已关闭传输流" , "未能解析此远程名称" ,
1979+ "由于目标计算机积极拒绝" , "基础连接已经关闭"
1980+ }.Any( Function (k) Detail.ContainsF(k, True ))
19671981 End Function
19681982
19691983End Module
0 commit comments