Skip to content

Commit 9315ed8

Browse files
author
邹嵩
committed
修正一个自动重定向的问题
1 parent 4103132 commit 9315ed8

File tree

4 files changed

+75
-243
lines changed

4 files changed

+75
-243
lines changed

‎src/DotnetSpider.Core/DotnetSpider.Core.projitems‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
<Compile Include="$(MSBuildThisFileDirectory)Downloader\FileCookieInject.cs" />
2121
<Compile Include="$(MSBuildThisFileDirectory)Downloader\FileDownloader.cs" />
2222
<Compile Include="$(MSBuildThisFileDirectory)Downloader\HttpClientDownloader.cs" />
23-
<Compile Include="$(MSBuildThisFileDirectory)Downloader\HttpClientPool.cs" />
2423
<Compile Include="$(MSBuildThisFileDirectory)Downloader\IAfterDownloadCompleteHandler.cs" />
2524
<Compile Include="$(MSBuildThisFileDirectory)Downloader\IBeforeDownloadHandler.cs" />
2625
<Compile Include="$(MSBuildThisFileDirectory)Downloader\ICookieInterceptor.cs" />
@@ -39,6 +38,7 @@
3938
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\BloomFilter.cs" />
4039
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\Browser.cs" />
4140
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\Cache.cs" />
41+
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\GlobalRedirectHandler.cs" />
4242
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\ConcurrentDictionaryExtensions.cs" />
4343
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\ConsoleHelper.cs" />
4444
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\ContentType.cs" />

‎src/DotnetSpider.Core/Downloader/HttpClientPool.cs‎

Lines changed: 0 additions & 185 deletions
This file was deleted.

‎src/DotnetSpider.Core/Infrastructure/GlobalRedirectHandler.cs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
using System;
2+
using System.Net;
3+
using System.Net.Http;
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
7+
namespace DotnetSpider.Core.Infrastructure
8+
{
9+
public class GlobalRedirectHandler : DelegatingHandler
10+
{
11+
public GlobalRedirectHandler() : this(new HttpClientHandler
12+
{
13+
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
14+
UseProxy = true,
15+
UseCookies = true,
16+
AllowAutoRedirect = false,
17+
MaxAutomaticRedirections = 10
18+
})
19+
{
20+
}
21+
22+
public GlobalRedirectHandler(HttpMessageHandler innerHandler)
23+
{
24+
InnerHandler = innerHandler;
25+
}
26+
27+
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
28+
{
29+
var response = await base.SendAsync(request, cancellationToken);
30+
31+
if (response.StatusCode == HttpStatusCode.MovedPermanently
32+
|| response.StatusCode == HttpStatusCode.Moved
33+
|| response.StatusCode == HttpStatusCode.Redirect
34+
|| response.StatusCode == HttpStatusCode.Found
35+
|| response.StatusCode == HttpStatusCode.SeeOther
36+
|| response.StatusCode == HttpStatusCode.RedirectKeepVerb
37+
|| response.StatusCode == HttpStatusCode.TemporaryRedirect
38+
|| (int)response.StatusCode == 308)
39+
{
40+
41+
var newRequest = CopyRequest(response.RequestMessage);
42+
43+
if (response.StatusCode == HttpStatusCode.Redirect
44+
|| response.StatusCode == HttpStatusCode.Found
45+
|| response.StatusCode == HttpStatusCode.SeeOther)
46+
{
47+
newRequest.Content = null;
48+
newRequest.Method = HttpMethod.Get;
49+
}
50+
newRequest.RequestUri = new Uri(response.RequestMessage.RequestUri, response.Headers.Location);
51+
52+
response = await SendAsync(newRequest, cancellationToken);
53+
}
54+
return response;
55+
}
56+
57+
private static HttpRequestMessage CopyRequest(HttpRequestMessage oldRequest)
58+
{
59+
var newrequest = new HttpRequestMessage(oldRequest.Method, oldRequest.RequestUri);
60+
61+
foreach (var header in oldRequest.Headers)
62+
{
63+
newrequest.Headers.TryAddWithoutValidation(header.Key, header.Value);
64+
}
65+
foreach (var property in oldRequest.Properties)
66+
{
67+
newrequest.Properties.Add(property);
68+
}
69+
if (oldRequest.Content != null) newrequest.Content = new StreamContent(oldRequest.Content.ReadAsStreamAsync().Result);
70+
return newrequest;
71+
}
72+
}
73+
}

‎src/DotnetSpider.Core/Infrastructure/IHttpClientPool.cs‎

Lines changed: 1 addition & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
using System.Net;
33
using System.Net.Http;
44
using System.Runtime.CompilerServices;
5-
using System.Threading;
6-
using System.Threading.Tasks;
75

86
namespace DotnetSpider.Core.Infrastructure
97
{
@@ -29,7 +27,7 @@ internal void Init(bool allowAutoRedirect, Action configAction, Func<CookieConta
2927
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
3028
UseProxy = true,
3129
UseCookies = true,
32-
AllowAutoRedirect = allowAutoRedirect,
30+
AllowAutoRedirect = false,
3331
MaxAutomaticRedirections = 10
3432
};
3533
Client = allowAutoRedirect ? new HttpClient(new GlobalRedirectHandler(Handler)) : new HttpClient(Handler);
@@ -41,60 +39,6 @@ internal void Init(bool allowAutoRedirect, Action configAction, Func<CookieConta
4139

4240
_inited = true;
4341
}
44-
45-
public class GlobalRedirectHandler : DelegatingHandler
46-
{
47-
public GlobalRedirectHandler(HttpMessageHandler innerHandler)
48-
{
49-
InnerHandler = innerHandler;
50-
}
51-
52-
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
53-
{
54-
var response = await base.SendAsync(request, cancellationToken);
55-
56-
if (response.StatusCode == HttpStatusCode.MovedPermanently
57-
|| response.StatusCode == HttpStatusCode.Moved
58-
|| response.StatusCode == HttpStatusCode.Redirect
59-
|| response.StatusCode == HttpStatusCode.Found
60-
|| response.StatusCode == HttpStatusCode.SeeOther
61-
|| response.StatusCode == HttpStatusCode.RedirectKeepVerb
62-
|| response.StatusCode == HttpStatusCode.TemporaryRedirect
63-
|| (int)response.StatusCode == 308)
64-
{
65-
66-
var newRequest = CopyRequest(response.RequestMessage);
67-
68-
if (response.StatusCode == HttpStatusCode.Redirect
69-
|| response.StatusCode == HttpStatusCode.Found
70-
|| response.StatusCode == HttpStatusCode.SeeOther)
71-
{
72-
newRequest.Content = null;
73-
newRequest.Method = HttpMethod.Get;
74-
}
75-
newRequest.RequestUri = new Uri(response.RequestMessage.RequestUri, response.Headers.Location);
76-
77-
response = await SendAsync(newRequest, cancellationToken);
78-
}
79-
return response;
80-
}
81-
82-
private static HttpRequestMessage CopyRequest(HttpRequestMessage oldRequest)
83-
{
84-
var newrequest = new HttpRequestMessage(oldRequest.Method, oldRequest.RequestUri);
85-
86-
foreach (var header in oldRequest.Headers)
87-
{
88-
newrequest.Headers.TryAddWithoutValidation(header.Key, header.Value);
89-
}
90-
foreach (var property in oldRequest.Properties)
91-
{
92-
newrequest.Properties.Add(property);
93-
}
94-
if (oldRequest.Content != null) newrequest.Content = new StreamContent(oldRequest.Content.ReadAsStreamAsync().Result);
95-
return newrequest;
96-
}
97-
}
9842
}
9943
/// <summary>
10044
/// Httpclient pool impletion for <see cref="IHttpClientPool"/>

0 commit comments

Comments
 (0)