Skip to content

Commit 4103132

Browse files
author
邹嵩
committed
2 parents 3ecb3ca + 284da4f commit 4103132

File tree

117 files changed

+1167
-1806
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

117 files changed

+1167
-1806
lines changed

‎src/DotnetSpider.Core.Test/DotnetSpider.Core.Test.projitems‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
<Compile Include="$(MSBuildThisFileDirectory)EnvironmentTest.cs" />
4343
<Compile Include="$(MSBuildThisFileDirectory)HtmlTest.cs" />
4444
<Compile Include="$(MSBuildThisFileDirectory)HttpExecuteRecordTest.cs" />
45-
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\HttpSenderTest.cs" />
4645
<Compile Include="$(MSBuildThisFileDirectory)IocManagerTests.cs" />
4746
<Compile Include="$(MSBuildThisFileDirectory)PageTest.cs" />
4847
<Compile Include="$(MSBuildThisFileDirectory)Pipeline\FilePipelineTest.cs" />

‎src/DotnetSpider.Core.Test/Downloader/HttpClientPoolTest.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using DotnetSpider.Core.Downloader;
2+
using DotnetSpider.Core.Infrastructure;
23
using DotnetSpider.Core.Proxy;
34
using System;
45
using System.Collections.Concurrent;

‎src/DotnetSpider.Core.Test/HttpExecuteRecordTest.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,7 @@ public void Record()
1111
{
1212
if (Env.IsWindows)
1313
{
14-
var result = HttpSender.Request((new HttpRequest
15-
{
16-
Url = "http://localhost:30013"
17-
}));
14+
var result = HttpClientPool.HttpClient.GetAsync("http://localhost:30013").Result;
1815
if (result.StatusCode == HttpStatusCode.OK)
1916
{
2017
Env.HubServiceTaskApiUrl = "http://localhost:30013/api/v1.0/task";

‎src/DotnetSpider.Core.Test/Infrastructure/HttpSenderTest.cs‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎src/DotnetSpider.Core.Test/Scheduler/QueueSchedulerTest.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public void PushAndPollDepthFirst()
5959
public void PushAndPollBreadthFirst()
6060
{
6161
QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
62-
scheduler.TraverseStrategy = TraverseStrategy.BFS;
62+
scheduler.TraverseStrategy = TraverseStrategy.Bfs;
6363
ISpider spider = new DefaultSpider("test", new Site());
6464
scheduler.Init(spider);
6565

‎src/DotnetSpider.Core/AppBase.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
using DotnetSpider.Core.Infrastructure;
22
using Serilog;
33
using System;
4-
using System.Diagnostics;
54
using System.Threading.Tasks;
65

76
namespace DotnetSpider.Core
@@ -29,7 +28,7 @@ public abstract class AppBase : Named, IAppBase
2928
/// <summary>
3029
/// 唯一标识
3130
/// </summary>
32-
public virtual string Identity
31+
public string Identity
3332
{
3433
get => _identity;
3534
set

‎src/DotnetSpider.Core/DefaultSpider.cs‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,32 +7,32 @@
77
namespace DotnetSpider.Core
88
{
99
/// <summary>
10-
/// 默认爬虫, 用于测试和一些默认情况使用, 框架使用者可忽略
10+
/// 默认爬虫, 用于测试和一些默认情况使用, 框架使用者可忽略
1111
/// </summary>
12-
public class DefaultSpider : Spider
12+
public sealed class DefaultSpider : Spider
1313
{
1414
/// <summary>
15-
/// 构造方法
15+
/// 构造方法
1616
/// </summary>
1717
public DefaultSpider() : this(Guid.NewGuid().ToString("N"), new Site())
1818
{
1919
}
2020

2121
/// <summary>
22-
/// 构造方法
22+
/// 构造方法
2323
/// </summary>
24-
/// <param name="id">爬虫标识</param>
25-
/// <param name="site">网站信息</param>
24+
/// <param name="id">爬虫标识</param>
25+
/// <param name="site">网站信息</param>
2626
public DefaultSpider(string id, Site site) : this(id, site, new QueueDuplicateRemovedScheduler())
2727
{
2828
}
2929

3030
/// <summary>
31-
/// 构造方法
31+
/// 构造方法
3232
/// </summary>
33-
/// <param name="id">爬虫标识</param>
34-
/// <param name="site">网站信息</param>
35-
/// <param name="scheduler">URL队列</param>
33+
/// <param name="id">爬虫标识</param>
34+
/// <param name="site">网站信息</param>
35+
/// <param name="scheduler">URL队列</param>
3636
public DefaultSpider(string id, Site site, IScheduler scheduler) : base(site, id, scheduler, new[] { new SimplePageProcessor() }, new[] { new ConsolePipeline() })
3737
{
3838
Logger = LogUtil.Create(id);

‎src/DotnetSpider.Core/DotnetSpider.Core.projitems‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,11 @@
2020
<Compile Include="$(MSBuildThisFileDirectory)Downloader\FileCookieInject.cs" />
2121
<Compile Include="$(MSBuildThisFileDirectory)Downloader\FileDownloader.cs" />
2222
<Compile Include="$(MSBuildThisFileDirectory)Downloader\HttpClientDownloader.cs" />
23-
<Compile Include="$(MSBuildThisFileDirectory)Downloader\HttpClientPool.1.cs" />
2423
<Compile Include="$(MSBuildThisFileDirectory)Downloader\HttpClientPool.cs" />
2524
<Compile Include="$(MSBuildThisFileDirectory)Downloader\IAfterDownloadCompleteHandler.cs" />
2625
<Compile Include="$(MSBuildThisFileDirectory)Downloader\IBeforeDownloadHandler.cs" />
2726
<Compile Include="$(MSBuildThisFileDirectory)Downloader\ICookieInterceptor.cs" />
2827
<Compile Include="$(MSBuildThisFileDirectory)Downloader\IDownloader.cs" />
29-
<Compile Include="$(MSBuildThisFileDirectory)Downloader\IHttpClientPool.cs" />
3028
<Compile Include="$(MSBuildThisFileDirectory)Downloader\LoginHandler.cs" />
3129
<Compile Include="$(MSBuildThisFileDirectory)Env.cs" />
3230
<Compile Include="$(MSBuildThisFileDirectory)HttpExecuteRecord.cs" />
@@ -48,15 +46,15 @@
4846
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\Database\Database.cs" />
4947
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\Database\DatabaseExtensions.cs" />
5048
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\Database\DbProviderFactories.cs" />
51-
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\DataTypeNames.cs" />
5249
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\DateTimeUtil.cs" />
5350
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\Description.cs" />
5451
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\DirectoryExtensions.cs" />
5552
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\EncodingExtensions.cs" />
53+
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\HttpClientPool.cs" />
5654
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\HttpSink.cs" />
5755
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\HubService.cs" />
5856
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\FileUtil.cs" />
59-
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\HttpSender.cs" />
57+
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\IHttpClientPool.cs" />
6058
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\ILock.cs" />
6159
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\LogInfo.cs" />
6260
<Compile Include="$(MSBuildThisFileDirectory)Infrastructure\LogUtil.cs" />

‎src/DotnetSpider.Core/DotnetSpider.Core.shproj‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
1-
<?xml version="1.0" encoding="utf-8"?>
1+
<?xml version="1.0" encoding="utf-8"?>
22
<Project ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3-
<PropertyGroup Label="Globals">
4-
<ProjectGuid>{6C8BEE01-E343-486A-95C9-00B784FE66FB}</ProjectGuid>
5-
</PropertyGroup>
6-
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props"
7-
Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')"/>
8-
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.Common.Default.props"/>
9-
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.Common.props"/>
10-
<Import Project="DotnetSpider.Core.projitems" Label="Shared"/>
11-
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.CSharp.targets"/>
12-
</Project>
3+
<PropertyGroup Label="Globals">
4+
<ProjectGuid>{6C8BEE01-E343-486A-95C9-00B784FE66FB}</ProjectGuid>
5+
</PropertyGroup>
6+
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
7+
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.Common.Default.props" />
8+
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.Common.props" />
9+
<Import Project="DotnetSpider.Core.projitems" Label="Shared" />
10+
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.CSharp.targets" />
11+
</Project>

‎src/DotnetSpider.Core/Downloader/AfterDownloadCompleteHandler.Extensions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public TimingUpdateCookieHandler(int interval, ICookieInjector injector)
8686
throw new ArgumentException("interval should be large than 0.");
8787
}
8888

89-
_cookieInjector = injector ?? throw new ArgumentNullException("CookieInjector should not be null.");
89+
_cookieInjector = injector ?? throw new SpiderException($"{nameof(injector)} should not be null.");
9090
_next = DateTime.Now.AddSeconds(_interval);
9191
_interval = interval;
9292
}

0 commit comments

Comments
 (0)