默认Pipeline设为Console

邹嵩 · 邹嵩 · commit 0f3c94ced5bc · 2018-01-17T10:02:09.000+08:00
修复WebDriverDownloader的一个错误
支持设置Take的方向
diff --git a/.gitignore b/.gitignore
@@ -202,3 +202,4 @@ FakesAssemblies/
 *.opt
 
 /src/DotnetSpider.Test/Example/TestExample.cs
+/src/DotnetSpider.Sample/Class2.cs
diff --git a/nuget/DotnetSpider.Core.nuspec b/nuget/DotnetSpider.Core.nuspec
@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
   <metadata>
     <id>DotnetSpider2.Core</id>
-    <version>2.3.8</version>
+    <version>2.3.9-beta2</version>
     <authors>zlzforever@163.com;Walterwhatwater;xiaohuan0204</authors>
     <owners>zlzforever@163.com</owners>
     <iconUrl>https://github.com/zlzforever/DotnetSpider/blob/master/images/icon.png?raw=true</iconUrl>
diff --git a/nuget/DotnetSpider.Extension.nuspec b/nuget/DotnetSpider.Extension.nuspec
@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
   <metadata>
     <id>DotnetSpider2.Extension</id>
-    <version>2.3.8</version>
+    <version>2.3.9-beta2</version>
     <authors>zlzforever@163.com;Walterwhatwater;xiaohuan0204</authors>
     <owners>zlzforever@163.com</owners>
     <iconUrl>https://github.com/zlzforever/DotnetSpider/blob/master/images/icon.png?raw=true</iconUrl>
@@ -13,7 +13,7 @@
     <description>A .NET Standard web crawling library similar to WebMagic and Scrapy. It is a lightweight ,efficient and fast high-level web crawling &amp; scraping framework for .NET</description>
     <dependencies>
       <group targetFramework=".NETStandard2.0">
-        <dependency id="DotnetSpider2.Core" version="2.3.8" />
+        <dependency id="DotnetSpider2.Core" version="2.3.9-beta2" />
         <dependency id="Dapper" version="1.50.2"/>
         <dependency id="MailKit" version="1.20.0"/>
         <dependency id="MongoDB.Driver" version="2.4.4"/>
@@ -28,7 +28,7 @@
         <dependency id="MessagePack" version="1.7.2"/>
       </group>
       <group targetFramework=".NETFramework4.5" >
-        <dependency id="DotnetSpider2.Core" version="2.3.8" />
+        <dependency id="DotnetSpider2.Core" version="2.3.9-beta2" />
         <dependency id="Dapper" version="1.50.2"/> 
         <dependency id="MailKit" version="1.20.0"/>
         <dependency id="MongoDB.Driver" version="2.4.4"/>
diff --git a/src/DotnetSpider.Core/DefaultSpider.cs b/src/DotnetSpider.Core/DefaultSpider.cs
@@ -22,7 +22,7 @@ public class DefaultSpider : Spider
 		/// </summary>
 		/// <param name="id">�����ʶ</param>
 		/// <param name="site">��վ��Ϣ</param>
-		public DefaultSpider(string id, Site site) : base(site, id, new QueueDuplicateRemovedScheduler(), new[] { new SimplePageProcessor() }, new[] { new NullPipeline() })
+		public DefaultSpider(string id, Site site) : base(site, id, new QueueDuplicateRemovedScheduler(), new[] { new SimplePageProcessor() }, new[] { new ConsolePipeline() })
 		{
 		}
 
@@ -32,7 +32,7 @@ public class DefaultSpider : Spider
 		/// <param name="id">�����ʶ</param>
 		/// <param name="site">��վ��Ϣ</param>
 		/// <param name="scheduler">URL����</param>
-		public DefaultSpider(string id, Site site, IScheduler scheduler) : base(site, id, scheduler, new[] { new SimplePageProcessor() }, new[] { new NullPipeline() })
+		public DefaultSpider(string id, Site site, IScheduler scheduler) : base(site, id, scheduler, new[] { new SimplePageProcessor() }, new[] { new ConsolePipeline() })
 		{
 		}
 	}
diff --git a/src/DotnetSpider.Core/Pipeline/NullPipeline.cs b/src/DotnetSpider.Core/Pipeline/NullPipeline.cs
diff --git a/src/DotnetSpider.Core/Selector/BaseSelectable.cs b/src/DotnetSpider.Core/Selector/BaseSelectable.cs
@@ -161,6 +161,5 @@ public List<string> GetValues(bool isPlainText)
 		/// <param name="selector">查询器</param>
 		/// <returns>查询接口</returns>
 		public abstract ISelectable SelectList(ISelector selector);
-
 	}
 }
diff --git a/src/DotnetSpider.Core/Spider.cs b/src/DotnetSpider.Core/Spider.cs
@@ -978,7 +978,7 @@ protected virtual void InitComponent(params string[] arguments)
 		/// <returns>数据管道</returns>
 		protected virtual IPipeline GetDefaultPipeline()
 		{
-			return new NullPipeline();
+			return new ConsolePipeline();
 		}
 
 		/// <summary>
diff --git a/src/DotnetSpider.Extension/CommonSpider.cs b/src/DotnetSpider.Extension/CommonSpider.cs
@@ -82,7 +82,7 @@ protected override void RunApp(params string[] arguments)
 
 				if (Pipelines == null || Pipelines.Count == 0)
 				{
-					AddPipeline(new NullPipeline());
+					AddPipeline(new ConsolePipeline());
 				}
 				if (PageProcessors == null || PageProcessors.Count == 0)
 				{
diff --git a/src/DotnetSpider.Extension/Downloader/WebDriverDownloader.cs b/src/DotnetSpider.Extension/Downloader/WebDriverDownloader.cs
@@ -26,7 +26,7 @@ public class WebDriverDownloader : BaseDownloader
 		private readonly Browser _browser;
 		private readonly Option _option;
 		private bool _isDisposed;
-		private readonly string[] _domains;
+		private readonly List<string> _domains = new List<string>();
 		/// <summary>
 		/// 每次navigate完成后, WebDriver 需要执行的操作
 		/// </summary>
@@ -44,7 +44,16 @@ public WebDriverDownloader(Browser browser, string[] domains = null, int webDriv
 			_webDriverWaitTime = webDriverWaitTime;
 			_browser = browser;
 			_option = option ?? new Option();
-			_domains = domains;
+			if (domains != null)
+			{
+				foreach (var domain in domains)
+				{
+					if (!string.IsNullOrWhiteSpace(domain) && !_domains.Contains(domain))
+					{
+						_domains.Add(domain);
+					}
+				}
+			}
 
 			if (browser == Browser.Firefox && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
 			{
@@ -96,12 +105,15 @@ protected override Page DowloadContent(Request request, ISpider spider)
 					{
 						_webDriver = WebDriverUtil.Open(_browser, _option);
 
-						foreach (var domain in _domains)
+						if (_domains != null)
 						{
-							var cookies = CookieContainer.GetCookies(new Uri(domain));
-							foreach (System.Net.Cookie cookie in cookies)
+							foreach (var domain in _domains)
 							{
-								AddCookieToDownloadClient(cookie);
+								var cookies = CookieContainer.GetCookies(new Uri(domain));
+								foreach (System.Net.Cookie cookie in cookies)
+								{
+									AddCookieToDownloadClient(cookie);
+								}
 							}
 						}
 
@@ -177,6 +189,10 @@ protected override Page DowloadContent(Request request, ISpider spider)
 		[MethodImpl(MethodImplOptions.Synchronized)]
 		protected override void AddCookieToDownloadClient(System.Net.Cookie cookie)
 		{
+			if (!_domains.Contains(cookie.Domain))
+			{
+				_domains.Add(cookie.Domain);
+			}
 			_webDriver?.Manage().Cookies.AddCookie(new Cookie(cookie.Name, cookie.Value, cookie.Domain, cookie.Path, null));
 		}
 	}
diff --git a/src/DotnetSpider.Extension/Model/Attribute/EntitySelector.cs b/src/DotnetSpider.Extension/Model/Attribute/EntitySelector.cs
@@ -12,5 +12,10 @@ public class EntitySelector : SelectorAttribute
 		/// 从最终解析到的结果中取前 Take 个实体
 		/// </summary>
 		public int Take { get; set; } = -1;
+
+		/// <summary>
+		/// 设置 Take 的方向, 默认是从头部取
+		/// </summary>
+		public bool TakeFromHead { get; set; } = true;
 	}
 }
diff --git a/src/DotnetSpider.Extension/Model/EntityDefine.cs b/src/DotnetSpider.Extension/Model/EntityDefine.cs
@@ -48,6 +48,11 @@ public interface IEntityDefine
 		/// </summary>
 		int Take { get; }
 
+		/// <summary>
+		/// ���� Take �ķ���, Ĭ���Ǵ�ͷ��ȡ
+		/// </summary>
+		bool TakeFromHead { get; set; }
+
 		/// <summary>
 		/// Ŀ�����ӵ�ѡ����
 		/// </summary>
@@ -100,6 +105,11 @@ public class EntityDefine<T> : IEntityDefine
 		/// </summary>
 		public int Take { get; set; }
 
+		/// <summary>
+		/// ���� Take �ķ���, Ĭ���Ǵ�ͷ��ȡ
+		/// </summary>
+		public bool TakeFromHead { get; set; } = true;
+
 		/// <summary>
 		/// Ŀ�����ӵ�ѡ����
 		/// </summary>
@@ -143,6 +153,7 @@ public EntityDefine()
 			{
 				Multi = true;
 				Take = entitySelector.Take;
+				TakeFromHead = entitySelector.TakeFromHead;
 				SelectorAttribute = new SelectorAttribute { Expression = entitySelector.Expression, Type = entitySelector.Type };
 			}
 			else
diff --git a/src/DotnetSpider.Extension/Model/EntityExtractor.cs b/src/DotnetSpider.Extension/Model/EntityExtractor.cs
@@ -77,9 +77,16 @@ public List<T> Extract(Page page)
 				}
 				else
 				{
-					if (EntityDefine.Take > 0)
+					if (EntityDefine.Take > 0 && list.Count > EntityDefine.Take)
 					{
-						list = list.Take(EntityDefine.Take).ToList();
+						if (EntityDefine.TakeFromHead)
+						{
+							list = list.Take(EntityDefine.Take).ToList();
+						}
+						else
+						{
+							list = list.Skip(list.Count - EntityDefine.Take).ToList();
+						}
 					}
 
 					for (int i = 0; i < list.Count; ++i)
diff --git a/src/DotnetSpider.Extension/Pipeline/BaseEntityPipeline.cs b/src/DotnetSpider.Extension/Pipeline/BaseEntityPipeline.cs
@@ -133,7 +133,7 @@ public static IPipeline GetPipelineFromAppConfig()
 					}
 				default:
 					{
-						pipeline = new NullPipeline();
+						pipeline = new ConsolePipeline();
 						break;
 					}
 			}
diff --git a/src/DotnetSpider.Sample/Program.cs b/src/DotnetSpider.Sample/Program.cs
@@ -1,6 +1,7 @@
 ﻿using DotnetSpider.Core;
 using DotnetSpider.Core.Downloader;
 using DotnetSpider.Core.Infrastructure;
+using DotnetSpider.Core.Processor;
 using DotnetSpider.Extension.Test;
 using System;
 using System.Diagnostics;
@@ -23,6 +24,9 @@ public static void Main(string[] args)
 			ThreadPool.SetMinThreads(200, 200);
 			OcrDemo.Process();
 #endif
+			//string[] commands = "-s:AppIOSRankSpider -i:guid -tid:AppIOSRankSpider -c:app.config -a:游戏".Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
+			//Startup.Run(commands);
+
 			MyTest();
 
 			Startup.Run("-s:BaiduSearchSpider", "-tid:1", "-i:guid");
@@ -80,6 +84,17 @@ public static void Main(string[] args)
 		/// </summary>
 		private static void MyTest()
 		{
+			var site = new Site { CycleRetryTimes = 3, SleepTime = 300 };
+			Spider.Create(site, new GithubZlzforeverProcessor()).Run();
+		}
+
+		private class GithubZlzforeverProcessor : BasePageProcessor
+		{
+			protected override void Handle(Page page)
+			{
+				page.AddTargetRequests(page.Selectable.Links().Regex("(https://github\\.com/\\w+/\\w+)").GetValues());
+				page.AddResultItem("author", page.Selectable.XPath("").GetValue());
+			}
 		}
 	}
 

Original file line number	Diff line number	Diff line change
`@@ -202,3 +202,4 @@ FakesAssemblies/`
`202`	`202`	`*.opt`
`203`	`203`
`204`	`204`	`/src/DotnetSpider.Test/Example/TestExample.cs`
	`205`	`+/src/DotnetSpider.Sample/Class2.cs`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ public class DefaultSpider : Spider`
`22`	`22`	`/// </summary>`
`23`	`23`	`/// <param name="id">爬虫标识</param>`
`24`	`24`	`/// <param name="site">网站信息</param>`
`25`		`- public DefaultSpider(string id, Site site) : base(site, id, new QueueDuplicateRemovedScheduler(), new[] { new SimplePageProcessor() }, new[] { new NullPipeline() })`
	`25`	`+ public DefaultSpider(string id, Site site) : base(site, id, new QueueDuplicateRemovedScheduler(), new[] { new SimplePageProcessor() }, new[] { new ConsolePipeline() })`
`26`	`26`	`{`
`27`	`27`	`}`
`28`	`28`
`@@ -32,7 +32,7 @@ public class DefaultSpider : Spider`
`32`	`32`	`/// <param name="id">爬虫标识</param>`
`33`	`33`	`/// <param name="site">网站信息</param>`
`34`	`34`	`/// <param name="scheduler">URL队列</param>`
`35`		`- public DefaultSpider(string id, Site site, IScheduler scheduler) : base(site, id, scheduler, new[] { new SimplePageProcessor() }, new[] { new NullPipeline() })`
	`35`	`+ public DefaultSpider(string id, Site site, IScheduler scheduler) : base(site, id, scheduler, new[] { new SimplePageProcessor() }, new[] { new ConsolePipeline() })`
`36`	`36`	`{`
`37`	`37`	`}`
`38`	`38`	`}`
Original file line number	Diff line number	Diff line change
`@@ -161,6 +161,5 @@ public List<string> GetValues(bool isPlainText)`
`161`	`161`	`/// <param name="selector">查询器</param>`
`162`	`162`	`/// <returns>查询接口</returns>`
`163`	`163`	`public abstract ISelectable SelectList(ISelector selector);`
`164`		`-`
`165`	`164`	`}`
`166`	`165`	`}`
Original file line number	Diff line number	Diff line change
`@@ -978,7 +978,7 @@ protected virtual void InitComponent(params string[] arguments)`
`978`	`978`	`/// <returns>数据管道</returns>`
`979`	`979`	`protected virtual IPipeline GetDefaultPipeline()`
`980`	`980`	`{`
`981`		`- return new NullPipeline();`
	`981`	`+ return new ConsolePipeline();`
`982`	`982`	`}`
`983`	`983`
`984`	`984`	`/// <summary>`
Original file line number	Diff line number	Diff line change
`@@ -82,7 +82,7 @@ protected override void RunApp(params string[] arguments)`
`82`	`82`
`83`	`83`	`if (Pipelines == null \|\| Pipelines.Count == 0)`
`84`	`84`	`{`
`85`		`- AddPipeline(new NullPipeline());`
	`85`	`+ AddPipeline(new ConsolePipeline());`
`86`	`86`	`}`
`87`	`87`	`if (PageProcessors == null \|\| PageProcessors.Count == 0)`
`88`	`88`	`{`
Original file line number	Diff line number	Diff line change
`@@ -12,5 +12,10 @@ public class EntitySelector : SelectorAttribute`
`12`	`12`	`/// 从最终解析到的结果中取前 Take 个实体`
`13`	`13`	`/// </summary>`
`14`	`14`	`public int Take { get; set; } = -1;`
	`15`	`+`
	`16`	`+ /// <summary>`
	`17`	`+ /// 设置 Take 的方向, 默认是从头部取`
	`18`	`+ /// </summary>`
	`19`	`+ public bool TakeFromHead { get; set; } = true;`
`15`	`20`	`}`
`16`	`21`	`}`