修复数据字段长度有大小限制

Author: 邹嵩

src/DotnetSpider.Extension/Pipeline/MySqlEntityPipeline.cs

@@ -284,7 +284,7 @@ private string GetDataTypeSql(DataType type, int length)
 					}
 				default:
 					{
-						dataType = (length <= 0) ? "LONGTEXT" : $"VARCHAR({length})";
+						dataType = length <= 0 || length > 8000 ? "LONGTEXT" : $"VARCHAR({length})";
 						break;
 					}
 			}

src/DotnetSpider.Extension/Pipeline/SqlServerEntityPipeline.cs

@@ -267,7 +267,7 @@ private string GetDataTypeSql(DataType type, int length)
 					}
 				default:
 					{
-						dataType = length <= 0 ? "NVARCHAR(MAX)" : $"NVARCHAR({length})";
+						dataType = length <= 0 || length >= 8000 ? "NVARCHAR(MAX)" : $"NVARCHAR({length})";
 						break;
 					}
 			}
@@ -288,7 +288,7 @@ protected override Sqls GenerateSqls(IModel model)
 			var sqls = new Sqls();
 
 			sqls.InsertSql = GenerateInsertSql(model);
-			sqls.InsertAndIgnoreDuplicateSql= GenerateInsertSql(model);
+			sqls.InsertAndIgnoreDuplicateSql = GenerateInsertSql(model);
 			sqls.UpdateSql = GenerateUpdateSql(model);
 			sqls.SelectSql = GenerateSelectSql(model);
 			return sqls;

src/DotnetSpider.Sample/Program.cs

@@ -27,7 +27,8 @@ public static void Main(string[] args)
 			OcrDemo.Process();
 #endif
 
-			ModelSpider.Run();
+			TestSpider spider = new TestSpider();
+			spider.Run();
 		}
 
 

src/netstandard2.0/DotnetSpider.Sample/TestSpider.cs

@@ -23,7 +23,7 @@ protected override void MyInit(params string[] arguments)
 			var word = "可乐|雪碧";
 			AddStartUrl(string.Format("http://news.baidu.com/ns?word={0}&tn=news&from=news&cl=2&pn=0&rn=20&ct=1", word), new Dictionary<string, dynamic> { { "Keyword", word } });
 			AddEntityType<BaiduSearchEntry>();
-			AddPipeline(new MySqlEntityPipeline("Database='mysql';Data Source=localhost;User ID=root;Port=3306;SslMode=None;"));
+			AddPipeline(new SqlServerEntityPipeline("Server=.\\SQLEXPRESS;Database=master;Trusted_Connection=True;MultipleActiveResultSets=true"));
 		}
 
 		[TableInfo("baidu", "baidu_search_entity_model")]
@@ -45,16 +45,16 @@ class BaiduSearchEntry : BaseEntity
 			[ReplaceFormatter(NewValue = "-", OldValue = "&nbsp;")]
 			public string Website { get; set; }
 
-			[Field(Expression = ".//div/span/a[@class='c-cache']/@href")]
+			[Field(Expression = ".//div/span/a[@class='c-cache']/@href", Length = 0)]
 			public string Snapshot { get; set; }
 
-			[Field(Expression = ".//div[@class='c-summary c-row ']", Option = FieldOptions.InnerText)]
+			[Field(Expression = ".//div[@class='c-summary c-row ']", Option = FieldOptions.InnerText, Length = 0)]
 			[ReplaceFormatter(NewValue = "", OldValue = "<em>")]
 			[ReplaceFormatter(NewValue = "", OldValue = "</em>")]
 			[ReplaceFormatter(NewValue = " ", OldValue = "&nbsp;")]
 			public string Details { get; set; }
 
-			[Field(Expression = ".", Option = FieldOptions.InnerText)]
+			[Field(Expression = ".", Option = FieldOptions.InnerText, Length = 0)]
 			[ReplaceFormatter(NewValue = "", OldValue = "<em>")]
 			[ReplaceFormatter(NewValue = "", OldValue = "</em>")]
 			[ReplaceFormatter(NewValue = " ", OldValue = "&nbsp;")]