Image scanning

Author: Decimation

SmartImage.Lib/Engines/Impl/IqdbEngine.cs

@@ -123,7 +123,7 @@ private IDocument GetDocument(ImageQuery query)
 			rq.AddHeader("Content-Type", "multipart/form-data");
 
 			byte[] fileBytes = Array.Empty<byte>();
-			object uri       = string.Empty;
+			object uri       = String.Empty;
 
 			if (query.IsFile) {
 				fileBytes = File.ReadAllBytes(query.Value);

SmartImage.Lib/Utilities/ImageHelper.cs

@@ -3,6 +3,7 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
 using AngleSharp;
 using AngleSharp.Dom;
@@ -12,6 +13,7 @@
 using AngleSharp.XPath;
 using Newtonsoft.Json.Linq;
 using SimpleCore.Net;
+using SimpleCore.Utilities;
 using static SimpleCore.Diagnostics.LogCategories;
 using MimeType = SimpleCore.Net.MimeType;
 
@@ -99,6 +101,100 @@ public static bool IsDirect2(string url)
 				RegexOptions.IgnoreCase);
 		}
 
+		/// <summary>
+		/// Break a list of items into chunks of a specific size
+		/// </summary>
+		public static IEnumerable<IEnumerable<T>> Chunk<T>(this IEnumerable<T> source, int chunksize)
+		{
+			while (source.Any()) {
+				yield return source.Take(chunksize);
+				source = source.Skip(chunksize);
+			}
+		}
+
+		public static List<string> FindDirectImagesEx(string url)
+		{
+
+			var rg = new List<string>();
+
+			//<img.*?src="(.*?)"
+			//href\s*=\s*"(.+?)"
+			//var src  = "<img.*?src=\"(.*?)\"";
+			//var href = "href\\s*=\\s*\"(.+?)\"";
+
+			string html;
+
+			try {
+				html = WebUtilities.GetString(url);
+			}
+			catch (Exception e) {
+				Debug.WriteLine($"{e.Message}", C_ERROR);
+				return null;
+			}
+
+			var p = new HtmlParser();
+			var d = p.ParseDocument(html);
+
+			var img = d.QuerySelectorAll("img");
+			var a   = d.QuerySelectorAll("a");
+
+			//Debug.WriteLine($"{img.Length} | {a.Length}");
+
+			//rg.AddRange(img.Select(s=>s.GetAttribute("src")));
+			CancellationTokenSource cts = new CancellationTokenSource();
+
+			var flat = new List<string>();
+
+			flat.AddRange(a.Select(s => s.GetAttribute("href")).ToList());
+			flat.AddRange(img.Select(s => s.GetAttribute("src")));
+			
+			var seg = flat.Chunk(10).ToArray();
+			var trg = new List<Task>();
+			
+
+			for (int i = 0; i < seg.Length; i++) {
+				//ThreadPool.QueueUserWorkItem(GetWorker(v.ToList()), cts.Token);
+				//var t=new Thread(() =>GetWorker(v.ToList()));
+				//t.Priority = ThreadPriority.AboveNormal;
+				//trg.Add(t);
+				//t.Start();
+
+				int i1 = i;
+
+				trg.Add(Task.Factory.StartNew(() =>
+				{
+					Debug.WriteLine("Init");
+
+					foreach (string s in seg[i1]) {
+
+						if (Network.IsUri(s, out var u2) && Network.IsUriAlive(u2, TimeSpan.FromSeconds(1))) {
+
+							var vb = IsDirect2(s);
+
+							if (vb) {
+								rg.Add(s);
+								Debug.WriteLine($">>>{s}");
+							}
+
+
+						}
+					}
+				}));
+
+			}
+
+
+			//var c=ThreadPool.PendingWorkItemCount;
+
+			Task.WaitAll(trg.ToArray());
+			//while (trg.Any(t => !t.IsCompleted)) { } //spin wait
+			//SpinWait.SpinUntil(() => ThreadPool.PendingWorkItemCount == 0);
+			//SpinWait.SpinUntil(() => trg.All(x=>!x.IsAlive));
+
+			return rg;
+
+		}
+
 
 		/// <summary>
 		/// Scans for direct image links in <paramref name="url"/>

Test/Program.cs

@@ -141,17 +141,23 @@ public static async Task Main(string[] args)
 			}*/
 
 
-			var i  = new IqdbEngine();
-			var i2 = i.GetResultAsync(q);
-			var r2 = await i2;
+			//var i  = new IqdbEngine();
+			//var i2 = i.GetResultAsync(q);
+			//var r2 = await i2;
 
-			Console.WriteLine(">> {0}", r2);
+			//Console.WriteLine(">> {0}", r2);
 
-			var ix = new SauceNaoEngine() { };
-			var i2x = ix.GetResultAsync(q);
-			var r2x = await i2x;
+			//var ix = new SauceNaoEngine() { };
+			//var i2x = ix.GetResultAsync(q);
+			//var r2x = await i2x;
 
-			Console.WriteLine(">> {0}", r2x);
+			//Console.WriteLine(">> {0}", r2x);
+
+			var t = ImageHelper.FindDirectImagesEx("https://www.zerochan.net/2750747");
+
+			foreach (var s in t) {
+				Console.WriteLine(s);
+			}
 		}
 	}
 }

UnitTest/UnitTest1.cs

@@ -84,6 +84,34 @@ public async Task TestSauceNao(string art, string name)
 			Assert.True(a || b);
 
 
+		}
+
+		[Test]
+		[TestCase(@"C:\Users\Deci\Pictures\Test Images\Test1.jpg")]
+		[TestCase(@"C:\Users\Deci\Pictures\Test Images\Test2.jpg")]
+		public async Task TestIqdb(string art)
+		{
+			var q  = new ImageQuery(art);
+			var i  = new IqdbEngine();
+			var rt = i.GetResultAsync(q);
+			var t  = await rt;
+
+			if (t.Status == ResultStatus.Unavailable) {
+				Assert.Inconclusive();
+			}
+
+			//t.Consolidate();
+
+
+			var a = t.IsNonPrimitive;
+
+
+			var b = t.OtherResults.Any(r =>
+			{
+				return r.DetailScore >= 3 && r.Site!=null;
+			});
+
+			Assert.True(a || b);
 
 
 		}