Updates

Author: Decimation

SmartImage.Lib/SearchClient.cs

@@ -215,31 +215,50 @@ public ImageResult[] FindDirectResults()
 
 			var best = FindBestResults();
 
-			const int FRAG_SIZE = 10;
+			//const int FRAG_SIZE = 10;
 
-			var frag = best.Chunk(FRAG_SIZE).ToList();
+			//var frag = best.Chunk(FRAG_SIZE).ToList();
 
-			var tasks = new List<Task>();
+			//var tasks = new List<Task>();
 
-			for (int i = 0; i < frag.Count; i++) {
-				int iCopy = i;
+			//for (int i = 0; i < frag.Count; i++) {
+			//	int iCopy = i;
 
-				tasks.Add(Task.Factory.StartNew(() =>
-				{
-					foreach (var result in frag[iCopy]) {
-						result.FindDirectImages();
-					}
-				}));
-			}
+			//	tasks.Add(Task.Factory.StartNew(() =>
+			//	{
+			//		foreach (var result in frag[iCopy]) {
+			//			result.FindDirectImages();
+			//		}
+			//	}));
+			//}
+
+
+			//Task.WaitAll(tasks.ToArray());
 
+			var best1 = best;
 
-			Task.WaitAll(tasks.ToArray());
+			var options = new ParallelOptions()
+			{
+				MaxDegreeOfParallelism = Int32.MaxValue,
+				TaskScheduler          = TaskScheduler.Default,
+			};
+
+			Parallel.For(0, best.Length, options, (i) =>
+			{
+				best1[i].FindDirectImages();
+
+			});
+
+			best = best1;
 
 			best = best.Where(x => x.Direct != null)
 			           .OrderByDescending(r => r.Similarity)
-			           .ThenByDescending(i => i.PixelResolution)
 			           .ToArray();
 
+
+			best = best.OrderByDescending(b => b.PixelResolution).ToArray();
+			Debug.WriteLine($"Found {best.Length} direct results");
+
 			return best;
 		}
 

SmartImage.Lib/Searching/ImageResult.cs

@@ -213,7 +213,7 @@ public void FindDirectImages()
 			if (Url == null || Direct != null) 
 				return;
 
-			if (ImageHelper.IsDirect(Url.ToString())) {
+			if (ImageHelper.IsDirect(Url.ToString(), DirectImageType.Binary)) {
 				Direct = Url;
 			}
 			else {

SmartImage.Lib/Searching/SearchResult.cs

@@ -128,6 +128,8 @@ public void Consolidate()
 
 		public override string ToString() => Strings.ViewString(this);
 
+		
+
 		public Dictionary<string, object> View
 		{
 			get

SmartImage.Lib/Utilities/ImageHelper.cs

@@ -10,6 +10,7 @@
 using System.Threading.Tasks;
 using AngleSharp.Html.Dom;
 using AngleSharp.Html.Parser;
+using JetBrains.Annotations;
 using Novus.Win32;
 using SimpleCore.Net;
 using SimpleCore.Utilities;
@@ -150,6 +151,24 @@ public static Dictionary<string, string> UtilitiesMap
 		#endregion
 
 
+		public static Image GetImage(string s)
+		{
+			// TODO: Using Image objects creates a memory leak; disposing them doesn't fix anything either (?)
+
+			using var wc = new WebClient();
+
+			byte[] buf = wc.DownloadData(s);
+
+			var stream = new MemoryStream(buf);
+
+			Debug.WriteLine($"Alloc {buf.Length}");
+
+			var image = Image.FromStream(stream);
+
+			return image;
+		}
+
+
 		/// <summary>
 		/// Scans for direct images within a webpage.
 		/// </summary>
@@ -160,12 +179,12 @@ public static Dictionary<string, string> UtilitiesMap
 		/// <param name="readImage">Whether to read image metadata</param>
 		/// <param name="imageFilter">Filter criteria for images (applicable iff <paramref name="readImage"/> is <c>true</c>)</param>
 		public static List<DirectImage> FindDirectImages(string url, DirectImageType directType = DirectImageType.Regex,
-		                                                 int count = 5, double pingTimeSec = 1,
-		                                                 bool readImage = true, Predicate<Image> imageFilter = null)
+		                                                 int count = 5, double pingTimeSec = 1, bool readImage = true,
+		                                                 Predicate<Image> imageFilter = null)
 		{
-			var directImages = new List<DirectImage>();
+			var images = new List<DirectImage>();
 
-			/*string gallerydl = UtilitiesMap[GALLERY_DL_EXE];
+			string gallerydl = UtilitiesMap[GALLERY_DL_EXE];
 
 			if (gallerydl != null) {
 
@@ -188,56 +207,57 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
 				var standardOutput = output.StandardOutput;
 
 				while (!standardOutput.EndOfStream) {
-					string str = standardOutput.ReadLine().Trim().Trim('|');
+					string str = standardOutput.ReadLine()
+					                           .Split('|')
+					                           .First();
 
-					directImages.Add(new DirectImage
+					var di = new DirectImage
 					{
 						Direct = new Uri(str),
-						Image  = Image.FromStream(WebUtilities.GetStream(str))
-					});
-					//Debug.WriteLine($">>{d}");
+					};
+
+					if (readImage) {
+						di.Image = GetImage(str);
+					}
+
+					images.Add(di);
 				}
 
 				var standardError = output.StandardError;
 
 				while (!standardError.EndOfStream) {
-					var line = standardError.ReadLine();
-					Debug.WriteLine($"{GALLERY_DL_EXE}: {line}", C_ERROR);
+					string line = standardError.ReadLine();
 
-					if (line!=null) {
+					if (line != null) {
 						goto manual;
 					}
 				}
 
 
-
-				return directImages.OrderByDescending(x => x.Image.Width * x.Image.Height).ToList();
+				goto ret;
 			}
 
-			manual:*/
-
-			var sw = Stopwatch.StartNew();
+			manual:
 
 			imageFilter ??= (x) => true;
 
 			var pingTime = TimeSpan.FromSeconds(pingTimeSec);
 
-
 			IHtmlDocument document;
 
 			try {
 				string html   = WebUtilities.GetString(url);
 				var    parser = new HtmlParser();
 
 				document = parser.ParseDocument(html);
+
 			}
 			catch (Exception e) {
 				Debug.WriteLine($"{e.Message}");
 
 				return null;
 			}
 
-
 			var cts = new CancellationTokenSource();
 
 			var flat = new List<string>();
@@ -247,77 +267,69 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
 
 			flat = flat.Distinct().ToList();
 
-			//var fragments = flat.Chunk(fragmentSize).ToArray();
-
-			//var tasks = new List<Task>();
-
-			//count = Math.Clamp(count, count, flat.Count);
-			var act = new List<Action>();
-
-			for (int i = 0; i < flat.Count; i++) {
-
-				int iCopy = i;
+			var options = new ParallelOptions
+			{
+				MaxDegreeOfParallelism = Int32.MaxValue,
+				TaskScheduler          = TaskScheduler.Default,
+				CancellationToken      = cts.Token
+			};
 
-				var imagesCopy = directImages;
+			var imagesCopy = images;
 
-				void Function()
-				{
-					string currentUrl = flat[iCopy];
-
-					if (directImages.Count >= count) {
-						return;
-					}
+			Parallel.For(0, flat.Count, options, i =>
+			{
+				string currentUrl = flat[i];
 
-					if (!Network.IsUri(currentUrl, out var uri)) 
-						return;
+				if (imagesCopy.Count >= count) {
+					return;
+				}
 
-					if (!Network.IsAlive(uri, (long) pingTime.TotalMilliseconds)) 
-						return;
+				if (!Network.IsUri(currentUrl, out var uri))
+					return;
 
-					if (!IsDirect(currentUrl, directType)) 
-						return;
+				if (!Network.IsAlive(uri, (long) pingTime.TotalMilliseconds)) {
+					Debug.WriteLine($"{uri} isn't alive");
+					return;
+				}
 
-					var di = new DirectImage {Direct = new Uri(currentUrl)};
+				if (!IsDirect(currentUrl, directType))
+					return;
 
+				var di = new DirectImage
+				{
+					Direct = new Uri(currentUrl)
+				};
 
-					bool isValid = !readImage;
 
-					if (readImage) {
-						var stream = WebUtilities.GetStream(currentUrl);
+				bool isValid = !readImage;
 
-						if (stream.CanRead) {
+				if (readImage) {
+					try {
+						var img = GetImage(currentUrl);
 
-							try {
-								var img = Image.FromStream(stream);
-								//isValid = true;
+						isValid = imageFilter(img);
 
-								//Debug.WriteLine($"{img.Width} {img.Height}");
-								isValid = imageFilter(img);
+						if (isValid) {
+							di.Image = img;
 
-								if (isValid) {
-									di.Image = img;
-								}
-							}
-							catch (Exception) {
-								isValid = false;
-							}
+						}
+						else {
+							img.Dispose();
 						}
 					}
-
-					if (directImages.Count >= count) {
-						return;
+					catch (Exception) {
+						isValid = false;
 					}
+				}
 
-					if (isValid) {
-						imagesCopy.Add(di);
-						//Debug.WriteLine($">>> {currentUrl}");
-
-					}
+				if (imagesCopy.Count >= count) {
+					return;
 				}
 
-				//tasks.Add(Task.Factory.StartNew(function, cts.Token));
-				act.Add(Function);
-			}
+				if (isValid) {
+					imagesCopy.Add(di);
+				}
+			});
 
 
 			/*
@@ -343,24 +355,28 @@ void Function()
 			 * 12		3.52
 			 * 13		3.63
 			 * 14		3.52
+			 *
+			 * 15		3.39
+			 * 16		3.52
+			 * 17		3.58
+			 * 18		3.47
+			 * 19		3.33
+			 *
+			 * 20		3.13
+			 * 21		2.89
+			 * 22		2.87
+			 * 23		2.89
 			 */
 
-			//Task.WaitAll(tasks.ToArray());
-
-
-			Parallel.Invoke(new ParallelOptions()
-			{
-				MaxDegreeOfParallelism = Int32.MaxValue,
-				TaskScheduler          = TaskScheduler.Default,
-				CancellationToken      = cts.Token
-			}, act.ToArray());
-
-			sw.Stop();
+			images = imagesCopy;
 
-			Trace.WriteLine($"{sw.Elapsed.TotalSeconds}");
+			ret:
 
+			if (readImage) {
+				images = images.OrderByDescending(x => x.Image.Width * x.Image.Height).ToList();
+			}
 
-			return directImages.OrderByDescending(x => x.Image.Width * x.Image.Height).ToList();
+			return images;
 
 		}
 
@@ -375,15 +391,21 @@ internal static string AsPercent(this float n)
 		}
 	}
 
-	public struct DirectImage
+	public struct DirectImage : IDisposable
 	{
 		public Uri Direct { get; internal set; }
 
+		[CanBeNull]
 		public Image Image { get; internal set; }
 
 		public override string ToString()
 		{
-			return $"{Direct} {Image.Width}x{Image.Height}";
+			return $"{Direct} {Image?.Width}x{Image?.Height}";
+		}
+
+		public void Dispose()
+		{
+			Image?.Dispose();
 		}
 	}