Algorithm improvements

Author: Decimation

SmartImage.Lib/Engines/Impl/YandexEngine.cs

@@ -4,6 +4,7 @@
 using System.Drawing;
 using System.Linq;
 using System.Text;
+using System.Text.Unicode;
 using AngleSharp.Dom;
 using AngleSharp.Html.Dom;
 using AngleSharp.XPath;
@@ -216,8 +217,9 @@ protected override SearchResult Process(IDocument doc, SearchResult sr)
 			if (looksLike != null) {
 				//todo
 
-				sr.PrimaryResult.Description = Encoding.Unicode.GetString(
-					Encoding.Convert(Encoding.UTF8, Encoding.Unicode, Encoding.UTF8.GetBytes(looksLike)));
+				// sr.PrimaryResult.Description = Encoding.UTF8.GetString(
+				// 	Encoding.Convert(Encoding.Unicode, Encoding.UTF8, Encoding.Unicode.GetBytes(looksLike)));
+				sr.PrimaryResult.Description = looksLike;
 			}
 
 			sr.OtherResults.AddRange(images);
@@ -232,5 +234,7 @@ protected override SearchResult Process(IDocument doc, SearchResult sr)
 
 			return sr;
 		}
+
+		
 	}
 }

SmartImage.Lib/SearchClient.cs

@@ -4,6 +4,7 @@
 using SmartImage.Lib.Searching;
 using SmartImage.Lib.Utilities;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Diagnostics;
@@ -213,7 +214,7 @@ public List<SearchResult> MaximizeResults<T>(Func<SearchResult, T> property)
 		public ImageResult[] FindDirectResults()
 		{
 
-			var best = FindBestResults();
+			var best = FindBestResults().ToList();
 
 			//const int FRAG_SIZE = 10;
 
@@ -235,31 +236,51 @@ public ImageResult[] FindDirectResults()
 
 			//Task.WaitAll(tasks.ToArray());
 
-			var best1 = best;
+			var bestCopy = best;
 
 			var options = new ParallelOptions()
 			{
 				MaxDegreeOfParallelism = Int32.MaxValue,
 				TaskScheduler          = TaskScheduler.Default,
 			};
 
-			Parallel.For(0, best.Length, options, (i) =>
+			Parallel.For(0, best.Count, options, (i) =>
 			{
-				best1[i].FindDirectImages();
+				bestCopy[i].FindDirectImages();
 
 			});
 
-			best = best1;
+
+			best = bestCopy;
+
+
+			bestCopy = best;
+
+
+			var rx = new ConcurrentBag<ImageResult>();
+
+			Parallel.For(0, best.Count, options,(i) =>
+			{
+				if (ImageHelper.IsDirect(bestCopy[i].Direct?.ToString(), DirectImageType.Binary))
+				{
+					Debug.WriteLine($"Adding {bestCopy[i].Direct}");
+					rx.Add(bestCopy[i]);
+				}
+			});
+
+
+			best = rx.ToList();
 
 			best = best.Where(x => x.Direct != null)
 			           .OrderByDescending(r => r.Similarity)
-			           .ToArray();
+			           .ToList();
+
+			//best = best.OrderByDescending(b => b.PixelResolution).ToList();
 
 
-			best = best.OrderByDescending(b => b.PixelResolution).ToArray();
-			Debug.WriteLine($"Found {best.Length} direct results");
+			Debug.WriteLine($"Found {best.Count} direct results");
 
-			return best;
+			return best.ToArray();
 		}
 
 		[CanBeNull]

SmartImage.Lib/Searching/ImageResult.cs

@@ -125,13 +125,13 @@ public ImageResult()
 			OtherMetadata = new Dictionary<string, object>();
 		}
 
-		public static ImageResult FromDirectImage(DirectImage di)
+		public static ImageResult FromDirectImage(string di)
 		{
 			var ir = new ImageResult
 			{
-				Image  = di.Image,
-				Url    = (di.Direct),
-				Direct = (di.Direct)
+				Image  = ImageHelper.GetImage(di),
+				Url    = new Uri((di)),
+				Direct = new Uri((di))
 			};
 
 			ir.UpdateImageData();
@@ -224,8 +224,7 @@ public void FindDirectImages()
 					var direct = directImages?.FirstOrDefault();
 
 					if (direct != null) {
-						Direct = (direct.Value.Direct);
-						Image  = direct.Value.Image;
+						Direct = new Uri((direct));
 						//Debug.WriteLine($"{Url} -> {Direct}");
 
 					}

SmartImage.Lib/Utilities/ImageHelper.cs

@@ -18,16 +18,11 @@
 
 // ReSharper disable CognitiveComplexity
 // ReSharper disable PossibleNullReferenceException
-
 // ReSharper disable UnusedParameter.Local
-
 // ReSharper disable PossibleMultipleEnumeration
 // ReSharper disable AssignNullToNotNullAttribute
-
 // ReSharper disable LoopCanBeConvertedToQuery
-
 // ReSharper disable InconsistentNaming
-
 // ReSharper disable UnusedMember.Global
 
 namespace SmartImage.Lib.Utilities
@@ -150,22 +145,21 @@ public static Dictionary<string, string> UtilitiesMap
 
 		#endregion
 
-
 		public static Image GetImage(string s)
 		{
-			// TODO: Using Image objects creates a memory leak; disposing them doesn't fix anything either (?)
-
 			using var wc = new WebClient();
 
-			byte[] buf = wc.DownloadData(s);
+			//byte[] buf = wc.DownloadData(s);
 
-			var stream = new MemoryStream(buf);
+			//using var stream = new MemoryStream(buf);
+			//Debug.WriteLine($"Alloc {buf.Length}");
 
-			Debug.WriteLine($"Alloc {buf.Length}");
+			using var stream = wc.OpenRead(s);
 
 			var image = Image.FromStream(stream);
 
 			return image;
+
 		}
 
 
@@ -176,13 +170,17 @@ public static Image GetImage(string s)
 		/// <param name="directType">Which criterion to use to determine whether a URI is a direct image </param>
 		/// <param name="count">Number of direct images to return</param>
 		/// <param name="pingTimeSec"></param>
-		/// <param name="readImage">Whether to read image metadata</param>
-		/// <param name="imageFilter">Filter criteria for images (applicable iff <paramref name="readImage"/> is <c>true</c>)</param>
-		public static List<DirectImage> FindDirectImages(string url, DirectImageType directType = DirectImageType.Regex,
-		                                                 int count = 5, double pingTimeSec = 1, bool readImage = true,
-		                                                 Predicate<Image> imageFilter = null)
+		public static List<string> FindDirectImages(string url, DirectImageType directType = DirectImageType.Regex,
+		                                            int count = 5, double pingTimeSec = 1)
 		{
-			var images = new List<DirectImage>();
+			/*
+			 * TODO
+			 *
+			 * This function creates an insane memory leak.
+			 * Disposing neither the images nor the streams does anything (?)
+			 */
+
+			var images = new List<string>();
 
 			string gallerydl = UtilitiesMap[GALLERY_DL_EXE];
 
@@ -206,21 +204,15 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
 
 				var standardOutput = output.StandardOutput;
 
+
 				while (!standardOutput.EndOfStream) {
 					string str = standardOutput.ReadLine()
 					                           .Split('|')
 					                           .First();
+					if (!string.IsNullOrWhiteSpace(str)) {
+						images.Add(str);
 
-					var di = new DirectImage
-					{
-						Direct = new Uri(str),
-					};
-
-					if (readImage) {
-						di.Image = GetImage(str);
 					}
-
-					images.Add(di);
 				}
 
 				var standardError = output.StandardError;
@@ -239,8 +231,6 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
 
 			manual:
 
-			imageFilter ??= (x) => true;
-
 			var pingTime = TimeSpan.FromSeconds(pingTimeSec);
 
 			IHtmlDocument document;
@@ -295,40 +285,13 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
 				if (!IsDirect(currentUrl, directType))
 					return;
 
-				var di = new DirectImage
-				{
-					Direct = new Uri(currentUrl)
-				};
-
-
-				bool isValid = !readImage;
-
-				if (readImage) {
-					try {
-						var img = GetImage(currentUrl);
-
-						isValid = imageFilter(img);
-
-						if (isValid) {
-							di.Image = img;
-
-						}
-						else {
-							img.Dispose();
-						}
-					}
-					catch (Exception) {
-						isValid = false;
-					}
-				}
 
 				if (imagesCopy.Count >= count) {
 					return;
 				}
 
-				if (isValid) {
-					imagesCopy.Add(di);
-				}
+				imagesCopy.Add(currentUrl);
+
 			});
 
 
@@ -372,9 +335,6 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
 
 			ret:
 
-			if (readImage) {
-				images = images.OrderByDescending(x => x.Image.Width * x.Image.Height).ToList();
-			}
 
 			return images;
 
@@ -391,23 +351,7 @@ internal static string AsPercent(this float n)
 		}
 	}
 
-	public struct DirectImage : IDisposable
-	{
-		public Uri Direct { get; internal set; }
-
-		[CanBeNull]
-		public Image Image { get; internal set; }
-
-		public override string ToString()
-		{
-			return $"{Direct} {Image?.Width}x{Image?.Height}";
-		}
-
-		public void Dispose()
-		{
-			Image?.Dispose();
-		}
-	}
+	
 
 	public enum DirectImageType
 	{

SmartImage/Core/AppInterface.cs

@@ -9,6 +9,7 @@
 using System.IO;
 using System.Linq;
 using System.Linq.Expressions;
+using System.Net;
 using System.Net.NetworkInformation;
 using System.Reflection;
 using System.Runtime.CompilerServices;
@@ -371,7 +372,8 @@ public static NConsoleOption[] CreateResultOptions(IEnumerable<ImageResult> resu
 			return result.Select(r => CreateResultOption(r, $"{n} #{i++}", c)).ToArray();
 		}
 
-		internal static NConsoleOption CreateResultOption(ImageResult result, string n, Color c, float correction=-.3f)
+		internal static NConsoleOption CreateResultOption(ImageResult result, string n, Color c,
+		                                                  float correction = -.3f)
 		{
 			var option = new NConsoleOption
 			{
@@ -447,19 +449,25 @@ public static void ShowToast()
 				var direct = Client.FindDirectResult();
 
 				Debug.WriteLine(direct);
-				//Debug.WriteLine($"{Network.IsAlive(direct.Direct)} {ImageHelper.IsDirect(direct.Direct.ToString(), DirectImageType.Binary)}");
+				Debug.WriteLine(direct.Direct.ToString());
 
 
+				if (direct is {Direct: { }}) {
 
-				if (direct is { Direct: { } })
-				{
 
+					var tmp = Path.GetTempPath();
 
-					string file = WebUtilities.Download(direct.Direct.ToString(), Path.GetTempPath());
+					string filename = Path.GetFileName(direct.Direct.AbsolutePath);
 
-					Debug.WriteLine($"Downloaded {file} tmp");
-					builder.AddHeroImage(new Uri(file));
+					var file = Path.Combine(tmp, filename);
+
+					new WebClient().DownloadFile(direct.Direct, file);
 
+					//string file     = WebUtilities.Download(abs.ToString(), s);
+
+					Debug.WriteLine($"Downloaded {file}");
+
+					builder.AddHeroImage(new Uri(file));
 
 					AppDomain.CurrentDomain.ProcessExit += (sender, args) =>
 					{