Filtering

Decimation · Decimation · commit 23368f21d7fe · 2021-01-14T01:25:21.000-06:00
diff --git a/SmartImage/Core/SearchConfig.cs b/SmartImage/Core/SearchConfig.cs
@@ -8,6 +8,7 @@
 using SmartImage.Engines;
 using SmartImage.Engines.Imgur;
 using SmartImage.Engines.SauceNao;
+using SmartImage.Searching;
 using SmartImage.Utilities;
 
 #pragma warning disable HAA0502, HAA0302, HAA0505, HAA0601, HAA0301, HAA0501, HAA0101, HAA0102, RCS1036
@@ -82,6 +83,7 @@ private SearchConfig()
 
 		/// <summary>
 		///     Does not open results from priority engines if the result similarity (if available) is below a certain threshold.
+		/// <see cref="ISearchResult.Filter"/> is <c>true</c> if <see cref="ISearchEngine.FilterThreshold"/> is less than <see cref="ISearchResult.Similarity"/>
 		/// </summary>
 		[field: ConfigComponent("filter_results", "--filter-results", true, true)]
 		public bool FilterResults { get; set; }
diff --git a/SmartImage/Engines/BasicSearchEngine.cs b/SmartImage/Engines/BasicSearchEngine.cs
@@ -20,6 +20,9 @@ protected BasicSearchEngine(string baseUrl)
 
 		public abstract Color Color { get; }
 
+		
+		public virtual float? FilterThreshold => null;
+
 		public virtual FullSearchResult GetResult(string url)
 		{
 			string rawUrl = GetRawResultUrl(url);
diff --git a/SmartImage/Engines/ISearchEngine.cs b/SmartImage/Engines/ISearchEngine.cs
@@ -12,5 +12,7 @@ public interface ISearchEngine
 		public FullSearchResult GetResult(string url);
 
 		public Color Color { get; }
+		
+		public float? FilterThreshold { get; }
 	}
 }
diff --git a/SmartImage/Engines/Imgur/ImgurClient.cs b/SmartImage/Engines/Imgur/ImgurClient.cs
@@ -50,7 +50,7 @@ public string Upload(string path)
 			var response = client.Execute(request);
 
 			var des = new JsonDeserializer();
-			return des.Deserialize<ImgurResponse<ImgurImage>>(response).Data.Link;
+			return des.Deserialize<ImgurDataResponse<ImgurImage>>(response).Data.Link;
 		}
 	}
 }
diff --git a/SmartImage/Engines/Imgur/ImgurDataResponse.cs b/SmartImage/Engines/Imgur/ImgurDataResponse.cs
@@ -3,7 +3,7 @@
 namespace SmartImage.Engines.Imgur
 {
 	[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
-	internal sealed class ImgurResponse<T>
+	internal sealed class ImgurDataResponse<T>
 	{
 		public T Data { get; set; }
 
diff --git a/SmartImage/Engines/Other/IqdbEngine.cs b/SmartImage/Engines/Other/IqdbEngine.cs
@@ -22,9 +22,12 @@ public IqdbEngine() : base("https://iqdb.org/?url=") { }
 
 		public override SearchEngineOptions Engine => SearchEngineOptions.Iqdb;
 
+		public override float? FilterThreshold => 70.00F;
+
 		private struct IqdbResult : ISearchResult
 		{
-			public bool?   Filter  { get; set; }
+			public bool    Filter  { get; set; }
+			
 			public string? Caption { get; set; }
 
 			public string Source { get; }
@@ -45,36 +48,38 @@ public IqdbResult(string caption, string source, string url, int width, int heig
 				Width      = width;
 				Height     = height;
 				Similarity = similarity;
-				Filter     = null;
+				Filter     = false; // set later
 			}
 
 			public override string ToString()
 			{
 				return
-					$"{nameof(Caption)}: {Caption}, {nameof(Source)}: {Source}, {nameof(Width)}: {Width}, {nameof(Height)}: {Height}, {nameof(Url)}: {Url}, {nameof(Similarity)}: {Similarity}";
+					$"{nameof(Caption)}: {Caption}, {nameof(Source)}: "       +
+					$"{Source}, {nameof(Width)}: {Width}, {nameof(Height)}: " +
+					$"{Height}, {nameof(Url)}: {Url}, {nameof(Similarity)}: {Similarity}";
 			}
 		}
 
-		private static IqdbResult ParseResult(HtmlNodeCollection tr)
+		private IqdbResult ParseResult(HtmlNodeCollection tr)
 		{
 			var caption = tr[0];
-			var img = tr[1];
-			var src = tr[2];
+			var img     = tr[1];
+			var src     = tr[2];
 
 			string url = null!;
 
 			var urlNode = img.FirstChild.FirstChild;
 
 			if (urlNode.Name != "img") {
-				var origUrl= urlNode.Attributes["href"].Value;
+				var origUrl = urlNode.Attributes["href"].Value;
 
 				Debug.WriteLine(origUrl);
-				
+
 				// Links must begin with http:// in order to work with "start"
 				if (origUrl.StartsWith("//")) {
 					origUrl = "http:" + origUrl;
 				}
-				
+
 
 				url = origUrl;
 			}
@@ -83,7 +88,7 @@ private static IqdbResult ParseResult(HtmlNodeCollection tr)
 
 			if (tr.Count >= 4) {
 				var res = tr[3];
-				var wh = res.InnerText.Split("�");
+				var wh  = res.InnerText.Split("�");
 
 				var wStr = wh[0].SelectOnlyDigits();
 				w = int.Parse(wStr);
@@ -98,7 +103,7 @@ private static IqdbResult ParseResult(HtmlNodeCollection tr)
 
 			if (tr.Count >= 5) {
 				var simNode = tr[4];
-				var simStr = simNode.InnerText.Split('%')[0];
+				var simStr  = simNode.InnerText.Split('%')[0];
 				sim = float.Parse(simStr);
 			}
 			else {
@@ -107,7 +112,7 @@ private static IqdbResult ParseResult(HtmlNodeCollection tr)
 
 
 			var i = new IqdbResult(caption.InnerText, src.InnerText, url, w, h, sim);
-
+			i.Filter = i.Similarity < FilterThreshold;
 			return i;
 		}
 
@@ -116,20 +121,20 @@ public override FullSearchResult GetResult(string url)
 			var sr = base.GetResult(url);
 
 			try {
-				
+
 				var html = Network.GetSimpleResponse(sr.RawUrl);
 
 				//Network.WriteResponse(html);
 
 				var doc = new HtmlDocument();
 				doc.LoadHtml(html.Content);
-				
+
 
 				//var tables = doc.DocumentNode.SelectNodes("//table");
 
 				// Don't select other results
 
-				var pages = doc.DocumentNode.SelectSingleNode("//div[@id='pages']");
+				var pages  = doc.DocumentNode.SelectSingleNode("//div[@id='pages']");
 				var tables = pages.SelectNodes("div/table");
 
 				// No relevant results?
@@ -155,8 +160,9 @@ public override FullSearchResult GetResult(string url)
 				images.RemoveAt(0);
 
 				var best = images[0];
-				sr.Url = best.Url;
+				sr.Url        = best.Url;
 				sr.Similarity = best.Similarity;
+				sr.Filter     = best.Filter;
 
 				sr.AddExtendedResults(images.ToArray());
 
diff --git a/SmartImage/Engines/Other/YandexEngine.cs b/SmartImage/Engines/Other/YandexEngine.cs
@@ -23,7 +23,7 @@ public YandexEngine() : base("https://yandex.com/images/search?rpt=imageview&url
 
 		private struct YandexResult : ISearchResult
 		{
-			public bool?  Filter     { get; set; }
+			public bool  Filter     { get; set; }
 			public float? Similarity { get; set; }
 
 			public int? Width { get; set; }
@@ -42,7 +42,7 @@ internal YandexResult(int width, int height, string url)
 				Url        = url;
 				Caption    = null;
 				Similarity = null;
-				Filter     = null;
+				Filter     = false;
 			}
 
 			public override string ToString()
diff --git a/SmartImage/Engines/SauceNao/SauceNaoDataResponse.cs b/SmartImage/Engines/SauceNao/SauceNaoDataResponse.cs
@@ -4,14 +4,14 @@
 namespace SmartImage.Engines.SauceNao
 {
 	[DataContract]
-	public class SauceNaoResponse
+	public class SauceNaoDataResponse
 	{
 		//ignore
 		//[DeserializeAs(Name = "header")]
 		//public object Header { get; set; }
 
 		[DataMember(Name = "results")]
-		public SauceNaoResult[] Results { get; set; }
+		public SauceNaoDataResult[] Results { get; set; }
 
 		public override string ToString()
 		{
diff --git a/SmartImage/Engines/SauceNao/SauceNaoDataResult.cs b/SmartImage/Engines/SauceNao/SauceNaoDataResult.cs
@@ -4,7 +4,7 @@
 namespace SmartImage.Engines.SauceNao
 {
 	[DataContract]
-	public class SauceNaoResult
+	public class SauceNaoDataResult
 	{
 		/// <summary>
 		///     The url(s) where the source is from. Multiple will be returned if the exact same image is found in multiple places
diff --git a/SmartImage/Engines/SauceNao/SauceNaoEngine.cs b/SmartImage/Engines/SauceNao/SauceNaoEngine.cs
@@ -39,11 +39,13 @@ public sealed class SauceNaoEngine : BasicSearchEngine
 		public override SearchEngineOptions Engine => SearchEngineOptions.SauceNao;
 
 		public override Color Color => Color.OrangeRed;
+		
+		public override float? FilterThreshold => 70.00F;
 
 		private struct SauceNaoSimpleResult : ISearchResult
 		{
 			public string? Caption    { get; set; }
-			public bool?   Filter     { get; set; }
+			public bool   Filter     { get; set; }
 			public string  Url        { get; set; }
 			public float?  Similarity { get; set; }
 			public int?    Width      { get; set; }
@@ -56,7 +58,7 @@ public SauceNaoSimpleResult(string? title, string url, float? similarity)
 				Similarity = similarity;
 				Width      = null;
 				Height     = null;
-				Filter     = null;
+				Filter     = false;//set later
 			}
 
 			public override string ToString()
@@ -81,15 +83,16 @@ private SauceNaoEngine(string apiKey) : base(BASIC_RESULT)
 
 		public SauceNaoEngine() : this(SearchConfig.Config.SauceNaoAuth) { }
 
-		private static ISearchResult[] ConvertResults(SauceNaoResult[] results)
+		private ISearchResult[] ConvertResults(SauceNaoDataResult[] results)
 		{
 			var rg = new List<ISearchResult>();
 
 			foreach (var sn in results) {
 				if (sn.Url != null) {
 					var url = sn.Url.FirstOrDefault(u => u != null);
-
-					rg.Add(new SauceNaoSimpleResult(sn.WebsiteTitle, url, sn.Similarity));
+					var x   = new SauceNaoSimpleResult(sn.WebsiteTitle, url, sn.Similarity);
+					x.Filter = x.Similarity < FilterThreshold;
+					rg.Add(x);
 				}
 			}
 
@@ -113,11 +116,12 @@ public override FullSearchResult GetResult(string url)
 					.Where(e => e.Url != null)
 					.OrderByDescending(e => e.Similarity)
 					.First();
-
+				
+				// Copy
 				result.Url        = best.Url;
 				result.Similarity = best.Similarity;
 				result.Caption    = best.Caption;
-
+				result.Filter     = best.Filter;
 
 				result.AddExtendedResults(extended);
 
@@ -136,7 +140,7 @@ public override FullSearchResult GetResult(string url)
 		}
 		
 		
-		private IEnumerable<SauceNaoResult>? GetResults(string url)
+		private IEnumerable<SauceNaoDataResult>? GetResults(string url)
 		{
 
 			var req = new RestRequest();
@@ -154,7 +158,7 @@ public override FullSearchResult GetResult(string url)
 		}
 
 
-		private static IEnumerable<SauceNaoResult>? ReadResults(string js)
+		private static IEnumerable<SauceNaoDataResult>? ReadResults(string js)
 		{
 			// todo: rewrite this using Newtonsoft
 
@@ -180,8 +184,8 @@ public override FullSearchResult GetResult(string url)
 				using var stream =
 					JsonReaderWriterFactory.CreateJsonReader(Encoding.UTF8.GetBytes(json),
 						XmlDictionaryReaderQuotas.Max);
-				var serializer = new DataContractJsonSerializer(typeof(SauceNaoResponse));
-				var result     = serializer.ReadObject(stream) as SauceNaoResponse;
+				var serializer = new DataContractJsonSerializer(typeof(SauceNaoDataResponse));
+				var result     = serializer.ReadObject(stream) as SauceNaoDataResponse;
 				stream.Dispose();
 
 				if (result is null)
diff --git a/SmartImage/Engines/TraceMoe/TraceMoeDataResult.cs b/SmartImage/Engines/TraceMoe/TraceMoeDataResult.cs
diff --git a/SmartImage/Engines/TraceMoe/TraceMoeEngine.cs b/SmartImage/Engines/TraceMoe/TraceMoeEngine.cs
@@ -31,8 +31,7 @@ private static TraceMoeRootObject GetApiResults(string url, out HttpStatusCode c
 			code = re.StatusCode;
 			status = re.ResponseStatus;
 			msg = re.ErrorMessage;
-
-			// todo: null sometimes
+			
 			return re.Data;
 		}
 
@@ -62,7 +61,9 @@ private ISearchResult[] ConvertResults(TraceMoeRootObject obj)
 		//https://myanimelist.net/anime/{id}/
 		private const string MAL_URL = "https://myanimelist.net/anime/";
 
-		public const double THRESHOLD = 87.00D;
+		
+
+		public override float? FilterThreshold => 87.00F;
 
 		public override FullSearchResult GetResult(string url)
 		{
@@ -80,12 +81,8 @@ public override FullSearchResult GetResult(string url)
 
 					r         = new FullSearchResult(this, best.Url, best.Similarity);
 					r.Caption = best.Caption;
-
-					if (r.Similarity < THRESHOLD) {
-						r.ExtendedInfo.Add($"Similarity below threshold ({THRESHOLD}%)");
-						r.Filter = true;
-					}
-
+					r.Filter  = r.Similarity < FilterThreshold;
+					
 
 					r.AddExtendedResults(results);
 				}
diff --git a/SmartImage/Searching/FullSearchResult.cs b/SmartImage/Searching/FullSearchResult.cs
@@ -140,7 +140,7 @@ public override NConsoleFunction ComboFunction
 		public int? Width { get; set; }
 		
 
-		public bool? Filter { get; set; }
+		public bool Filter { get; set; }
 		
 		
 
@@ -171,7 +171,7 @@ public override string ToString()
 				? String.Format($"({ExtendedResults.Count})")
 				: String.Empty;
 
-			var fstr = Filter.HasValue && Filter.Value ? $"{Formatting.BALLOT_X}" : "";
+			var fstr = Filter ? $"{Formatting.BALLOT_X}" : "";
 			
 			sb.Append($"{attrSuccess} {ex} {fstr}\n");
 
diff --git a/SmartImage/Searching/ISearchResult.cs b/SmartImage/Searching/ISearchResult.cs
@@ -1,4 +1,6 @@
-﻿#nullable enable
+﻿using SmartImage.Core;
+
+#nullable enable
 namespace SmartImage.Searching
 {
 	/// <summary>
@@ -36,7 +38,13 @@ public interface ISearchResult
 		/// </summary>
 		public string? Caption { get; set; }
 		
-		public bool? Filter { get; set; }
+		
+		/// <summary>
+		/// Filter this result if <see cref="SearchConfig.FilterResults"/> is used
+		/// </summary>
+		public bool Filter { get; set; }
+		
+		
 
 		
 	}
diff --git a/SmartImage/Searching/SearchClient.cs b/SmartImage/Searching/SearchClient.cs
@@ -260,9 +260,7 @@ private static void HandleResultOpen(FullSearchResult result)
 			 * Determine if it passes the threshold
 			 */
 
-			bool openResult = result.Filter.HasValue && !result.Filter.Value;
-
-			if (openResult) {
+			if (!result.Filter) {
 				// Open result
 				result.Function();
 			}

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,9 @@ protected BasicSearchEngine(string baseUrl)`
`20`	`20`
`21`	`21`	`public abstract Color Color { get; }`
`22`	`22`
	`23`	`+`
	`24`	`+ public virtual float? FilterThreshold => null;`
	`25`	`+`
`23`	`26`	`public virtual FullSearchResult GetResult(string url)`
`24`	`27`	`{`
`25`	`28`	`string rawUrl = GetRawResultUrl(url);`
Original file line number	Diff line number	Diff line change
`@@ -12,5 +12,7 @@ public interface ISearchEngine`
`12`	`12`	`public FullSearchResult GetResult(string url);`
`13`	`13`
`14`	`14`	`public Color Color { get; }`
	`15`	`+`
	`16`	`+ public float? FilterThreshold { get; }`
`15`	`17`	`}`
`16`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ public string Upload(string path)`
`50`	`50`	`var response = client.Execute(request);`
`51`	`51`
`52`	`52`	`var des = new JsonDeserializer();`
`53`		`- return des.Deserialize<ImgurResponse<ImgurImage>>(response).Data.Link;`
	`53`	`+ return des.Deserialize<ImgurDataResponse<ImgurImage>>(response).Data.Link;`
`54`	`54`	`}`
`55`	`55`	`}`
`56`	`56`	`}`
Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@`
`3`	`3`	`namespace SmartImage.Engines.Imgur`
`4`	`4`	`{`
`5`	`5`	`[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]`
`6`		`- internal sealed class ImgurResponse<T>`
	`6`	`+ internal sealed class ImgurDataResponse<T>`
`7`	`7`	`{`
`8`	`8`	`public T Data { get; set; }`
`9`	`9`
Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ public YandexEngine() : base("https://yandex.com/images/search?rpt=imageview&url`
`23`	`23`
`24`	`24`	`private struct YandexResult : ISearchResult`
`25`	`25`	`{`
`26`		`- public bool? Filter { get; set; }`
	`26`	`+ public bool Filter { get; set; }`
`27`	`27`	`public float? Similarity { get; set; }`
`28`	`28`
`29`	`29`	`public int? Width { get; set; }`
`@@ -42,7 +42,7 @@ internal YandexResult(int width, int height, string url)`
`42`	`42`	`Url = url;`
`43`	`43`	`Caption = null;`
`44`	`44`	`Similarity = null;`
`45`		`- Filter = null;`
	`45`	`+ Filter = false;`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`public override string ToString()`
Original file line number	Diff line number	Diff line change
`@@ -4,14 +4,14 @@`
`4`	`4`	`namespace SmartImage.Engines.SauceNao`
`5`	`5`	`{`
`6`	`6`	`[DataContract]`
`7`		`- public class SauceNaoResponse`
	`7`	`+ public class SauceNaoDataResponse`
`8`	`8`	`{`
`9`	`9`	`//ignore`
`10`	`10`	`//[DeserializeAs(Name = "header")]`
`11`	`11`	`//public object Header { get; set; }`
`12`	`12`
`13`	`13`	`[DataMember(Name = "results")]`
`14`		`- public SauceNaoResult[] Results { get; set; }`
	`14`	`+ public SauceNaoDataResult[] Results { get; set; }`
`15`	`15`
`16`	`16`	`public override string ToString()`
`17`	`17`	`{`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@`
`4`	`4`	`namespace SmartImage.Engines.SauceNao`
`5`	`5`	`{`
`6`	`6`	`[DataContract]`
`7`		`- public class SauceNaoResult`
	`7`	`+ public class SauceNaoDataResult`
`8`	`8`	`{`
`9`	`9`	`/// <summary>`
`10`	`10`	`/// The url(s) where the source is from. Multiple will be returned if the exact same image is found in multiple places`