Skip to content

Commit 3c131af

Browse files
DecimationDecimation
committed
Optimized image scanning algorithm
1 parent 4cd1246 commit 3c131af

File tree

2 files changed

+109
-60
lines changed

2 files changed

+109
-60
lines changed

‎SmartImage.Lib/Utilities/ImageHelper.cs‎

Lines changed: 96 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -156,17 +156,16 @@ public static Dictionary<string, string> UtilitiesMap
156156
/// <param name="url">Url to search</param>
157157
/// <param name="directType">Which criterion to use to determine whether a URI is a direct image </param>
158158
/// <param name="count">Number of direct images to return</param>
159-
/// <param name="fragmentSize">Size of the fragments which a respective task operates on</param>
160159
/// <param name="pingTimeSec"></param>
161160
/// <param name="readImage">Whether to read image metadata</param>
162161
/// <param name="imageFilter">Filter criteria for images (applicable iff <paramref name="readImage"/> is <c>true</c>)</param>
163162
public static List<DirectImage> FindDirectImages(string url, DirectImageType directType = DirectImageType.Regex,
164-
int count = 5, int fragmentSize = 10, double pingTimeSec = 1,
163+
int count = 5, double pingTimeSec = 1,
165164
bool readImage = true, Predicate<Image> imageFilter = null)
166165
{
167166
var directImages = new List<DirectImage>();
168167

169-
string gallerydl = UtilitiesMap[GALLERY_DL_EXE];
168+
/*string gallerydl = UtilitiesMap[GALLERY_DL_EXE];
170169
171170
if (gallerydl != null) {
172171
@@ -204,11 +203,21 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
204203
while (!standardError.EndOfStream) {
205204
var line = standardError.ReadLine();
206205
Debug.WriteLine($"{GALLERY_DL_EXE}: {line}", C_ERROR);
206+
207+
if (line!=null) {
208+
goto manual;
209+
}
207210
}
208211
212+
213+
209214
return directImages.OrderByDescending(x => x.Image.Width * x.Image.Height).ToList();
210215
}
211216
217+
manual:*/
218+
219+
var sw = Stopwatch.StartNew();
220+
212221
imageFilter ??= (x) => true;
213222

214223
var pingTime = TimeSpan.FromSeconds(pingTimeSec);
@@ -238,81 +247,117 @@ public static List<DirectImage> FindDirectImages(string url, DirectImageType dir
238247

239248
flat = flat.Distinct().ToList();
240249

241-
var fragments = flat.Chunk(fragmentSize).ToArray();
250+
//var fragments = flat.Chunk(fragmentSize).ToArray();
242251

243-
var tasks = new List<Task>();
252+
//var tasks = new List<Task>();
244253

245254
//count = Math.Clamp(count, count, flat.Count);
255+
var act = new List<Action>();
246256

247-
for (int i = 0; i < fragments.Length; i++) {
257+
for (int i = 0; i < flat.Count; i++) {
248258

249259
int iCopy = i;
250260

251261
var imagesCopy = directImages;
252262

253-
tasks.Add(Task.Factory.StartNew(() =>
263+
void Function()
254264
{
265+
string currentUrl = flat[iCopy];
255266

256-
foreach (string currentUrl in fragments[iCopy]) {
257-
258-
if (directImages.Count >= count) {
259-
return;
260-
}
267+
if (directImages.Count >= count) {
268+
return;
269+
}
261270

262-
if (!Network.IsUri(currentUrl, out var uri))
263-
continue;
271+
if (!Network.IsUri(currentUrl, out var uri))
272+
return;
264273

265-
if (!Network.IsAlive(uri, (long)pingTime.TotalMilliseconds))
266-
continue;
274+
if (!Network.IsAlive(uri, (long) pingTime.TotalMilliseconds))
275+
return;
267276

268-
if (!IsDirect(currentUrl, directType))
269-
continue;
277+
if (!IsDirect(currentUrl, directType))
278+
return;
270279

271-
var di = new DirectImage
272-
{
273-
Direct = new Uri(currentUrl)
274-
};
280+
var di = new DirectImage {Direct = new Uri(currentUrl)};
275281

276282

277-
bool isValid = !readImage;
283+
bool isValid = !readImage;
278284

279-
if (readImage) {
280-
var stream = WebUtilities.GetStream(currentUrl);
285+
if (readImage) {
286+
var stream = WebUtilities.GetStream(currentUrl);
281287

282-
if (stream.CanRead) {
288+
if (stream.CanRead) {
283289

284-
try {
285-
var img = Image.FromStream(stream);
286-
//isValid = true;
290+
try {
291+
var img = Image.FromStream(stream);
292+
//isValid = true;
287293

288-
//Debug.WriteLine($"{img.Width} {img.Height}");
289-
isValid = imageFilter(img);
294+
//Debug.WriteLine($"{img.Width} {img.Height}");
295+
isValid = imageFilter(img);
290296

291-
if (isValid) {
292-
di.Image = img;
293-
}
294-
}
295-
catch (Exception) {
296-
isValid = false;
297+
if (isValid) {
298+
di.Image = img;
297299
}
298300
}
301+
catch (Exception) {
302+
isValid = false;
303+
}
299304
}
305+
}
300306

301-
if (directImages.Count >= count) {
302-
return;
303-
}
307+
if (directImages.Count >= count) {
308+
return;
309+
}
304310

305-
if (isValid) {
306-
imagesCopy.Add(di);
307-
//Debug.WriteLine($">>> {currentUrl}");
311+
if (isValid) {
312+
imagesCopy.Add(di);
313+
//Debug.WriteLine($">>> {currentUrl}");
308314

309-
}
310315
}
311-
}, cts.Token));
316+
}
312317

318+
//tasks.Add(Task.Factory.StartNew(function, cts.Token));
319+
act.Add(Function);
313320
}
314321

315-
Task.WaitAll(tasks.ToArray());
322+
323+
/*
324+
* Tasks
325+
* 1 5.19
326+
* 2 4.68
327+
* 3 4.54
328+
* 4 4.42
329+
*
330+
* Parallel
331+
* 1 4.59
332+
* 2 4.37
333+
* 3 4.28
334+
* 4 4.34
335+
* 5 4.38
336+
* 6 4.55
337+
* 7 4.36
338+
* 8 4.45
339+
*
340+
* 9 3.84
341+
* 10 3.56
342+
* 11 3.45
343+
* 12 3.52
344+
* 13 3.63
345+
* 14 3.52
346+
*/
347+
348+
//Task.WaitAll(tasks.ToArray());
349+
350+
351+
Parallel.Invoke(new ParallelOptions()
352+
{
353+
MaxDegreeOfParallelism = Int32.MaxValue,
354+
TaskScheduler = TaskScheduler.Default,
355+
CancellationToken = cts.Token
356+
}, act.ToArray());
357+
358+
sw.Stop();
359+
360+
Trace.WriteLine($"{sw.Elapsed.TotalSeconds}");
316361

317362

318363
return directImages.OrderByDescending(x => x.Image.Width * x.Image.Height).ToList();
@@ -335,6 +380,11 @@ public struct DirectImage
335380
public Uri Direct { get; internal set; }
336381

337382
public Image Image { get; internal set; }
383+
384+
public override string ToString()
385+
{
386+
return $"{Direct} {Image.Width}x{Image.Height}";
387+
}
338388
}
339389

340390
public enum DirectImageType

‎Test/Program.cs‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public static async Task Main(string[] args)
4646
//Console.InputEncoding = Encoding.Unicode;
4747

4848

49-
var q = new ImageQuery(@"C:\Users\Deci\Pictures\Test Images\Test3.png");
49+
//var q = new ImageQuery(@"C:\Users\Deci\Pictures\Test Images\Test3.png");
5050
//var q = new ImageQuery(@"C:\Users\Deci\Pictures\Test Images\Test4.png");
5151

5252

@@ -100,11 +100,11 @@ public static async Task Main(string[] args)
100100
// Console.WriteLine(result);
101101
//}
102102

103-
var i = new TraceMoeEngine() { };
104-
var i2 = i.GetResultAsync(q);
103+
//var i = new TraceMoeEngine() { };
104+
//var i2 = i.GetResultAsync(q);
105105

106-
await i2;
107-
Console.WriteLine(i2.Result);
106+
//await i2;
107+
//Console.WriteLine(i2.Result);
108108

109109
//Console.WriteLine(r2);
110110

@@ -167,15 +167,14 @@ public static async Task Main(string[] args)
167167

168168
//Console.WriteLine(">> {0}", r2x);
169169

170-
// var t = ImageHelper.FindDirectImages("https://www.zerochan.net/2750747", out var i);
171-
//
172-
// foreach (var s in t) {
173-
// Console.WriteLine(s);
174-
// }
175-
//
176-
// foreach (var image in i) {
177-
// Console.WriteLine(image);
178-
// }
170+
var t = ImageHelper.FindDirectImages("https://www.zerochan.net/2750747");
171+
172+
foreach (var s in t)
173+
{
174+
Console.WriteLine(s);
175+
}
176+
177+
179178

180179

181180
}

0 commit comments

Comments
 (0)