Merge pull request #14 from zetroot/feature/retry-on-failure

Added a retry queue for failed sites #12
+semver: feature

Author: zetroot

src/NTorSpectator.Observer/Services/Spectator.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.Options;
 using NTorSpectator.Observer.TorIntegration;
 using NTorSpectator.Services;
+using NTorSpectator.Services.Models;
 
 namespace NTorSpectator.Observer.Services;
 
@@ -45,20 +46,31 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     private async Task Watch()
     {
         var sites = await _sitesCatalogue.GetAllSites();
-        foreach (var site in sites)
+        var siteQueue = new Queue<QueuedSite>(sites.Select(x => new QueuedSite(x, 0)));
+        
+        while(siteQueue.TryDequeue(out var queuedSite))
         {
-            using (_logger.BeginScope(new Dictionary<string, object> { { "HiddenService", site.SiteUri } }))
+            using var _ = _logger.BeginScope(new Dictionary<string, object> { { "HiddenService", queuedSite.Site.SiteUri } });
+            try
             {
-                try
-                {
-                    var observations = await ObserveSite(site.SiteUri);
-                    await _siteObserver.AddNewObservation(site.SiteUri, observations.IsOk);
-                    _logger.LogInformation("Site observed");
-                }
-                catch (Exception e)
+                var observations = await ObserveSite(queuedSite.Site.SiteUri);
+                if (!observations.IsOk)
                 {
-                    _logger.LogError(e, "Observation for site failed");
+                    _logger.LogDebug("Site observed as not available");
+                    var siteObservationsCount = queuedSite.ObservationsCount;
+                    if (siteObservationsCount < 3)
+                    {
+                        _logger.LogDebug("Site has been observed {Count} times, returning it to queue", siteObservationsCount);
+                        siteQueue.Enqueue(queuedSite with{ObservationsCount = siteObservationsCount + 1});
+                        continue;
+                    }
                 }
+                await _siteObserver.AddNewObservation(queuedSite.Site.SiteUri, observations.IsOk);
+                _logger.LogInformation("Site observed");
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Observation for site failed");
             }
         }
     }
@@ -78,6 +90,8 @@ private async Task<TorWatchResults> ObserveSite(string site)
         var negative = torReply.Count(x => x.Action == HsDescAction.Failed);
         return new(site, positive, negative);
     }
+
+    private record QueuedSite(Site Site, int ObservationsCount);
 }
 
 public record TorWatchResults(string Site, int Positive, int Negative)