Better URI validation and error handling

S1mplector · S1mplector · commit df7e33a55d6d · 2025-12-04T20:34:30.000+01:00
diff --git a/Koware.Cli/Program.cs b/Koware.Cli/Program.cs
@@ -96,7 +96,7 @@ static IHost BuildHost(string[] args)
     builder.Services.AddSingleton<IMangaListStore, SqliteMangaListStore>();
     builder.Logging.SetMinimumLevel(LogLevel.Warning);
     builder.Logging.AddFilter("koware", LogLevel.Information);
-    builder.Logging.AddFilter("Koware.Infrastructure.Scraping.AllAnimeCatalog", LogLevel.Error);
+    builder.Logging.AddFilter("Koware.Infrastructure.Scraping.AllAnimeCatalog", LogLevel.Debug);
 
     return builder.Build();
 }
diff --git a/Koware.Cli/appsettings.json b/Koware.Cli/appsettings.json
@@ -5,6 +5,7 @@
     "PreferredMatchIndex": null
   },
   "AllAnime": {
+    "Enabled": true,
     "BaseHost": "allanime.day",
     "ApiBase": "https://api.allanime.day",
     "Referer": "https://allmanga.to",
@@ -13,6 +14,7 @@
     "SearchLimit": 20
   },
   "AllManga": {
+    "Enabled": true,
     "BaseHost": "allmanga.to",
     "ApiBase": "https://api.allanime.day",
     "Referer": "https://allmanga.to",
diff --git a/Koware.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs b/Koware.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs
@@ -6,7 +6,6 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
-using Microsoft.Extensions.Logging;
 using System.Net;
 
 namespace Koware.Infrastructure.DependencyInjection;
@@ -18,29 +17,25 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi
         if (configuration is not null)
         {
             services.Configure<AllAnimeOptions>(configuration.GetSection("AllAnime"));
-            services.Configure<GogoAnimeOptions>(configuration.GetSection("GogoAnime"));
             services.Configure<AllMangaOptions>(configuration.GetSection("AllManga"));
-            services.Configure<ProviderToggleOptions>(configuration.GetSection("Providers"));
         }
         else
         {
             services.Configure<AllAnimeOptions>(_ => { });
-            services.Configure<GogoAnimeOptions>(_ => { });
             services.Configure<AllMangaOptions>(_ => { });
-            services.Configure<ProviderToggleOptions>(_ => { });
         }
 
         services.AddHttpClient<AllAnimeCatalog>((sp, client) =>
         {
             var options = sp.GetRequiredService<IOptions<AllAnimeOptions>>().Value;
             // Only configure if source is properly set up (user must provide config)
-            if (!string.IsNullOrWhiteSpace(options.ApiBase))
+            if (!string.IsNullOrWhiteSpace(options.ApiBase) && Uri.TryCreate(options.ApiBase.Trim(), UriKind.Absolute, out var apiBaseUri))
             {
-                client.BaseAddress = new Uri(options.ApiBase);
+                client.BaseAddress = apiBaseUri;
             }
-            if (!string.IsNullOrWhiteSpace(options.Referer))
+            if (!string.IsNullOrWhiteSpace(options.Referer) && Uri.TryCreate(options.Referer.Trim(), UriKind.Absolute, out var refererUri))
             {
-                client.DefaultRequestHeaders.Referrer = new Uri(options.Referer);
+                client.DefaultRequestHeaders.Referrer = refererUri;
             }
             client.DefaultRequestHeaders.UserAgent.ParseAdd(options.UserAgent);
             client.DefaultRequestHeaders.Accept.ParseAdd("application/json, */*");
@@ -52,29 +47,17 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi
             AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate
         });
 
-        services.AddHttpClient<GogoAnimeCatalog>((sp, client) =>
-        {
-            var options = sp.GetRequiredService<IOptions<GogoAnimeOptions>>().Value;
-            // Only configure if source is properly set up (user must provide config)
-            if (!string.IsNullOrWhiteSpace(options.ApiBase))
-            {
-                client.BaseAddress = new Uri(options.ApiBase);
-            }
-            client.DefaultRequestHeaders.UserAgent.ParseAdd(options.UserAgent);
-            client.DefaultRequestHeaders.Accept.ParseAdd("application/json, */*");
-        });
-
         services.AddHttpClient<AllMangaCatalog>((sp, client) =>
         {
             var options = sp.GetRequiredService<IOptions<AllMangaOptions>>().Value;
             // Only configure if source is properly set up (user must provide config)
-            if (!string.IsNullOrWhiteSpace(options.ApiBase))
+            if (!string.IsNullOrWhiteSpace(options.ApiBase) && Uri.TryCreate(options.ApiBase.Trim(), UriKind.Absolute, out var apiBaseUri))
             {
-                client.BaseAddress = new Uri(options.ApiBase);
+                client.BaseAddress = apiBaseUri;
             }
-            if (!string.IsNullOrWhiteSpace(options.Referer))
+            if (!string.IsNullOrWhiteSpace(options.Referer) && Uri.TryCreate(options.Referer.Trim(), UriKind.Absolute, out var refererUri))
             {
-                client.DefaultRequestHeaders.Referrer = new Uri(options.Referer);
+                client.DefaultRequestHeaders.Referrer = refererUri;
             }
             client.DefaultRequestHeaders.UserAgent.ParseAdd(options.UserAgent);
             client.DefaultRequestHeaders.Accept.ParseAdd("application/json, */*");
@@ -87,17 +70,9 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi
         });
 
         services.AddSingleton<AllAnimeCatalog>();
-        services.AddSingleton<GogoAnimeCatalog>();
         services.AddSingleton<AllMangaCatalog>();
+        services.AddSingleton<IAnimeCatalog>(sp => sp.GetRequiredService<AllAnimeCatalog>());
         services.AddSingleton<IMangaCatalog>(sp => sp.GetRequiredService<AllMangaCatalog>());
-        services.AddSingleton<IAnimeCatalog>(sp =>
-        {
-            var primary = sp.GetRequiredService<AllAnimeCatalog>();
-            var secondary = sp.GetRequiredService<GogoAnimeCatalog>();
-            var toggles = sp.GetRequiredService<IOptions<ProviderToggleOptions>>();
-            var logger = sp.GetRequiredService<ILogger<MultiSourceAnimeCatalog>>();
-            return new MultiSourceAnimeCatalog(primary, secondary, toggles, logger);
-        });
         return services;
     }
 }
diff --git a/Koware.Infrastructure/Scraping/AllAnimeCatalog.cs b/Koware.Infrastructure/Scraping/AllAnimeCatalog.cs
@@ -41,58 +41,77 @@ public AllAnimeCatalog(HttpClient httpClient, IOptions<AllAnimeOptions> options,
 
     public async Task<IReadOnlyCollection<Anime>> SearchAsync(string query, CancellationToken cancellationToken = default)
     {
-        if (!_options.IsConfigured)
+        try
         {
-            _logger.LogWarning("AllAnime source not configured. Add configuration to ~/.config/koware/appsettings.user.json");
-            return Array.Empty<Anime>();
-        }
+            _logger.LogDebug("SearchAsync called. IsConfigured={IsConfigured}, ApiBase='{ApiBase}', BaseHost='{BaseHost}'",
+                _options.IsConfigured, _options.ApiBase ?? "(null)", _options.BaseHost ?? "(null)");
 
-        // Note: Removed translationType from search to show all anime regardless of translation availability.
-        // Translation type is still used when fetching episodes and streams.
-        var gql = "query( $search: SearchInput $limit: Int $page: Int $countryOrigin: VaildCountryOriginEnumType ) { shows( search: $search limit: $limit page: $page countryOrigin: $countryOrigin ) { edges { _id name thumbnail description availableEpisodes __typename } }}";
-        var variables = new
-        {
-            search = new { allowAdult = false, allowUnknown = false, query },
-            limit = _options.SearchLimit,
-            page = 1,
-            countryOrigin = "ALL"
-        };
+            if (!_options.IsConfigured)
+            {
+                _logger.LogWarning("AllAnime source not configured. Add configuration to ~/.config/koware/appsettings.user.json");
+                return Array.Empty<Anime>();
+            }
 
-        var uri = BuildApiUri(gql, variables);
-        using var response = await SendWithRetryAsync(uri, cancellationToken);
-        response.EnsureSuccessStatusCode();
+            // Note: Removed translationType from search to show all anime regardless of translation availability.
+            // Translation type is still used when fetching episodes and streams.
+            var gql = "query( $search: SearchInput $limit: Int $page: Int $countryOrigin: VaildCountryOriginEnumType ) { shows( search: $search limit: $limit page: $page countryOrigin: $countryOrigin ) { edges { _id name thumbnail description availableEpisodes __typename } }}";
+            var variables = new
+            {
+                search = new { allowAdult = false, allowUnknown = false, query },
+                limit = _options.SearchLimit,
+                page = 1,
+                countryOrigin = "ALL"
+            };
+
+            var uri = BuildApiUri(gql, variables);
+            using var response = await SendWithRetryAsync(uri, cancellationToken);
+            response.EnsureSuccessStatusCode();
 
-        using var json = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken);
-        var edges = json.RootElement
-            .GetProperty("data")
-            .GetProperty("shows")
-            .GetProperty("edges");
+            using var json = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken);
+            var edges = json.RootElement
+                .GetProperty("data")
+                .GetProperty("shows")
+                .GetProperty("edges");
 
-        var results = new List<Anime>();
-        foreach (var edge in edges.EnumerateArray())
-        {
-            var id = edge.GetProperty("_id").GetString()!;
-            var title = edge.GetProperty("name").GetString() ?? id;
-            var synopsis = edge.TryGetProperty("description", out var desc) ? desc.GetString() : null;
-            Uri? coverImage = null;
-            if (edge.TryGetProperty("thumbnail", out var thumb) && thumb.ValueKind == JsonValueKind.String)
+            var results = new List<Anime>();
+            foreach (var edge in edges.EnumerateArray())
             {
-                var thumbUrl = thumb.GetString();
-                if (!string.IsNullOrWhiteSpace(thumbUrl))
+                var id = edge.GetProperty("_id").GetString()!;
+                var title = edge.GetProperty("name").GetString() ?? id;
+                var synopsis = edge.TryGetProperty("description", out var desc) ? desc.GetString() : null;
+                Uri? coverImage = null;
+                if (edge.TryGetProperty("thumbnail", out var thumb) && thumb.ValueKind == JsonValueKind.String)
                 {
-                    coverImage = new Uri(thumbUrl);
+                    var thumbUrl = thumb.GetString();
+                    if (!string.IsNullOrWhiteSpace(thumbUrl))
+                    {
+                        var absoluteThumb = EnsureAbsolute(thumbUrl);
+                        if (Uri.TryCreate(absoluteThumb, UriKind.Absolute, out var parsedThumb))
+                        {
+                            coverImage = parsedThumb;
+                        }
+                        else
+                        {
+                            _logger.LogDebug("Skipping invalid thumbnail '{Thumb}' for anime {AnimeId}", thumbUrl, id);
+                        }
+                    }
                 }
+                results.Add(new Anime(
+                    new AnimeId(id),
+                    title,
+                    synopsis: synopsis,
+                    coverImage: coverImage,
+                    detailPage: BuildDetailUri(id),
+                    episodes: Array.Empty<Episode>()));
             }
-            results.Add(new Anime(
-                new AnimeId(id),
-                title,
-                synopsis: synopsis,
-                coverImage: coverImage,
-                detailPage: BuildDetailUri(id),
-                episodes: Array.Empty<Episode>()));
-        }
 
-        return results;
+            return results;
+        }
+        catch (UriFormatException ex)
+        {
+            _logger.LogError(ex, "Invalid URI during AllAnime search. ApiBase={ApiBase}, BaseHost={BaseHost}, Referer={Referer}", _options.ApiBase, _options.BaseHost, _options.Referer);
+            return Array.Empty<Anime>();
+        }
     }
 
     public async Task<IReadOnlyCollection<Episode>> GetEpisodesAsync(Anime anime, CancellationToken cancellationToken = default)
@@ -199,7 +218,7 @@ public async Task<IReadOnlyCollection<StreamLink>> GetStreamsAsync(Episode episo
     private async Task ResolveSourceAsync(ProviderSource source, ConcurrentBag<StreamLink> collector, ConcurrentBag<string> attempts, CancellationToken cancellationToken)
     {
         using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-        timeoutCts.CancelAfter(TimeSpan.FromSeconds(5));
+        timeoutCts.CancelAfter(TimeSpan.FromSeconds(10));
         var host = "unknown";
 
         try
@@ -210,7 +229,16 @@ private async Task ResolveSourceAsync(ProviderSource source, ConcurrentBag<Strea
 
             using var response = await SendWithRetryAsync(new Uri(absoluteUrl), timeoutCts.Token);
             response.EnsureSuccessStatusCode();
-            var payload = await response.Content.ReadAsStringAsync(timeoutCts.Token);
+            const int maxBytes = 10 * 1024 * 1024; // 10 MB safety cap
+            var length = response.Content.Headers.ContentLength;
+            if (length.HasValue && length.Value > maxBytes)
+            {
+                attempts.Add($"{source.Name}@{host}: payload-too-large");
+                _logger.LogDebug("Skipping source {Source} because payload length {Length} exceeds cap {Cap}", source.Name, length, maxBytes);
+                return;
+            }
+
+            var payload = await ReadContentWithLimitAsync(response.Content, maxBytes, timeoutCts.Token);
 
             var links = await ExtractLinksAsync(payload, absoluteUrl, source.Name, timeoutCts.Token);
             foreach (var link in links)
@@ -237,6 +265,30 @@ private async Task ResolveSourceAsync(ProviderSource source, ConcurrentBag<Strea
         }
     }
 
+    private static async Task<string> ReadContentWithLimitAsync(HttpContent content, int maxBytes, CancellationToken cancellationToken)
+    {
+        await using var stream = await content.ReadAsStreamAsync(cancellationToken);
+        await using var buffer = new MemoryStream();
+        var temp = new byte[8192];
+        try
+        {
+            int read;
+            while ((read = await stream.ReadAsync(temp.AsMemory(0, temp.Length), cancellationToken)) > 0)
+            {
+                if (buffer.Length + read > maxBytes)
+                {
+                    throw new HttpRequestException($"Content exceeded limit of {maxBytes} bytes");
+                }
+                buffer.Write(temp, 0, read);
+            }
+
+            return Encoding.UTF8.GetString(buffer.ToArray());
+        }
+        finally
+        {
+        }
+    }
+
     private async Task<IReadOnlyCollection<StreamLink>> ExtractLinksAsync(string payload, string sourceUrl, string provider, CancellationToken cancellationToken)
     {
         var links = new List<StreamLink>();
@@ -604,11 +656,32 @@ private static (string showId, int episodeNumber) ParseEpisodeId(Episode episode
 
     private Uri BuildApiUri(string gql, object variables)
     {
+        if (string.IsNullOrWhiteSpace(_options.ApiBase))
+        {
+            throw new InvalidOperationException("AllAnime ApiBase is not configured. Check your appsettings.json or appsettings.user.json.");
+        }
+        
+        var apiBase = _options.ApiBase.Trim();
+        if (!Uri.TryCreate(apiBase, UriKind.Absolute, out var baseUri))
+        {
+            throw new InvalidOperationException($"AllAnime ApiBase '{apiBase}' is not a valid URI. It should be like 'https://api.example.com'.");
+        }
+        
+        var builder = new UriBuilder(baseUri)
+        {
+            Path = $"{baseUri.AbsolutePath.TrimEnd('/')}/api"
+        };
+
         var query = $"query={Uri.EscapeDataString(gql)}&variables={Uri.EscapeDataString(JsonSerializer.Serialize(variables, _serializerOptions))}";
-        return new Uri($"{_options.ApiBase.TrimEnd('/')}/api?{query}");
+        builder.Query = query;
+        return builder.Uri;
     }
 
-    private Uri BuildDetailUri(string id) => new($"https://{_options.BaseHost}/anime/{id}");
+    private Uri BuildDetailUri(string id)
+    {
+        var host = ResolveBaseHost();
+        return new UriBuilder("https", host, -1, $"anime/{id}").Uri;
+    }
 
     private static int ParseQualityScore(string quality)
     {
@@ -627,15 +700,49 @@ private string EnsureAbsolute(string path)
             return path;
         }
 
-        var baseUrl = $"https://{_options.BaseHost}";
-        return path.StartsWith('/') ? $"{baseUrl}{path}" : $"{baseUrl}/{path}";
+        var host = ResolveBaseHost();
+        var builder = new UriBuilder("https", host)
+        {
+            Path = path.StartsWith('/') ? path : $"/{path}"
+        };
+        return builder.Uri.ToString();
+    }
+
+    private string ResolveBaseHost()
+    {
+        if (!string.IsNullOrWhiteSpace(_options.BaseHost))
+        {
+            var hostText = _options.BaseHost.Trim();
+            if (hostText.Contains("://", StringComparison.Ordinal))
+            {
+                if (Uri.TryCreate(hostText, UriKind.Absolute, out var parsed))
+                {
+                    return parsed.Host;
+                }
+            }
+            hostText = hostText.TrimEnd('/');
+            if (!string.IsNullOrWhiteSpace(hostText))
+            {
+                return hostText;
+            }
+        }
+
+        if (!string.IsNullOrWhiteSpace(_options.ApiBase) && Uri.TryCreate(_options.ApiBase, UriKind.Absolute, out var api))
+        {
+            return api.Host;
+        }
+
+        throw new InvalidOperationException("AllAnime BaseHost is not configured and ApiBase is invalid.");
     }
 
     private HttpRequestMessage BuildRequest(Uri uri)
     {
         var request = new HttpRequestMessage(HttpMethod.Get, uri);
-        request.Headers.Referrer = new Uri(_options.Referer);
-        request.Headers.TryAddWithoutValidation("Origin", _options.Referer.TrimEnd('/'));
+        if (Uri.TryCreate(_options.Referer, UriKind.Absolute, out var refUri))
+        {
+            request.Headers.Referrer = refUri;
+            request.Headers.TryAddWithoutValidation("Origin", refUri.GetLeftPart(UriPartial.Authority));
+        }
         request.Headers.UserAgent.ParseAdd(_options.UserAgent);
         request.Headers.Accept.ParseAdd("application/json, */*");
         request.Headers.AcceptLanguage.ParseAdd("en-US,en;q=0.9");

Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ static IHost BuildHost(string[] args)`
`96`	`96`	`builder.Services.AddSingleton<IMangaListStore, SqliteMangaListStore>();`
`97`	`97`	`builder.Logging.SetMinimumLevel(LogLevel.Warning);`
`98`	`98`	`builder.Logging.AddFilter("koware", LogLevel.Information);`
`99`		`- builder.Logging.AddFilter("Koware.Infrastructure.Scraping.AllAnimeCatalog", LogLevel.Error);`
	`99`	`+ builder.Logging.AddFilter("Koware.Infrastructure.Scraping.AllAnimeCatalog", LogLevel.Debug);`
`100`	`100`
`101`	`101`	`return builder.Build();`
`102`	`102`	`}`