Merge pull request #4 from skttl/verify-urls

adds config and method to verify that a url exists (returns success s…

Author: skttl

docs/configuration.md

@@ -4,6 +4,8 @@ The `RemoteImageProviderOptions` class provides the following configuration opti
 
 - `Settings`: A list of the different allowed sources for images. 
 
+- `FallbackMaxAge`: Specifies a fallback max age for the image being loaded. Used if the server does not return a cache-control header. By default, it is set to `0.01:00:00` (1 hour).
+- 
 Each setting (`RemoteImageProviderSetting`) provides the following configuration options:
 
 - `Prefix`: Specified in the constructor, and defines the local path to prefix all remote image requests with. For example, setting this to `/remote` allows requests like `/remote/https://test.com/test.png` to pass through this provider.
@@ -26,6 +28,8 @@ Each setting (`RemoteImageProviderSetting`) provides the following configuration
 
 - `AdditionalOptions`: Allows specifying additional `RemoteImageProviderOptions` instances. This can be useful when you have multiple configurations with different prefixes or other settings.
 
+- `VerifyUrl`: Boolean value. If set to true, the URL will be verified before downloading the image. This can be useful to prevent downloading and processing images returning 404 or other error codes. By default, it is set to `truee`.`
+
 Please note that these options provide customization and control over how remote images are loaded and processed. You can adjust these options according to your specific requirements.
 
 Don't forget to configure these options in your application's services configuration as shown in the Usage section of this README.

src/ImageSharpCommunity.Providers.Remote/Configuration/RemoteImageProviderOptions.cs

@@ -1,8 +1,13 @@
-namespace ImageSharpCommunity.Providers.Remote.Configuration;
+namespace ImageSharpCommunity.Providers.Remote.Configuration;
 public class RemoteImageProviderOptions
 {
     /// <summary>
     /// A list of settings for remote image providers. Here you define your url prefixes, and which domains are allowed to fetch images from.
     /// </summary>
     public List<RemoteImageProviderSetting> Settings { get; set; } = new List<RemoteImageProviderSetting>();
-}
+
+    /// <summary>
+    /// Fallback max age for the image. If the server does not return a cache-control header, this value is used.
+    /// </summary>
+    public TimeSpan FallbackMaxAge { get; set; } = TimeSpan.FromHours(1);
+}

src/ImageSharpCommunity.Providers.Remote/Configuration/RemoteImageProviderSetting.cs

@@ -57,5 +57,10 @@ public RemoteImageProviderSetting(string prefix)
     /// </summary>
     public bool AllowAllDomains { get; set; }
 
+    /// <summary>
+    /// Verify that the input url returns a succesful status code.
+    /// </summary>
+    public bool VerifyUrl { get; set; } = true;
+
     internal string ClientDictionaryKey => $"{HttpClientName}_{UserAgent}_{Timeout}_{MaxBytes}";
 }

src/ImageSharpCommunity.Providers.Remote/RemoteImageProvider.cs

@@ -1,5 +1,6 @@
 using ImageSharpCommunity.Providers.Remote.Configuration;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using SixLabors.ImageSharp.Web.Providers;
@@ -13,13 +14,15 @@ public class RemoteImageProvider : IImageProvider
     private readonly RemoteImageProviderOptions _options;
     private readonly ILogger<RemoteImageProvider> _logger;
     private readonly ILogger<RemoteImageResolver> _resolverLogger;
+    private readonly IMemoryCache _cache;
 
-    public RemoteImageProvider(IHttpClientFactory clientFactory, IOptions<RemoteImageProviderOptions> options, ILogger<RemoteImageProvider> logger, ILogger<RemoteImageResolver> resolverLogger)
+    public RemoteImageProvider(IHttpClientFactory clientFactory, IOptions<RemoteImageProviderOptions> options, ILogger<RemoteImageProvider> logger, ILogger<RemoteImageResolver> resolverLogger, IMemoryCache cache)
     {
         _clientFactory = clientFactory;
         _options = options.Value;
         _logger = logger;
         _resolverLogger = resolverLogger;
+        _cache = cache;
     }
 
     public ProcessingBehavior ProcessingBehavior => ProcessingBehavior.All;
@@ -37,7 +40,8 @@ public bool IsValidRequest(HttpContext context)
             context.Request.Path.GetMatchingRemoteImageProviderSetting(_options) is RemoteImageProviderSetting setting
             && context.Request.Path.GetSourceUrlForRemoteImageProviderUrl(_options) is string url
             && Uri.TryCreate(url, UriKind.Absolute, out Uri? uri) && uri != null
-            && uri.IsValidForSetting(setting);
+            && uri.IsValidForSetting(setting)
+            && UrlReturnsSuccess(setting, uri);
     }
 
     public Task<IImageResolver?> GetAsync(HttpContext context)
@@ -53,11 +57,37 @@ public bool IsValidRequest(HttpContext context)
         else
         {
             _logger.LogDebug("Found matching remote image provider setting for path: {path}", context.Request.Path);
-            return Task.FromResult((IImageResolver?)new RemoteImageResolver(_clientFactory, url, options, _resolverLogger));
+            return Task.FromResult((IImageResolver?)new RemoteImageResolver(_clientFactory, url, options, _resolverLogger, _options));
         }
     }
     private bool IsMatch(HttpContext context)
     {
         return context.Request.Path.GetMatchingRemoteImageProviderSetting(_options) != null;
     }
+
+    private bool UrlReturnsSuccess(RemoteImageProviderSetting setting, Uri uri)
+    {
+        if (setting.VerifyUrl == false)
+        {
+            _logger.LogDebug("Skipping verification of URL {Url} as VerifyUrl is set to false", uri);
+            return true;
+        }
+
+        if (_cache.TryGetValue(nameof(RemoteImageProvider) + uri, out bool cachedResult))
+        {
+            _logger.LogDebug("Using cached result for URL {Url}", uri);
+            return cachedResult;
+        }
+
+        var client = _clientFactory.GetRemoteImageProviderHttpClient(setting);
+        var request = new HttpRequestMessage(HttpMethod.Head, uri);
+        var response = client.SendAsync(request).Result;
+
+        if (response.Headers.CacheControl?.MaxAge is not null)
+        {
+            _cache.Set(nameof(RemoteImageProvider) + uri, response.IsSuccessStatusCode, response.Headers.CacheControl.MaxAge ?? _options.FallbackMaxAge);
+        }
+
+        return response.IsSuccessStatusCode;
+    }
 }

src/ImageSharpCommunity.Providers.Remote/RemoteImageResolver.cs

@@ -9,13 +9,15 @@ public class RemoteImageResolver : IImageResolver
     private readonly string _url;
     private readonly RemoteImageProviderSetting _setting;
     private readonly ILogger<RemoteImageResolver> _logger;
+    private readonly RemoteImageProviderOptions _options;
 
-    public RemoteImageResolver(IHttpClientFactory clientFactory, string url, RemoteImageProviderSetting setting, ILogger<RemoteImageResolver> logger)
+    public RemoteImageResolver(IHttpClientFactory clientFactory, string url, RemoteImageProviderSetting setting, ILogger<RemoteImageResolver> logger, RemoteImageProviderOptions options)
     {
         _clientFactory = clientFactory;
         _url = url;
         _setting = setting;
         _logger = logger;
+        _options = options;
     }
 
     public async Task<ImageMetadata> GetMetaDataAsync()
@@ -38,10 +40,14 @@ public async Task<ImageMetadata> GetMetaDataAsync()
 
         if (response.Headers.CacheControl?.MaxAge is null)
         {
-            _logger.LogDebug("MaxAge header is null from {Url}", _url);
+            _logger.LogDebug("MaxAge header is null from {Url}, falling back to configured FallbackMaxAge {FallbackMaxAge}", _url, _options.FallbackMaxAge);
         }
 
-        return new ImageMetadata(response.Content.Headers.LastModified.GetValueOrDefault().UtcDateTime, (response.Headers.CacheControl?.MaxAge).GetValueOrDefault(), response.Content.Headers.ContentLength.GetValueOrDefault());
+        return new ImageMetadata(
+            response.Content.Headers.LastModified.GetValueOrDefault().UtcDateTime,
+            response.Headers.CacheControl?.MaxAge ?? _options.FallbackMaxAge,
+            response.Content.Headers.ContentLength.GetValueOrDefault()
+        );
     }
 
     public async Task<Stream> OpenReadAsync()

src/Umbraco.Community.ImageSharpRemoteImages/appsettings-schema.umbraco-community-imagesharpremoteimages.json

@@ -45,6 +45,12 @@
           "items": {
             "$ref": "#/definitions/UmbracoCommunityImageSharpRemoteImagesSettingDefinition"
           }
+        },
+        "FallbackMaxAge": {
+          "type": "string",
+          "description": "Fallback max age for the image. If the server does not return a cache-control header, this value is used.",
+          "format": "duration",
+          "default": "0.01:00:00"
         }
       }
     },
@@ -96,6 +102,10 @@
         "AllowAllDomains": {
           "type": "boolean",
           "description": "Allows all domains to be processed."
+        },
+        "VerifyUrl": {
+          "type": "boolean",
+          "description": "Verify that the input url returns a succesful status code."
         }
       },
       "required": [ "Prefix" ]