Adds support for excluding URLs. Closes #109 (#243)

waldekmastykarz · web-flow · commit 948c7d3ab735 · 2023-04-14T17:37:33.000+02:00
diff --git a/msgraph-developer-proxy-abstractions/BaseProxyPlugin.cs b/msgraph-developer-proxy-abstractions/BaseProxyPlugin.cs
@@ -2,18 +2,17 @@
 // Licensed under the MIT License.
 
 using Microsoft.Extensions.Configuration;
-using System.Text.RegularExpressions;
 
 namespace Microsoft.Graph.DeveloperProxy.Abstractions;
 
 public abstract class BaseProxyPlugin: IProxyPlugin {
-  protected ISet<Regex>? _urlsToWatch;
+  protected ISet<UrlToWatch>? _urlsToWatch;
   protected ILogger? _logger;
 
   public virtual string Name => throw new NotImplementedException();
   public virtual void Register(IPluginEvents pluginEvents,
                        IProxyContext context,
-                       ISet<Regex> urlsToWatch,
+                       ISet<UrlToWatch> urlsToWatch,
                        IConfigurationSection? configSection = null) {
     if (pluginEvents is null) {
       throw new ArgumentNullException(nameof(pluginEvents));
diff --git a/msgraph-developer-proxy-abstractions/IProxyPlugin.cs b/msgraph-developer-proxy-abstractions/IProxyPlugin.cs
@@ -2,14 +2,13 @@
 // Licensed under the MIT License.
 
 using Microsoft.Extensions.Configuration;
-using System.Text.RegularExpressions;
 
 namespace Microsoft.Graph.DeveloperProxy.Abstractions;
 
 public interface IProxyPlugin {
     string Name { get; }
     void Register(IPluginEvents pluginEvents,
                   IProxyContext context,
-                  ISet<Regex> urlsToWatch,
+                  ISet<UrlToWatch> urlsToWatch,
                   IConfigurationSection? configSection = null);
 }
diff --git a/msgraph-developer-proxy-abstractions/PluginEvents.cs b/msgraph-developer-proxy-abstractions/PluginEvents.cs
@@ -3,7 +3,6 @@
 
 using System.CommandLine;
 using System.CommandLine.Invocation;
-using System.Text.RegularExpressions;
 using Titanium.Web.Proxy.EventArguments;
 
 namespace Microsoft.Graph.DeveloperProxy.Abstractions;
@@ -18,8 +17,10 @@ internal ProxyHttpEventArgsBase(SessionEventArgs session) =>
 
     public SessionEventArgs Session { get; }
 
-    public bool HasRequestUrlMatch(ISet<Regex> watchedUrls) =>
-        watchedUrls.Any(r => r.IsMatch(Session.HttpClient.Request.RequestUri.AbsoluteUri));
+    public bool HasRequestUrlMatch(ISet<UrlToWatch> watchedUrls) {
+        var match = watchedUrls.FirstOrDefault(r => r.Url.IsMatch(Session.HttpClient.Request.RequestUri.AbsoluteUri));
+        return match is not null && !match.Exclude;
+    }
 }
 
 public class ProxyRequestArgs : ProxyHttpEventArgsBase {
@@ -28,7 +29,7 @@ public ProxyRequestArgs(SessionEventArgs session, ResponseState responseState) :
     }
     public ResponseState ResponseState { get; }
 
-    public bool ShouldExecute(ISet<Regex> watchedUrls) =>
+    public bool ShouldExecute(ISet<UrlToWatch> watchedUrls) =>
         !ResponseState.HasBeenSet
         && HasRequestUrlMatch(watchedUrls);
 }
diff --git a/msgraph-developer-proxy-abstractions/UrlToWatch.cs b/msgraph-developer-proxy-abstractions/UrlToWatch.cs
@@ -0,0 +1,14 @@
+using System.Text.RegularExpressions;
+
+namespace Microsoft.Graph.DeveloperProxy.Abstractions;
+
+public class UrlToWatch {
+    public bool Exclude { get; }
+    public Regex Url { get; }
+
+    public UrlToWatch(Regex url, bool exclude = false)
+    {
+      Exclude = exclude;
+      Url = url;
+    }
+}
diff --git a/msgraph-developer-proxy-plugins/Behavior/RateLimitingPlugin.cs b/msgraph-developer-proxy-plugins/Behavior/RateLimitingPlugin.cs
@@ -123,7 +123,7 @@ private void UpdateProxyResponse(ProxyHttpEventArgsBase e, HttpStatusCode errorS
 
     public override void Register(IPluginEvents pluginEvents,
                          IProxyContext context,
-                         ISet<Regex> urlsToWatch,
+                         ISet<UrlToWatch> urlsToWatch,
                          IConfigurationSection? configSection = null) {
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
diff --git a/msgraph-developer-proxy-plugins/Guidance/GraphBetaSupportGuidancePlugin.cs b/msgraph-developer-proxy-plugins/Guidance/GraphBetaSupportGuidancePlugin.cs
@@ -3,7 +3,6 @@
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Graph.DeveloperProxy.Abstractions;
-using System.Text.RegularExpressions;
 using Titanium.Web.Proxy.Http;
 
 namespace Microsoft.Graph.DeveloperProxy.Plugins.Guidance;
@@ -13,7 +12,7 @@ public class GraphBetaSupportGuidancePlugin : BaseProxyPlugin {
 
     public override void Register(IPluginEvents pluginEvents,
                             IProxyContext context,
-                            ISet<Regex> urlsToWatch,
+                            ISet<UrlToWatch> urlsToWatch,
                             IConfigurationSection? configSection = null) {
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
diff --git a/msgraph-developer-proxy-plugins/Guidance/GraphClientRequestIdGuidancePlugin.cs b/msgraph-developer-proxy-plugins/Guidance/GraphClientRequestIdGuidancePlugin.cs
@@ -3,7 +3,6 @@
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Graph.DeveloperProxy.Abstractions;
-using System.Text.RegularExpressions;
 using Titanium.Web.Proxy.Http;
 
 namespace Microsoft.Graph.DeveloperProxy.Plugins.Guidance;
@@ -14,7 +13,7 @@ public class GraphClientRequestIdGuidancePlugin : BaseProxyPlugin
 
   public override void Register(IPluginEvents pluginEvents,
                           IProxyContext context,
-                          ISet<Regex> urlsToWatch,
+                          ISet<UrlToWatch> urlsToWatch,
                           IConfigurationSection? configSection = null)
   {
     base.Register(pluginEvents, context, urlsToWatch, configSection);
diff --git a/msgraph-developer-proxy-plugins/Guidance/GraphSdkGuidancePlugin.cs b/msgraph-developer-proxy-plugins/Guidance/GraphSdkGuidancePlugin.cs
@@ -3,7 +3,6 @@
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Graph.DeveloperProxy.Abstractions;
-using System.Text.RegularExpressions;
 using Titanium.Web.Proxy.Http;
 
 namespace Microsoft.Graph.DeveloperProxy.Plugins.Guidance;
@@ -13,7 +12,7 @@ public class GraphSdkGuidancePlugin : BaseProxyPlugin {
 
     public override void Register(IPluginEvents pluginEvents,
                             IProxyContext context,
-                            ISet<Regex> urlsToWatch,
+                            ISet<UrlToWatch> urlsToWatch,
                             IConfigurationSection? configSection = null) {
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
diff --git a/msgraph-developer-proxy-plugins/Guidance/GraphSelectGuidancePlugin.cs b/msgraph-developer-proxy-plugins/Guidance/GraphSelectGuidancePlugin.cs
@@ -3,7 +3,6 @@
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Graph.DeveloperProxy.Abstractions;
-using System.Text.RegularExpressions;
 using Titanium.Web.Proxy.Http;
 
 namespace Microsoft.Graph.DeveloperProxy.Plugins.Guidance;
@@ -13,7 +12,7 @@ public class GraphSelectGuidancePlugin : BaseProxyPlugin {
 
     public override void Register(IPluginEvents pluginEvents,
                             IProxyContext context,
-                            ISet<Regex> urlsToWatch,
+                            ISet<UrlToWatch> urlsToWatch,
                             IConfigurationSection? configSection = null) {
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
diff --git a/msgraph-developer-proxy-plugins/Guidance/ODataPagingGuidancePlugin.cs b/msgraph-developer-proxy-plugins/Guidance/ODataPagingGuidancePlugin.cs
@@ -4,7 +4,6 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Graph.DeveloperProxy.Abstractions;
 using System.Text.Json;
-using System.Text.RegularExpressions;
 using System.Xml.Linq;
 
 namespace Microsoft.Graph.DeveloperProxy.Plugins.Guidance;
@@ -16,7 +15,7 @@ public class ODataPagingGuidancePlugin : BaseProxyPlugin
 
   public override void Register(IPluginEvents pluginEvents,
                           IProxyContext context,
-                          ISet<Regex> urlsToWatch,
+                          ISet<UrlToWatch> urlsToWatch,
                           IConfigurationSection? configSection = null)
   {
     base.Register(pluginEvents, context, urlsToWatch, configSection);
diff --git a/msgraph-developer-proxy-plugins/MockResponses/MockResponsePlugin.cs b/msgraph-developer-proxy-plugins/MockResponses/MockResponsePlugin.cs
@@ -41,7 +41,7 @@ public MockResponsePlugin() {
 
     public override void Register(IPluginEvents pluginEvents,
                             IProxyContext context,
-                            ISet<Regex> urlsToWatch,
+                            ISet<UrlToWatch> urlsToWatch,
                             IConfigurationSection? configSection = null) {
         base.Register(pluginEvents, context, urlsToWatch, configSection);
         
@@ -113,7 +113,7 @@ _configuration.Responses is null ||
 
             //turn mock URL with wildcard into a regex and match against the request URL
             var mockResponseUrlRegex = Regex.Escape(mockResponse.Url).Replace("\\*", ".*");
-            return Regex.IsMatch(request.Url, mockResponseUrlRegex);
+            return Regex.IsMatch(request.Url, $"^{mockResponseUrlRegex}$");
         });
         return mockResponse;
     }
@@ -133,7 +133,7 @@ private void ProcessMockResponse(SessionEventArgs e, MockResponse matchingRespon
                 headers.Add(new HttpHeader(key, matchingResponse.ResponseHeaders[key]));
             }
         }
-        // default the content type to application/json unlesss set in the mock response
+        // default the content type to application/json unless set in the mock response
         if (!headers.Any(h => h.Name.Equals("content-type", StringComparison.OrdinalIgnoreCase))) {
             headers.Add(new HttpHeader("content-type", "application/json"));
         }
diff --git a/msgraph-developer-proxy-plugins/RandomErrors/GenericRandomErrorPlugin.cs b/msgraph-developer-proxy-plugins/RandomErrors/GenericRandomErrorPlugin.cs
@@ -118,7 +118,7 @@ private void UpdateProxyResponse(ProxyRequestArgs ev, GenericErrorResponse error
 
     public override void Register(IPluginEvents pluginEvents,
                          IProxyContext context,
-                         ISet<Regex> urlsToWatch,
+                         ISet<UrlToWatch> urlsToWatch,
                          IConfigurationSection? configSection = null) {
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
diff --git a/msgraph-developer-proxy-plugins/RandomErrors/GraphRandomErrorPlugin.cs b/msgraph-developer-proxy-plugins/RandomErrors/GraphRandomErrorPlugin.cs
@@ -170,7 +170,7 @@ private void UpdateProxyResponse(ProxyRequestArgs ev, HttpStatusCode errorStatus
 
     public override void Register(IPluginEvents pluginEvents,
                          IProxyContext context,
-                         ISet<Regex> urlsToWatch,
+                         ISet<UrlToWatch> urlsToWatch,
                          IConfigurationSection? configSection = null) {
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
diff --git a/msgraph-developer-proxy-plugins/RequestLogs/ExecutionSummaryPlugin.cs b/msgraph-developer-proxy-plugins/RequestLogs/ExecutionSummaryPlugin.cs
@@ -6,7 +6,6 @@
 using System.CommandLine;
 using System.CommandLine.Invocation;
 using System.Text.Json.Serialization;
-using System.Text.RegularExpressions;
 
 namespace Microsoft.Graph.DeveloperProxy.Plugins.RequestLogs;
 
@@ -72,7 +71,7 @@ public ExecutionSummaryPlugin()
 
   public override void Register(IPluginEvents pluginEvents,
                           IProxyContext context,
-                          ISet<Regex> urlsToWatch,
+                          ISet<UrlToWatch> urlsToWatch,
                           IConfigurationSection? configSection = null)
   {
     base.Register(pluginEvents, context, urlsToWatch, configSection);
diff --git a/msgraph-developer-proxy/PluginLoader.cs b/msgraph-developer-proxy/PluginLoader.cs
@@ -9,9 +9,9 @@
 namespace Microsoft.Graph.DeveloperProxy;
 
 internal class PluginLoaderResult {
-    public ISet<Regex> UrlsToWatch { get; }
+    public ISet<UrlToWatch> UrlsToWatch { get; }
     public IEnumerable<IProxyPlugin> ProxyPlugins { get; }
-    public PluginLoaderResult(ISet<Regex> urlsToWatch, IEnumerable<IProxyPlugin> proxyPlugins) {
+    public PluginLoaderResult(ISet<UrlToWatch> urlsToWatch, IEnumerable<IProxyPlugin> proxyPlugins) {
         UrlsToWatch = urlsToWatch ?? throw new ArgumentNullException(nameof(urlsToWatch));
         ProxyPlugins = proxyPlugins ?? throw new ArgumentNullException(nameof(proxyPlugins));
     }
@@ -25,8 +25,8 @@ public PluginLoader(ILogger logger) {
     public PluginLoaderResult LoadPlugins(IPluginEvents pluginEvents, IProxyContext proxyContext) {
         List<IProxyPlugin> plugins = new();
         PluginConfig config = PluginConfig;
-        List<Regex> globallyWatchedUrls = PluginConfig.UrlsToWatch.Select(ConvertToRegex).ToList();
-        ISet<Regex> defaultUrlsToWatch = globallyWatchedUrls.ToHashSet();
+        List<UrlToWatch> globallyWatchedUrls = PluginConfig.UrlsToWatch.Select(ConvertToRegex).ToList();
+        ISet<UrlToWatch> defaultUrlsToWatch = globallyWatchedUrls.ToHashSet();
         string? rootDirectory = Path.GetDirectoryName(AppContext.BaseDirectory);
         if (!string.IsNullOrEmpty(rootDirectory)) {
             foreach (PluginReference h in config.Plugins) {
@@ -36,8 +36,8 @@ public PluginLoaderResult LoadPlugins(IPluginEvents pluginEvents, IProxyContext
                 PluginLoadContext pluginLoadContext = new PluginLoadContext(pluginLocation);
                 _logger.LogDebug($"Loading from: {pluginLocation}");
                 Assembly assembly = pluginLoadContext.LoadFromAssemblyName(new AssemblyName(Path.GetFileNameWithoutExtension(pluginLocation)));
-                IEnumerable<Regex>? pluginUrlsList = h.UrlsToWatch?.Select(ConvertToRegex);
-                ISet<Regex>? pluginUrls = null;
+                IEnumerable<UrlToWatch>? pluginUrlsList = h.UrlsToWatch?.Select(ConvertToRegex);
+                ISet<UrlToWatch>? pluginUrls = null;
                 if (pluginUrlsList is not null) {
                     pluginUrls = pluginUrlsList.ToHashSet();
                     globallyWatchedUrls.AddRange(pluginUrlsList);
@@ -69,8 +69,18 @@ private IProxyPlugin CreatePlugin(Assembly assembly, PluginReference h) {
             $"Available types: {availableTypes}");
     }
 
-    public static Regex ConvertToRegex(string stringMatcher) =>
-        new Regex(Regex.Escape(stringMatcher).Replace("\\*", ".*"), RegexOptions.Compiled | RegexOptions.IgnoreCase);
+    public static UrlToWatch ConvertToRegex(string stringMatcher) {
+        var exclude = false;
+        if (stringMatcher.StartsWith("!")) {
+            exclude = true;
+            stringMatcher = stringMatcher.Substring(1);
+        }
+
+        return new UrlToWatch(
+            new Regex($"^{Regex.Escape(stringMatcher).Replace("\\*", ".*")}$", RegexOptions.Compiled | RegexOptions.IgnoreCase),
+            exclude
+        );
+    }
 
     private PluginConfig? _pluginConfig;
     private ILogger _logger;
diff --git a/msgraph-developer-proxy/ProxyCommandHandler.cs b/msgraph-developer-proxy/ProxyCommandHandler.cs
@@ -5,7 +5,6 @@
 using Microsoft.Graph.DeveloperProxy.Abstractions;
 using System.CommandLine;
 using System.CommandLine.Invocation;
-using System.Text.RegularExpressions;
 
 namespace Microsoft.Graph.DeveloperProxy;
 
@@ -17,7 +16,7 @@ public class ProxyCommandHandler : ICommandHandler {
     public Option<IEnumerable<string>?> WatchProcessNames { get; set; }
 
     private readonly PluginEvents _pluginEvents;
-    private readonly ISet<Regex> _urlsToWatch;
+    private readonly ISet<UrlToWatch> _urlsToWatch;
     private readonly ILogger _logger;
 
     public ProxyCommandHandler(Option<int?> port,
@@ -26,7 +25,7 @@ public ProxyCommandHandler(Option<int?> port,
                                Option<IEnumerable<int>?> watchPids,
                                Option<IEnumerable<string>?> watchProcessNames,
                                PluginEvents pluginEvents,
-                               ISet<Regex> urlsToWatch,
+                               ISet<UrlToWatch> urlsToWatch,
                                ILogger logger) {
         Port = port ?? throw new ArgumentNullException(nameof(port));
         LogLevel = logLevel ?? throw new ArgumentNullException(nameof(logLevel));
diff --git a/msgraph-developer-proxy/ProxyEngine.cs b/msgraph-developer-proxy/ProxyEngine.cs
@@ -20,10 +20,10 @@ public class ProxyEngine {
     private ProxyServer? _proxyServer;
     private ExplicitProxyEndPoint? _explicitEndPoint;
     // lists of URLs to watch, used for intercepting requests
-    private ISet<Regex> _urlsToWatch = new HashSet<Regex>();
+    private ISet<UrlToWatch> _urlsToWatch = new HashSet<UrlToWatch>();
     // lists of hosts to watch extracted from urlsToWatch,
     // used for deciding which URLs to decrypt for further inspection
-    private ISet<Regex> _hostsToWatch = new HashSet<Regex>();
+    private ISet<UrlToWatch> _hostsToWatch = new HashSet<UrlToWatch>();
 
     private static string _productVersion = string.Empty;
     public static string ProductVersion {
@@ -41,7 +41,7 @@ public static string ProductVersion {
     private bool _isRecording = false;
     private List<RequestLog> _requestLogs = new List<RequestLog>();
 
-    public ProxyEngine(ProxyConfiguration config, ISet<Regex> urlsToWatch, PluginEvents pluginEvents, ILogger logger) {
+    public ProxyEngine(ProxyConfiguration config, ISet<UrlToWatch> urlsToWatch, PluginEvents pluginEvents, ILogger logger) {
         _config = config ?? throw new ArgumentNullException(nameof(config));
         _urlsToWatch = urlsToWatch ?? throw new ArgumentNullException(nameof(urlsToWatch));
         _pluginEvents = pluginEvents ?? throw new ArgumentNullException(nameof(pluginEvents));
@@ -177,25 +177,25 @@ private void PrintRecordingIndicator() {
     // We need this because before we decrypt a request, we only have access
     // to the host name, not the full URL.
     private void LoadHostNamesFromUrls() {
-        foreach (var url in _urlsToWatch) {
+        foreach (var urlToWatch in _urlsToWatch) {
             // extract host from the URL
-            string urlToWatch = Regex.Unescape(url.ToString()).Replace(".*", "*");
+            string urlToWatchPatter = Regex.Unescape(urlToWatch.Url.ToString()).Replace(".*", "*");
             string hostToWatch;
-            if (urlToWatch.ToString().Contains("://")) {
+            if (urlToWatchPatter.ToString().Contains("://")) {
                 // if the URL contains a protocol, extract the host from the URL
-                hostToWatch = urlToWatch.Split("://")[1].Substring(0, urlToWatch.Split("://")[1].IndexOf("/"));
+                hostToWatch = urlToWatchPatter.Split("://")[1].Substring(0, urlToWatchPatter.Split("://")[1].IndexOf("/"));
             }
             else {
                 // if the URL doesn't contain a protocol,
                 // we assume the whole URL is a host name
-                hostToWatch = urlToWatch;
+                hostToWatch = urlToWatchPatter;
             }
 
             var hostToWatchRegexString = Regex.Escape(hostToWatch).Replace("\\*", ".*");
-            Regex hostRegex = new Regex(hostToWatchRegexString, RegexOptions.Compiled | RegexOptions.IgnoreCase);
+            Regex hostRegex = new Regex($"^{hostToWatchRegexString}$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
             // don't add the same host twice
-            if (!_hostsToWatch.Any(h => h.ToString() == hostRegex.ToString())) {
-                _hostsToWatch.Add(hostRegex);
+            if (!_hostsToWatch.Any(h => h.Url.ToString() == hostRegex.ToString())) {
+                _hostsToWatch.Add(new UrlToWatch(hostRegex));
             }
         }
     }
@@ -342,7 +342,7 @@ private void HandleRequest(SessionEventArgs e) {
 
     private static void AddProxyHeader(Request r) => r.Headers?.AddHeader("Via", $"{r.HttpVersion} graph-proxy/{ProductVersion}");
 
-    private bool IsProxiedHost(string hostName) => _hostsToWatch.Any(h => h.IsMatch(hostName));
+    private bool IsProxiedHost(string hostName) => _hostsToWatch.Any(h => h.Url.IsMatch(hostName));
 
 
     // Modify response
diff --git a/msgraph-developer-proxy/ProxyHost.cs b/msgraph-developer-proxy/ProxyHost.cs
@@ -91,6 +91,6 @@ public RootCommand GetRootCommand() {
         return command;
     }
 
-    public ProxyCommandHandler GetCommandHandler(PluginEvents pluginEvents, ISet<Regex> urlsToWatch, ILogger logger) => new ProxyCommandHandler(_portOption, _logLevelOption, _recordOption, _watchPidsOption, _watchProcessNamesOption, pluginEvents, urlsToWatch, logger);
+    public ProxyCommandHandler GetCommandHandler(PluginEvents pluginEvents, ISet<UrlToWatch> urlsToWatch, ILogger logger) => new ProxyCommandHandler(_portOption, _logLevelOption, _recordOption, _watchPidsOption, _watchProcessNamesOption, pluginEvents, urlsToWatch, logger);
 }
 
