proxy: match NO_PROXY formats to libcurl behaviour

We aim to be compatible with the behaviour of Git as much as possible
when it comes to network settings. This enables users to setup Git proxy
settings and get the same setup "for free" with GCM.

Git uses libcurl to provide it's HTTP interactions. The NO_PROXY setting
is used by libcurl to disable proxy settings for specific hosts.

We previously attempted to plumb the value of NO_PROXY through to the
.NET WebProxy class' list of "bypassed addresses" (the set of hosts that
should not be proxied). However, the .NET class expects a set of
_regular expressions_ which is unlike libcurl!

As a result, libcurl permitted values for NO_PROXY were throwing errors
inside of GCM since they are not valid regexs.

In this commit we perform a transformation of the NO_PROXY list and
construct a set of regular expressions that match addresses in the same
way as libcurl does.

The transformation is as follows:

1. strip any leading periods '.' or wildcards '*.'
2. escape the remaining input to match literally (e.g.: '.' becomes '\.')
3. prepend a group that matches either a period '.' or a URI scheme
   delimiter '://' - this prevents partial domain matching
4. append a end-of-string symbol '$' to ensure we only match to the
   specified TLD and port

See the libcurl documentation on NO_PROXY behaviour:
https://curl.se/libcurl/c/CURLOPT_NOPROXY.html

Author: mjcheetham

docs/netconfig.md

@@ -62,8 +62,28 @@ addresses. GCM Core supports the cURL environment variable `NO_PROXY` for this
 scenariom, as does Git itself.
 
 The `NO_PROXY` environment variable should contain a comma (`,`) or space (` `)
-separated list of regular expressions to match hosts that should not be proxied
-(should connect directly).
+separated list of host names that should not be proxied (should connect
+directly).
+
+GCM Core attempts to match [libcurl's behaviour](https://curl.se/libcurl/c/CURLOPT_NOPROXY.html),
+which is briefly summarized here:
+
+- a value of `*` disables proxying for all hosts;
+- other wildcard use is **not** supported;
+- each name in the list is matched as a domain which contains the hostname,
+  or the hostname itself
+- a leading period/dot `.` matches against the provided hostname
+
+For example, setting `NO_PROXY` to `example.com` results in the following:
+
+Hostname|Matches?
+-|-
+`example.com`|:white_check_mark:
+`example.com:80`|:white_check_mark:
+`www.example.com`|:white_check_mark:
+`notanexample.com`|:x:
+`www.notanexample.com`|:x:
+`example.com.othertld`|:x:
 
 **Example:**
 

src/shared/Core.Tests/HttpClientFactoryTests.cs

@@ -88,13 +88,13 @@ public void HttpClientFactory_TryCreateProxy_ProxyWithBypass_ReturnsTrueOutProxy
             const string repoPath = "/tmp/repos/foo";
             const string repoRemote = "https://remote.example.com/foo.git";
 
-            var bypassList = new List<string> {"https://contoso.com", ".*fabrikam\\.com"};
+            var noProxyRaw = "contoso.com,fabrikam.com";
             var repoRemoteUri = new Uri(repoRemote);
             var proxyConfig = new ProxyConfiguration(
                 new Uri(proxyUrl),
                 userName: null,
                 password: null,
-                bypassHosts: bypassList);
+                noProxyRaw: noProxyRaw);
 
             var settings = new TestSettings
             {
@@ -117,6 +117,35 @@ public void HttpClientFactory_TryCreateProxy_ProxyWithBypass_ReturnsTrueOutProxy
             Assert.False(proxy.IsBypassed(repoRemoteUri));
         }
 
+        [Fact]
+        public void HttpClientFactory_TryCreateProxy_ProxyWithWildcardBypass_ReturnsFalse()
+        {
+            const string proxyUrl = "https://proxy.example.com/git";
+            const string repoPath = "/tmp/repos/foo";
+            const string repoRemote = "https://remote.example.com/foo.git";
+
+            var noProxyRaw = "*";
+            var repoRemoteUri = new Uri(repoRemote);
+            var proxyConfig = new ProxyConfiguration(
+                new Uri(proxyUrl),
+                userName: null,
+                password: null,
+                noProxyRaw: noProxyRaw);
+
+            var settings = new TestSettings
+            {
+                RemoteUri = repoRemoteUri,
+                RepositoryPath = repoPath,
+                ProxyConfiguration = proxyConfig
+            };
+            var httpFactory = new HttpClientFactory(Mock.Of<IFileSystem>(), Mock.Of<ITrace>(), settings, Mock.Of<IStandardStreams>());
+
+            bool result = httpFactory.TryCreateProxy(out IWebProxy proxy);
+
+            Assert.False(result);
+            Assert.Null(proxy);
+        }
+
         [Fact]
         public void HttpClientFactory_TryCreateProxy_ProxyWithCredentials_ReturnsTrueOutProxyWithUrlConfiguredCredentials()
         {

src/shared/Core.Tests/SettingsTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net;
 using GitCredentialManager.Tests.Objects;
 using Xunit;
 
@@ -427,6 +428,57 @@ public void Settings_IsWindowsIntegratedAuthenticationEnabled_ConfigNonBooleanyV
             Assert.True(settings.IsWindowsIntegratedAuthenticationEnabled);
         }
 
+        [Theory]
+        [InlineData("", new string[0])]
+        [InlineData("    ", new string[0])]
+        [InlineData(",", new string[0])]
+        [InlineData("example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
+        [InlineData("example.com:8080", new[] { @"(\.|\:\/\/)example\.com:8080$" })]
+        [InlineData("example.com,", new[] { @"(\.|\:\/\/)example\.com$" })]
+        [InlineData(",example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
+        [InlineData(",example.com,", new[] { @"(\.|\:\/\/)example\.com$" })]
+        [InlineData(".example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
+        [InlineData("..example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
+        [InlineData("*.example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
+        [InlineData("my.example.com", new[] { @"(\.|\:\/\/)my\.example\.com$" })]
+        [InlineData("example.com,contoso.com,fabrikam.com", new[]
+        {
+            @"(\.|\:\/\/)example\.com$",
+            @"(\.|\:\/\/)contoso\.com$",
+            @"(\.|\:\/\/)fabrikam\.com$"
+        })]
+        public void Settings_ProxyConfiguration_ConvertToBypassRegexArray(string input, string[] expected)
+        {
+            string[] actual = ProxyConfiguration.ConvertToBypassRegexArray(input).ToArray();
+            Assert.Equal(expected, actual);
+        }
+
+        [Theory]
+        [InlineData("example.com", "http://example.com", true)]
+        [InlineData("example.com", "https://example.com", true)]
+        [InlineData("example.com", "https://www.example.com", true)]
+        [InlineData("example.com", "http://www.example.com:80", true)]
+        [InlineData("example.com", "https://www.example.com:443", true)]
+        [InlineData("example.com", "https://www.example.com:8080", false)]
+        [InlineData("example.com", "http://notanexample.com", false)]
+        [InlineData("example.com", "https://notanexample.com", false)]
+        [InlineData("example.com", "https://www.notanexample.com", false)]
+        [InlineData("example.com", "https://example.com.otherltd", false)]
+        [InlineData("example.com:8080", "http://example.com", false)]
+        [InlineData("my.example.com", "http://example.com", false)]
+        public void Settings_ProxyConfiguration_ConvertToBypassRegexArray_WebProxyBypass(string noProxy, string address, bool expected)
+        {
+            var bypassList = ProxyConfiguration.ConvertToBypassRegexArray(noProxy).ToArray();
+            var webProxy = new WebProxy("https://localhost:8080/proxy")
+            {
+                BypassList = bypassList
+            };
+
+            bool actual = webProxy.IsBypassed(new Uri(address));
+
+            Assert.Equal(expected, actual);
+        }
+
         [Fact]
         public void Settings_ProxyConfiguration_Unset_ReturnsNull()
         {
@@ -458,11 +510,11 @@ public void Settings_ProxyConfiguration_GcmHttpConfig_ReturnsValue()
             const string expectedPassword = "letmein123";
             var expectedAddress = new Uri("http://proxy.example.com");
             var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
+            var expectedNoProxy = "contoso.com,fabrikam.com";
 
             var envars = new TestEnvironment
             {
-                Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)}
+                Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy}
             };
             var git = new TestGit();
             git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};
@@ -478,7 +530,7 @@ public void Settings_ProxyConfiguration_GcmHttpConfig_ReturnsValue()
             Assert.Equal(expectedAddress, actualConfig.Address);
             Assert.Equal(expectedUserName, actualConfig.UserName);
             Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
+            Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
             Assert.True(actualConfig.IsDeprecatedSource);
         }
 
@@ -494,11 +546,11 @@ public void Settings_ProxyConfiguration_GcmHttpsConfig_ReturnsValue()
             const string expectedPassword = "letmein123";
             var expectedAddress = new Uri("http://proxy.example.com");
             var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
+            var expectedNoProxy = "contoso.com,fabrikam.com";
 
             var envars = new TestEnvironment
             {
-                Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)}
+                Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy}
             };
             var git = new TestGit();
             git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};
@@ -514,7 +566,7 @@ public void Settings_ProxyConfiguration_GcmHttpsConfig_ReturnsValue()
             Assert.Equal(expectedAddress, actualConfig.Address);
             Assert.Equal(expectedUserName, actualConfig.UserName);
             Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
+            Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
             Assert.True(actualConfig.IsDeprecatedSource);
         }
 
@@ -530,11 +582,11 @@ public void Settings_ProxyConfiguration_GitHttpConfig_ReturnsValue()
             const string expectedPassword = "letmein123";
             var expectedAddress = new Uri("http://proxy.example.com");
             var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
+            var expectedNoProxy = "contoso.com,fabrikam.com";
 
             var envars = new TestEnvironment
             {
-                Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)}
+                Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy}
             };
             var git = new TestGit();
             git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};
@@ -550,7 +602,7 @@ public void Settings_ProxyConfiguration_GitHttpConfig_ReturnsValue()
             Assert.Equal(expectedAddress, actualConfig.Address);
             Assert.Equal(expectedUserName, actualConfig.UserName);
             Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
+            Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
             Assert.False(actualConfig.IsDeprecatedSource);
         }
 
@@ -579,42 +631,6 @@ public void Settings_ProxyConfiguration_GitHttpConfig_EmptyScopedUriUnscoped_Ret
             Assert.Null(actualConfig);
         }
 
-        [Fact]
-        public void Settings_ProxyConfiguration_NoProxyMixedSplitChar_ReturnsValue()
-        {
-            const string remoteUrl = "http://example.com/foo.git";
-            const string section = Constants.GitConfiguration.Http.SectionName;
-            const string property = Constants.GitConfiguration.Http.Proxy;
-            var remoteUri = new Uri(remoteUrl);
-
-            const string expectedUserName = "john.doe";
-            const string expectedPassword = "letmein123";
-            var expectedAddress = new Uri("http://proxy.example.com");
-            var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"contoso.com", "fabrikam.com", "example.com"};
-
-            var envars = new TestEnvironment
-            {
-                Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = "contoso.com, fabrikam.com example.com,"}
-            };
-            var git = new TestGit();
-            git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};
-
-            var settings = new Settings(envars, git)
-            {
-                RemoteUri = remoteUri
-            };
-
-            ProxyConfiguration actualConfig = settings.GetProxyConfiguration();
-
-            Assert.NotNull(actualConfig);
-            Assert.Equal(expectedAddress, actualConfig.Address);
-            Assert.Equal(expectedUserName, actualConfig.UserName);
-            Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
-            Assert.False(actualConfig.IsDeprecatedSource);
-        }
-
         [Fact]
         public void Settings_ProxyConfiguration_CurlHttpEnvar_ReturnsValue()
         {
@@ -625,14 +641,14 @@ public void Settings_ProxyConfiguration_CurlHttpEnvar_ReturnsValue()
             const string expectedPassword = "letmein123";
             var expectedAddress = new Uri("http://proxy.example.com");
             var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
+            var expectedNoProxy = "contoso.com,fabrikam.com";
 
             var envars = new TestEnvironment
             {
                 Variables =
                 {
                     [Constants.EnvironmentVariables.CurlHttpProxy] = settingValue.ToString(),
-                    [Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
+                    [Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
                 }
             };
             var git = new TestGit();
@@ -648,7 +664,7 @@ public void Settings_ProxyConfiguration_CurlHttpEnvar_ReturnsValue()
             Assert.Equal(expectedAddress, actualConfig.Address);
             Assert.Equal(expectedUserName, actualConfig.UserName);
             Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
+            Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
             Assert.False(actualConfig.IsDeprecatedSource);
         }
 
@@ -662,14 +678,14 @@ public void Settings_ProxyConfiguration_CurlHttpsEnvar_ReturnsValue()
             const string expectedPassword = "letmein123";
             var expectedAddress = new Uri("http://proxy.example.com");
             var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
+            var expectedNoProxy = "contoso.com,fabrikam.com";
 
             var envars = new TestEnvironment
             {
                 Variables =
                 {
                     [Constants.EnvironmentVariables.CurlHttpsProxy] = settingValue.ToString(),
-                    [Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
+                    [Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
                 }
             };
             var git = new TestGit();
@@ -685,7 +701,7 @@ public void Settings_ProxyConfiguration_CurlHttpsEnvar_ReturnsValue()
             Assert.Equal(expectedAddress, actualConfig.Address);
             Assert.Equal(expectedUserName, actualConfig.UserName);
             Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
+            Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
             Assert.False(actualConfig.IsDeprecatedSource);
         }
 
@@ -699,14 +715,14 @@ public void Settings_TryGetProxy_CurlAllEnvar_ReturnsValue()
             const string expectedPassword = "letmein123";
             var expectedAddress = new Uri("http://proxy.example.com");
             var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
+            var expectedNoProxy = "contoso.com,fabrikam.com";
 
             var envars = new TestEnvironment
             {
                 Variables =
                 {
                     [Constants.EnvironmentVariables.CurlAllProxy] = settingValue.ToString(),
-                    [Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
+                    [Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
                 }
             };
             var git = new TestGit();
@@ -722,7 +738,7 @@ public void Settings_TryGetProxy_CurlAllEnvar_ReturnsValue()
             Assert.Equal(expectedAddress, actualConfig.Address);
             Assert.Equal(expectedUserName, actualConfig.UserName);
             Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
+            Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
             Assert.False(actualConfig.IsDeprecatedSource);
         }
 
@@ -736,14 +752,14 @@ public void Settings_ProxyConfiguration_LegacyGcmEnvar_ReturnsValue()
             const string expectedPassword = "letmein123";
             var expectedAddress = new Uri("http://proxy.example.com");
             var settingValue = new Uri("http://john.doe:[email protected]");
-            var bypassList = new List<string> {"https://contoso.com", ".*fabrikam\\.com"};
+            var expectedNoProxy = "contoso.com,fabrikam.com";
 
             var envars = new TestEnvironment
             {
                 Variables =
                 {
                     [Constants.EnvironmentVariables.GcmHttpProxy] = settingValue.ToString(),
-                    [Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
+                    [Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
                 }
             };
             var git = new TestGit();
@@ -759,7 +775,7 @@ public void Settings_ProxyConfiguration_LegacyGcmEnvar_ReturnsValue()
             Assert.Equal(expectedAddress, actualConfig.Address);
             Assert.Equal(expectedUserName, actualConfig.UserName);
             Assert.Equal(expectedPassword, actualConfig.Password);
-            Assert.Equal(bypassList, actualConfig.BypassHosts);
+            Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
             Assert.True(actualConfig.IsDeprecatedSource);
         }