Merge pull request #481 from mjcheetham/autov2

Speed up host provider auto-detection and make more robust

Author: mjcheetham

docs/autodetect.md

@@ -0,0 +1,59 @@
+# Host provider auto-detection
+
+Git Credential Manager (GCM) supports authentication with multiple different Git
+host providers including: GitHub, Bitbucket, and Azure Repos. As well as the
+hosted/cloud offerings, GCM can also work with the self-hosted or "on-premises"
+versions of these services: GitHub Enterprise Server, Bitbucket DC Server, and
+Azure DevOps Server (TFS).
+
+By default, GCM will attempt to automatically detect which particular provider
+is behind the Git remote URL you're interacting with. For the cloud versions of
+the supported providers this is done by matching the hostname of the remote URL
+to the well-known hostnames of the services. For example "github.com" or
+"dev.azure.com".
+
+## Self-hosted/on-prem detection
+
+In order to detect which host provider to use for a self-hosted instance, each
+provider can provide some heuristic matching of the hostname. For example any
+hostname that begins "github.*" will be matched to the GitHub host provider.
+
+If a heuristic matches incorrectly, you can always [explicitly configure](#explicit-configuration)
+GCM to use a particular provider.
+
+## Remote URL probing
+
+In addition to heuristic matching, GCM will make a network call to the remote
+URL and inspect HTTP response headers to try and detect a self-hosted instance.
+
+This network call is only performed if neither an exact nor fuzzy match by
+hostname can be made. Only one HTTP `HEAD` call is made per credential request
+received by Git. To avoid this network call, please [explicitly configure](#explicit-configuration)
+the host provider for your self-hosted instance.
+
+### Timeout
+
+You can control how long GCM will wait for a response to the remote network call
+by setting the [`GCM_AUTODETECT_TIMEOUT`](environment.md#GCM_AUTODETECT_TIMEOUT)
+environment variable, or the [`credential.autoDetectTimeout`](configuration.md#credentialautodetecttimeout)
+Git configuration setting to the maximum number of milliseconds to wait.
+
+The default value is 2000 milliseconds (2 seconds). You can prevent the network
+call altogether by setting a zero or negative value, for example -1.
+
+## Explicit configuration
+
+If the auto-detection mechanism fails to select the correct host provider, or
+if the remote probing network call is causing performance issues, you can
+configure GCM to always use a particular host provider, for a given remote URL.
+
+You can either use the the [`GCM_PROVIDER`](environment.md#GCM_PROVIDER)
+environment variable, or the [`credential.provider`](configuration.md#credentialprovider)
+Git configuration setting for this purpose.
+
+For example to tell GCM to always use the GitHub host provider for the
+"ghe.example.com" hostname, you can run the following command:
+
+```shell
+git config --global credential.ghe.example.com.provider github
+```

docs/configuration.md

@@ -60,7 +60,7 @@ Define the host provider to use when authenticating.
 
 ID|Provider
 -|-
-`auto` _(default)_|_\[automatic\]_
+`auto` _(default)_|_\[automatic\]_ ([learn more](autodetect.md))
 `azure-repos`|Azure Repos
 `github`|GitHub
 `bitbucket`|Bitbucket
@@ -106,6 +106,27 @@ git config --global credential.ghe.contoso.com.authority github
 
 ---
 
+### credential.autoDetectTimeout
+
+Set the maximum length of time, in milliseconds, that GCM should wait for a
+network response during host provider auto-detection probing.
+
+See [here](autodetect.md) for more information.
+
+**Note:** Use a negative or zero value to disable probing altogether.
+
+Defaults to 2000 milliseconds (2 seconds).
+
+#### Example
+
+```shell
+git config --global credential.autoDetectTimeout -1
+```
+
+**Also see: [GCM_AUTODETECT_TIMEOUT](environment.md#GCM_AUTODETECT_TIMEOUT)**
+
+---
+
 ### credential.allowWindowsAuth
 
 Allow detection of Windows Integrated Authentication (WIA) support for generic host providers. Setting this value to `false` will prevent the use of WIA and force a basic authentication prompt when using the Generic host provider.

docs/environment.md

@@ -166,7 +166,7 @@ Define the host provider to use when authenticating.
 
 ID|Provider
 -|-
-`auto` _(default)_|_\[automatic\]_
+`auto` _(default)_|_\[automatic\]_ ([learn more](autodetect.md))
 `azure-repos`|Azure Repos
 `github`|GitHub
 `generic`|Generic (any other provider not listed above)
@@ -226,6 +226,35 @@ export GCM_AUTHORITY=github
 
 ---
 
+### GCM_AUTODETECT_TIMEOUT
+
+Set the maximum length of time, in milliseconds, that GCM should wait for a
+network response during host provider auto-detection probing.
+
+See [here](autodetect.md) for more information.
+
+**Note:** Use a negative or zero value to disable probing altogether.
+
+Defaults to 2000 milliseconds (2 seconds).
+
+#### Example
+
+##### Windows
+
+```batch
+SET GCM_AUTODETECT_TIMEOUT=-1
+```
+
+##### macOS/Linux
+
+```bash
+export GCM_AUTODETECT_TIMEOUT=-1
+```
+
+**Also see: [credential.autoDetectTimeout](configuration.md#credentialautodetecttimeout)**
+
+---
+
 ### GCM_ALLOW_WINDOWSAUTH
 
 Allow detection of Windows Integrated Authentication (WIA) support for generic host providers. Setting this value to `false` will prevent the use of WIA and force a basic authentication prompt when using the Generic host provider.

src/shared/Microsoft.Git.CredentialManager.Tests/HostProviderRegistryTests.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.Git.CredentialManager.Tests.Objects;
 using Moq;
@@ -257,5 +259,154 @@ public async Task HostProviderRegistry_GetProvider_AutoLegacyAuthoritySpecified_
 
             Assert.Same(provider2Mock.Object, result);
         }
+
+        [Fact]
+        public async Task HostProviderRegistry_GetProvider_Auto_NetworkProbe_ReturnsSupportedProvider()
+        {
+            var context = new TestCommandContext();
+            var registry = new HostProviderRegistry(context);
+            var remoteUri = new Uri("https://provider2.onprem.example.com");
+            var input = new InputArguments(
+                new Dictionary<string, string>
+                {
+                    ["protocol"] = remoteUri.Scheme,
+                    ["host"] = remoteUri.Host
+                }
+            );
+
+            var provider1Mock = new Mock<IHostProvider>();
+            provider1Mock.Setup(x => x.IsSupported(It.IsAny<InputArguments>())).Returns(false);
+            provider1Mock.Setup(x => x.IsSupported(It.IsAny<HttpResponseMessage>())).Returns(false);
+
+            var provider2Mock = new Mock<IHostProvider>();
+            provider2Mock.Setup(x => x.IsSupported(It.IsAny<InputArguments>())).Returns(false);
+            provider2Mock.Setup(x => x.IsSupported(It.IsAny<HttpResponseMessage>())).Returns(true);
+
+            var responseMessage = new HttpResponseMessage(HttpStatusCode.Unauthorized)
+            {
+                Headers = { { "X-Provider2", "true" } }
+            };
+
+            var httpHandler = new TestHttpMessageHandler();
+
+            httpHandler.Setup(HttpMethod.Head, remoteUri, responseMessage);
+            context.HttpClientFactory.MessageHandler = httpHandler;
+
+            registry.Register(provider1Mock.Object, HostProviderPriority.Normal);
+            registry.Register(provider2Mock.Object, HostProviderPriority.Normal);
+
+            IHostProvider result = await registry.GetProviderAsync(input);
+
+            httpHandler.AssertRequest(HttpMethod.Head, remoteUri, 1);
+            Assert.Same(provider2Mock.Object, result);
+        }
+
+        [Fact]
+        public async Task HostProviderRegistry_GetProvider_Auto_NetworkProbe_TimeoutZero_NoNetworkCall()
+        {
+            var context = new TestCommandContext();
+            var registry = new HostProviderRegistry(context);
+            var remoteUri = new Uri("https://onprem.example.com");
+            var input = new InputArguments(
+                new Dictionary<string, string>
+                {
+                    ["protocol"] = remoteUri.Scheme,
+                    ["host"] = remoteUri.Host
+                }
+            );
+
+            var providerMock = new Mock<IHostProvider>();
+            providerMock.Setup(x => x.IsSupported(It.IsAny<InputArguments>())).Returns(false);
+            providerMock.Setup(x => x.IsSupported(It.IsAny<HttpResponseMessage>())).Returns(true);
+
+            var responseMessage = new HttpResponseMessage(HttpStatusCode.Unauthorized);
+            var httpHandler = new TestHttpMessageHandler();
+
+            httpHandler.Setup(HttpMethod.Head, remoteUri, responseMessage);
+            context.HttpClientFactory.MessageHandler = httpHandler;
+
+            registry.Register(providerMock.Object, HostProviderPriority.Normal);
+
+            context.Settings.AutoDetectProviderTimeout = 0;
+
+            await Assert.ThrowsAnyAsync<Exception>(() => registry.GetProviderAsync(input));
+
+            httpHandler.AssertRequest(HttpMethod.Head, remoteUri, 0);
+        }
+
+        [Fact]
+        public async Task HostProviderRegistry_GetProvider_Auto_NetworkProbe_TimeoutNegative_NoNetworkCall()
+        {
+            var context = new TestCommandContext();
+            var registry = new HostProviderRegistry(context);
+            var remoteUri = new Uri("https://onprem.example.com");
+            var input = new InputArguments(
+                new Dictionary<string, string>
+                {
+                    ["protocol"] = remoteUri.Scheme,
+                    ["host"] = remoteUri.Host
+                }
+            );
+
+            var providerMock = new Mock<IHostProvider>();
+            providerMock.Setup(x => x.IsSupported(It.IsAny<InputArguments>())).Returns(false);
+            providerMock.Setup(x => x.IsSupported(It.IsAny<HttpResponseMessage>())).Returns(true);
+
+            var responseMessage = new HttpResponseMessage(HttpStatusCode.Unauthorized);
+            var httpHandler = new TestHttpMessageHandler();
+
+            httpHandler.Setup(HttpMethod.Head, remoteUri, responseMessage);
+            context.HttpClientFactory.MessageHandler = httpHandler;
+
+            registry.Register(providerMock.Object, HostProviderPriority.Normal);
+
+            context.Settings.AutoDetectProviderTimeout = -1;
+
+            await Assert.ThrowsAnyAsync<Exception>(() => registry.GetProviderAsync(input));
+
+            httpHandler.AssertRequest(HttpMethod.Head, remoteUri, 0);
+        }
+
+        [Fact]
+        public async Task HostProviderRegistry_GetProvider_Auto_NetworkProbe_NoNetwork_ReturnsLastProvider()
+        {
+            var context = new TestCommandContext();
+            var registry = new HostProviderRegistry(context);
+            var remoteUri = new Uri("https://provider2.onprem.example.com");
+            var input = new InputArguments(
+                new Dictionary<string, string>
+                {
+                    ["protocol"] = remoteUri.Scheme,
+                    ["host"] = remoteUri.Host
+                }
+            );
+
+            var highProviderMock = new Mock<IHostProvider>();
+            highProviderMock.Setup(x => x.IsSupported(It.IsAny<InputArguments>())).Returns(false);
+            highProviderMock.Setup(x => x.IsSupported(It.IsAny<HttpResponseMessage>())).Returns(false);
+            registry.Register(highProviderMock.Object, HostProviderPriority.Normal);
+
+            var lowProviderMock = new Mock<IHostProvider>();
+            lowProviderMock.Setup(x => x.IsSupported(It.IsAny<InputArguments>())).Returns(true);
+            registry.Register(lowProviderMock.Object, HostProviderPriority.Low);
+
+            var responseMessage = new HttpResponseMessage(HttpStatusCode.Unauthorized)
+            {
+                Headers = { { "X-Provider2", "true" } }
+            };
+
+            var httpHandler = new TestHttpMessageHandler
+            {
+                SimulateNoNetwork = true,
+            };
+
+            httpHandler.Setup(HttpMethod.Head, remoteUri, responseMessage);
+            context.HttpClientFactory.MessageHandler = httpHandler;
+
+            IHostProvider result = await registry.GetProviderAsync(input);
+
+            httpHandler.AssertRequest(HttpMethod.Head, remoteUri, 1);
+            Assert.Same(lowProviderMock.Object, result);
+        }
     }
 }

src/shared/Microsoft.Git.CredentialManager/Constants.cs

@@ -7,6 +7,7 @@ public static class Constants
     {
         public const string PersonalAccessTokenUserName = "PersonalAccessToken";
         public const string DefaultCredentialNamespace = "git";
+        public const int DefaultAutoDetectProviderTimeoutMs = 2000; // 2 seconds
 
         public const string ProviderIdAuto  = "auto";
         public const string AuthorityIdAuto = "auto";
@@ -70,6 +71,7 @@ public static class EnvironmentVariables
             public const string GcmDpapiStorePath     = "GCM_DPAPI_STORE_PATH";
             public const string GitExecutablePath     = "GIT_EXEC_PATH";
             public const string GpgExecutablePath     = "GCM_GPG_PATH";
+            public const string GcmAutoDetectTimeout  = "GCM_AUTODETECT_TIMEOUT";
         }
 
         public static class Http
@@ -103,6 +105,7 @@ public static class Credential
                 public const string PlaintextStorePath = "plaintextStorePath";
                 public const string DpapiStorePath = "dpapiStorePath";
                 public const string UserName = "username";
+                public const string AutoDetectTimeout = "autoDetectTimeout";
             }
 
             public static class Http
@@ -138,6 +141,7 @@ public static class HelpUrls
             public const string GcmTlsVerification     = "https://aka.ms/gcmcore-tlsverify";
             public const string GcmCredentialStores    = "https://aka.ms/gcmcore-credstores";
             public const string GcmWamComSecurity      = "https://aka.ms/gcmcore-wamadmin";
+            public const string GcmAutoDetect          = "https://aka.ms/gcmcore-autodetect";
         }
 
         private static Version _gcmVersion;