libraries/doltcore/remotestorage: Improve connection reuse when fetching chunks from remote storage.

Improves performance of fetches from DoltHub, sql-server, and doltlab.

Improves some situations where routers and network access points do not respond
well to clients which open a great number of connections to the same TCP
endpoint, many of which live for a very short period of time.

http.DefaultClient comes with a default MaxIdleConnsPerHost of 2. Fetching with
a high number of concurrent downloads results in many connections not being
reused efficiently because they cannot get back into the idle pool because it
is so small. Here we increase the MaxIdleConnsPerHost to be large enough so
that our active connections during a fetch will fit within it.

Note: S3 limits a single HTTP connection to 100 requests, after which it is
closed server-side. So currently dolt can still require a large number of new
connections over the course of a pull, and the rate at which they are opened
will depend on a number of factors including available throughput, round trip
time resolving storage locations, etc. But this change will always be a big
improvement over the old behavior.

Author: reltuk

go/libraries/doltcore/env/grpc_dial_provider.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"time"
 	"unicode"
 
 	"google.golang.org/grpc"
@@ -33,6 +34,26 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/grpcendpoint"
 )
 
+var defaultDialer = &net.Dialer{
+	Timeout:   30 * time.Second,
+	KeepAlive: 30 * time.Second,
+}
+
+var defaultTransport = &http.Transport{
+	Proxy:                 http.ProxyFromEnvironment,
+	DialContext:           defaultDialer.DialContext,
+	ForceAttemptHTTP2:     true,
+	MaxIdleConns:          1024,
+	MaxIdleConnsPerHost:   256,
+	IdleConnTimeout:       90 * time.Second,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 1 * time.Second,
+}
+
+var defaultHttpFetcher grpcendpoint.HTTPFetcher = &http.Client{
+	Transport: defaultTransport,
+}
+
 // GRPCDialProvider implements dbfactory.GRPCDialProvider. By default, it is not able to use custom user credentials, but
 // if it is initialized with a DoltEnv, it will load custom user credentials from it.
 type GRPCDialProvider struct {
@@ -66,18 +87,18 @@ func (p GRPCDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (dbfacto
 		}
 	}
 
-	var httpfetcher grpcendpoint.HTTPFetcher = http.DefaultClient
+	var httpfetcher grpcendpoint.HTTPFetcher = defaultHttpFetcher
 
 	var opts []grpc.DialOption
 	if config.TLSConfig != nil {
 		tc := credentials.NewTLS(config.TLSConfig)
 		opts = append(opts, grpc.WithTransportCredentials(tc))
 
+		transport := *defaultTransport
+		transport.TLSClientConfig = config.TLSConfig
+		transport.ForceAttemptHTTP2 = true
 		httpfetcher = &http.Client{
-			Transport: &http.Transport{
-				TLSClientConfig:   config.TLSConfig,
-				ForceAttemptHTTP2: true,
-			},
+			Transport: &transport,
 		}
 	} else if config.Insecure {
 		opts = append(opts, grpc.WithInsecure())
@@ -109,6 +130,7 @@ func (p GRPCDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (dbfacto
 			opts = append(opts, grpc.WithPerRPCCredentials(rpcCreds))
 		}
 	}
+
 	return dbfactory.GRPCRemoteConfig{
 		Endpoint:    endpoint,
 		DialOptions: opts,

go/libraries/doltcore/remotestorage/chunk_store.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"sort"
@@ -49,7 +50,25 @@ var ErrCacheCapacityExceeded = errors.New("too much data: the cache capacity has
 
 var ErrUploadFailed = errors.New("upload failed")
 
-var globalHttpFetcher HTTPFetcher = &http.Client{}
+var defaultDialer = &net.Dialer{
+	Timeout:   30 * time.Second,
+	KeepAlive: 30 * time.Second,
+}
+
+var defaultTransport = &http.Transport{
+	Proxy:                 http.ProxyFromEnvironment,
+	DialContext:           defaultDialer.DialContext,
+	ForceAttemptHTTP2:     true,
+	MaxIdleConns:          1024,
+	MaxIdleConnsPerHost:   256,
+	IdleConnTimeout:       90 * time.Second,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 1 * time.Second,
+}
+
+var globalHttpFetcher HTTPFetcher = &http.Client{
+	Transport: defaultTransport,
+}
 
 var _ chunks.TableFileStore = (*DoltChunkStore)(nil)
 var _ nbs.NBSCompressedChunkStore = (*DoltChunkStore)(nil)