Limit registry connections to prevent 503 errors during push

Add HTTP client configuration with connection limits to prevent registry
overload when pushing images. This addresses 503 errors that occur when
too many parallel requests are made to registries.

Changes:
- Add MaxConnsPerHost (default: 5) and MaxIdleConns (default: 50) options
- Add RequestTimeout configuration for large uploads
- Apply limits to both HTTPS and HTTP fallback scenarios
- Maintain backward compatibility with existing configurations

Signed-off-by: Adam Rozman <[email protected]>

Author: Rozzii

cmd/nerdctl/image/image_push.go

@@ -69,6 +69,17 @@ func PushCommand() *cobra.Command {
 
 	cmd.Flags().Bool(allowNonDistFlag, false, "Allow pushing images with non-distributable blobs")
 
+	// #region connection limit flags
+	cmd.Flags().Int("max-conns-per-host", 5, "Maximum number of connections per registry host")
+	cmd.Flags().Int("max-idle-conns", 50, "Maximum number of idle connections")
+	cmd.Flags().Int("request-timeout", 300, "Request timeout in seconds")
+	// #endregion
+
+	// #region retry flags
+	cmd.Flags().Int("max-retries", 3, "Maximum number of retry attempts for 503 errors")
+	cmd.Flags().Int("retry-initial-delay", 1000, "Initial delay before first retry in milliseconds")
+	// #endregion
+
 	return cmd
 }
 
@@ -113,6 +124,26 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 	if err != nil {
 		return types.ImagePushOptions{}, err
 	}
+	maxConnsPerHost, err := cmd.Flags().GetInt("max-conns-per-host")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	maxIdleConns, err := cmd.Flags().GetInt("max-idle-conns")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	requestTimeout, err := cmd.Flags().GetInt("request-timeout")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	maxRetries, err := cmd.Flags().GetInt("max-retries")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	retryInitialDelay, err := cmd.Flags().GetInt("retry-initial-delay")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
 	return types.ImagePushOptions{
 		GOptions:                       globalOptions,
 		SignOptions:                    signOptions,
@@ -124,6 +155,11 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 		IpfsAddress:                    ipfsAddress,
 		Quiet:                          quiet,
 		AllowNondistributableArtifacts: allowNonDist,
+		MaxConnsPerHost:                maxConnsPerHost,
+		MaxIdleConns:                   maxIdleConns,
+		RequestTimeout:                 requestTimeout,
+		MaxRetries:                     maxRetries,
+		RetryInitialDelay:              retryInitialDelay,
 		Stdout:                         cmd.OutOrStdout(),
 	}, nil
 }

pkg/api/types/image_types.go

@@ -200,6 +200,16 @@ type ImagePushOptions struct {
 	Quiet bool
 	// AllowNondistributableArtifacts allow pushing non-distributable artifacts
 	AllowNondistributableArtifacts bool
+	// MaxConnsPerHost maximum number of connections per registry host (default: 5)
+	MaxConnsPerHost int
+	// MaxIdleConns maximum number of idle connections (default: 50)
+	MaxIdleConns int
+	// RequestTimeout timeout for registry requests in seconds (default: 300)
+	RequestTimeout int
+	// MaxRetries maximum number of retry attempts for 503 errors (default: 3)
+	MaxRetries int
+	// RetryInitialDelay initial delay before first retry in milliseconds (default: 1000)
+	RetryInitialDelay int
 }
 
 // RemoteSnapshotterFlags are used for pulling with remote snapshotters

pkg/cmd/image/push.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -34,7 +35,6 @@ import (
 	"github.com/containerd/containerd/v2/core/images/converter"
 	"github.com/containerd/containerd/v2/core/remotes"
 	"github.com/containerd/containerd/v2/core/remotes/docker"
-	dockerconfig "github.com/containerd/containerd/v2/core/remotes/docker/config"
 	"github.com/containerd/containerd/v2/pkg/reference"
 	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/estargz"
@@ -165,17 +165,29 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 	}
 	dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(options.GOptions.HostsDir))
 
-	ho, err := dockerconfigresolver.NewHostOptions(ctx, refDomain, dOpts...)
-	if err != nil {
-		return err
+	// Configure connection limits to prevent registry overload (503 errors)
+	if options.MaxConnsPerHost > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxConnsPerHost(options.MaxConnsPerHost))
 	}
-
-	resolverOpts := docker.ResolverOptions{
-		Tracker: pushTracker,
-		Hosts:   dockerconfig.ConfigureHosts(ctx, *ho),
+	if options.MaxIdleConns > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxIdleConns(options.MaxIdleConns))
+	}
+	if options.RequestTimeout > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithRequestTimeout(time.Duration(options.RequestTimeout)*time.Second))
 	}
+	if options.MaxRetries > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxRetries(options.MaxRetries))
+	}
+	if options.RetryInitialDelay > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithRetryInitialDelay(time.Duration(options.RetryInitialDelay)*time.Millisecond))
+	}
+	// Use the local push tracker for this operation
+	dOpts = append(dOpts, dockerconfigresolver.WithTracker(pushTracker))
 
-	resolver := docker.NewResolver(resolverOpts)
+	resolver, err := dockerconfigresolver.New(ctx, refDomain, dOpts...)
+	if err != nil {
+		return err
+	}
 	if err = pushFunc(resolver); err != nil {
 		// In some circumstance (e.g. people just use 80 port to support pure http), the error will contain message like "dial tcp <port>: connection refused"
 		if !errors.Is(err, http.ErrSchemeMismatch) && !errutil.IsErrConnectionRefused(err) {
@@ -184,6 +196,22 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		if options.GOptions.InsecureRegistry {
 			log.G(ctx).WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain)
 			dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true))
+			// Apply same connection limits for HTTP fallback
+			if options.MaxConnsPerHost > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithMaxConnsPerHost(options.MaxConnsPerHost))
+			}
+			if options.MaxIdleConns > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithMaxIdleConns(options.MaxIdleConns))
+			}
+			if options.RequestTimeout > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithRequestTimeout(time.Duration(options.RequestTimeout)*time.Second))
+			}
+			if options.MaxRetries > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithMaxRetries(options.MaxRetries))
+			}
+			if options.RetryInitialDelay > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithRetryInitialDelay(time.Duration(options.RetryInitialDelay)*time.Millisecond))
+			}
 			resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...)
 			if err != nil {
 				return err