WIP Limit registry connections to prevent 503 errors during push

cmd/nerdctl/image/image_push.go

@@ -69,6 +69,17 @@ func PushCommand() *cobra.Command {
 
 	cmd.Flags().Bool(allowNonDistFlag, false, "Allow pushing images with non-distributable blobs")
 
+	// #region connection limit flags
+	cmd.Flags().Int("max-conns-per-host", 5, "Maximum number of connections per registry host")
+	cmd.Flags().Int("max-idle-conns", 50, "Maximum number of idle connections")
+	cmd.Flags().Int("request-timeout", 300, "Request timeout in seconds")
+	// #endregion
+
+	// #region retry flags
+	cmd.Flags().Int("max-retries", 3, "Maximum number of retry attempts for 503 errors")
+	cmd.Flags().Int("retry-initial-delay", 1000, "Initial delay before first retry in milliseconds")
+	// #endregion
+
 	return cmd
 }
 
@@ -113,6 +124,26 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 	if err != nil {
 		return types.ImagePushOptions{}, err
 	}
+	maxConnsPerHost, err := cmd.Flags().GetInt("max-conns-per-host")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	maxIdleConns, err := cmd.Flags().GetInt("max-idle-conns")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	requestTimeout, err := cmd.Flags().GetInt("request-timeout")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	maxRetries, err := cmd.Flags().GetInt("max-retries")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	retryInitialDelay, err := cmd.Flags().GetInt("retry-initial-delay")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
 	return types.ImagePushOptions{
 		GOptions:                       globalOptions,
 		SignOptions:                    signOptions,
@@ -124,6 +155,11 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 		IpfsAddress:                    ipfsAddress,
 		Quiet:                          quiet,
 		AllowNondistributableArtifacts: allowNonDist,
+		MaxConnsPerHost:                maxConnsPerHost,
+		MaxIdleConns:                   maxIdleConns,
+		RequestTimeout:                 requestTimeout,
+		MaxRetries:                     maxRetries,
+		RetryInitialDelay:              retryInitialDelay,
 		Stdout:                         cmd.OutOrStdout(),
 	}, nil
 }

pkg/api/types/image_types.go

@@ -200,6 +200,16 @@ type ImagePushOptions struct {
 	Quiet bool
 	// AllowNondistributableArtifacts allow pushing non-distributable artifacts
 	AllowNondistributableArtifacts bool
+	// MaxConnsPerHost maximum number of connections per registry host (default: 5)
+	MaxConnsPerHost int
+	// MaxIdleConns maximum number of idle connections (default: 50)
+	MaxIdleConns int
+	// RequestTimeout timeout for registry requests in seconds (default: 300)
+	RequestTimeout int
+	// MaxRetries maximum number of retry attempts for 503 errors (default: 3)
+	MaxRetries int
+	// RetryInitialDelay initial delay before first retry in milliseconds (default: 1000)
+	RetryInitialDelay int
 }
 
 // RemoteSnapshotterFlags are used for pulling with remote snapshotters

pkg/cmd/image/push.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -34,7 +35,6 @@ import (
 	"github.com/containerd/containerd/v2/core/images/converter"
 	"github.com/containerd/containerd/v2/core/remotes"
 	"github.com/containerd/containerd/v2/core/remotes/docker"
-	dockerconfig "github.com/containerd/containerd/v2/core/remotes/docker/config"
 	"github.com/containerd/containerd/v2/pkg/reference"
 	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/estargz"
@@ -165,17 +165,29 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 	}
 	dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(options.GOptions.HostsDir))
 
-	ho, err := dockerconfigresolver.NewHostOptions(ctx, refDomain, dOpts...)
-	if err != nil {
-		return err
+	// Configure connection limits to prevent registry overload (503 errors)
+	if options.MaxConnsPerHost > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxConnsPerHost(options.MaxConnsPerHost))
 	}
-
-	resolverOpts := docker.ResolverOptions{
-		Tracker: pushTracker,
-		Hosts:   dockerconfig.ConfigureHosts(ctx, *ho),
+	if options.MaxIdleConns > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxIdleConns(options.MaxIdleConns))
+	}
+	if options.RequestTimeout > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithRequestTimeout(time.Duration(options.RequestTimeout)*time.Second))
 	}
+	if options.MaxRetries > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxRetries(options.MaxRetries))
+	}
+	if options.RetryInitialDelay > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithRetryInitialDelay(time.Duration(options.RetryInitialDelay)*time.Millisecond))
+	}
+	// Use the local push tracker for this operation
+	dOpts = append(dOpts, dockerconfigresolver.WithTracker(pushTracker))
 
-	resolver := docker.NewResolver(resolverOpts)
+	resolver, err := dockerconfigresolver.New(ctx, refDomain, dOpts...)
+	if err != nil {
+		return err
+	}
 	if err = pushFunc(resolver); err != nil {
 		// In some circumstance (e.g. people just use 80 port to support pure http), the error will contain message like "dial tcp <port>: connection refused"
 		if !errors.Is(err, http.ErrSchemeMismatch) && !errutil.IsErrConnectionRefused(err) {
@@ -184,6 +196,22 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		if options.GOptions.InsecureRegistry {
 			log.G(ctx).WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain)
 			dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true))
+			// Apply same connection limits for HTTP fallback
+			if options.MaxConnsPerHost > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithMaxConnsPerHost(options.MaxConnsPerHost))
+			}
+			if options.MaxIdleConns > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithMaxIdleConns(options.MaxIdleConns))
+			}
+			if options.RequestTimeout > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithRequestTimeout(time.Duration(options.RequestTimeout)*time.Second))
+			}
+			if options.MaxRetries > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithMaxRetries(options.MaxRetries))
+			}
+			if options.RetryInitialDelay > 0 {
+				dOpts = append(dOpts, dockerconfigresolver.WithRetryInitialDelay(time.Duration(options.RetryInitialDelay)*time.Millisecond))
+			}
 			resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...)
 			if err != nil {
 				return err