Merge pull request containerd#4290 from haytok/issue_4027

fix: allow containers to start using a large numbers of ports

Author: AkihiroSuda

cmd/nerdctl/compose/compose_port.go

@@ -88,11 +88,18 @@ func portAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	dataStore, err := clientutil.DataStore(globalOptions.DataRoot, globalOptions.Address)
+	if err != nil {
+		return err
+	}
+
 	po := composer.PortOptions{
 		ServiceName: args[0],
 		Index:       index,
 		Port:        port,
 		Protocol:    protocol,
+		DataStore:   dataStore,
+		Namespace:   globalOptions.Namespace,
 	}
 
 	return c.Port(ctx, cmd.OutOrStdout(), po)

cmd/nerdctl/compose/compose_port_linux_test.go

@@ -20,7 +20,11 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/containerd/nerdctl/mod/tigron/expect"
+	"github.com/containerd/nerdctl/mod/tigron/test"
+
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
 
 func TestComposePort(t *testing.T) {
@@ -75,3 +79,42 @@ services:
 	base.ComposeCmd("-f", comp.YAMLFullPath(), "port", "--protocol", "udp", "svc0", "10000").AssertFail()
 	base.ComposeCmd("-f", comp.YAMLFullPath(), "port", "--protocol", "tcp", "svc0", "10001").AssertFail()
 }
+
+// TestComposeMultiplePorts tests whether it is possible to allocate a large
+// number of ports. (https://github.com/containerd/nerdctl/issues/4027)
+func TestComposeMultiplePorts(t *testing.T) {
+	var dockerComposeYAML = fmt.Sprintf(`
+services:
+  svc0:
+    image: %s
+    command: "sleep infinity"
+    ports:
+    - '32000-32060:32000-32060'
+`, testutil.AlpineImage)
+
+	testCase := nerdtest.Setup()
+
+	testCase.Setup = func(data test.Data, helpers test.Helpers) {
+		compYamlPath := data.Temp().Save(dockerComposeYAML, "compose.yaml")
+		data.Labels().Set("composeYaml", compYamlPath)
+
+		helpers.Ensure("compose", "-f", compYamlPath, "up", "-d")
+	}
+
+	testCase.Cleanup = func(data test.Data, helpers test.Helpers) {
+		helpers.Anyhow("compose", "-f", data.Temp().Path("compose.yaml"), "down", "-v")
+	}
+
+	testCase.SubTests = []*test.Case{
+		{
+			Description: "Issue #4027 - Allocate a large number of ports.",
+			NoParallel:  true,
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("compose", "-f", data.Labels().Get("composeYaml"), "port", "svc0", "32000")
+			},
+			Expected: test.Expects(expect.ExitCodeSuccess, nil, expect.Contains("0.0.0.0:32000")),
+		},
+	}
+
+	testCase.Run(t)
+}

cmd/nerdctl/compose/compose_ps.go

@@ -29,9 +29,9 @@ import (
 	"github.com/containerd/containerd/v2/core/runtime/restart"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/go-cni"
-	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers"
+	"github.com/containerd/nerdctl/v2/pkg/api/types"
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
 	"github.com/containerd/nerdctl/v2/pkg/cmd/compose"
 	"github.com/containerd/nerdctl/v2/pkg/containerutil"
@@ -183,9 +183,9 @@ func psAction(cmd *cobra.Command, args []string) error {
 			var p composeContainerPrintable
 			var err error
 			if format == "json" {
-				p, err = composeContainerPrintableJSON(ctx, container)
+				p, err = composeContainerPrintableJSON(ctx, container, globalOptions)
 			} else {
-				p, err = composeContainerPrintableTab(ctx, container)
+				p, err = composeContainerPrintableTab(ctx, container, globalOptions)
 			}
 			if err != nil {
 				return err
@@ -234,7 +234,7 @@ func psAction(cmd *cobra.Command, args []string) error {
 
 // composeContainerPrintableTab constructs composeContainerPrintable with fields
 // only for console output.
-func composeContainerPrintableTab(ctx context.Context, container containerd.Container) (composeContainerPrintable, error) {
+func composeContainerPrintableTab(ctx context.Context, container containerd.Container, gOptions types.GlobalCommandOptions) (composeContainerPrintable, error) {
 	info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata)
 	if err != nil {
 		return composeContainerPrintable{}, err
@@ -251,20 +251,32 @@ func composeContainerPrintableTab(ctx context.Context, container containerd.Cont
 	if err != nil {
 		return composeContainerPrintable{}, err
 	}
+	dataStore, err := clientutil.DataStore(gOptions.DataRoot, gOptions.Address)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
+	containerLabels, err := container.Labels(ctx)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
+	ports, err := portutil.LoadPortMappings(dataStore, gOptions.Namespace, info.ID, containerLabels)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
 
 	return composeContainerPrintable{
 		Name:    info.Labels[labels.Name],
 		Image:   image.Metadata().Name,
 		Command: formatter.InspectContainerCommandTrunc(spec),
 		Service: info.Labels[labels.ComposeService],
 		State:   status,
-		Ports:   formatter.FormatPorts(info.Labels),
+		Ports:   formatter.FormatPorts(ports),
 	}, nil
 }
 
 // composeContainerPrintableJSON constructs composeContainerPrintable with fields
 // only for json output and compatible docker output.
-func composeContainerPrintableJSON(ctx context.Context, container containerd.Container) (composeContainerPrintable, error) {
+func composeContainerPrintableJSON(ctx context.Context, container containerd.Container, gOptions types.GlobalCommandOptions) (composeContainerPrintable, error) {
 	info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata)
 	if err != nil {
 		return composeContainerPrintable{}, err
@@ -294,6 +306,18 @@ func composeContainerPrintableJSON(ctx context.Context, container containerd.Con
 	if err != nil {
 		return composeContainerPrintable{}, err
 	}
+	dataStore, err := clientutil.DataStore(gOptions.DataRoot, gOptions.Address)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
+	containerLabels, err := container.Labels(ctx)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
+	portMappings, err := portutil.LoadPortMappings(dataStore, gOptions.Namespace, info.ID, containerLabels)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
 
 	return composeContainerPrintable{
 		ID:         container.ID(),
@@ -305,7 +329,7 @@ func composeContainerPrintableJSON(ctx context.Context, container containerd.Con
 		State:      state,
 		Health:     "",
 		ExitCode:   exitCode,
-		Publishers: formatPublishers(info.Labels),
+		Publishers: formatPublishers(portMappings),
 	}, nil
 }
 
@@ -321,7 +345,7 @@ type PortPublisher struct {
 
 // formatPublishers parses and returns docker-compatible []PortPublisher from
 // label map. If an error happens, an empty slice is returned.
-func formatPublishers(labelMap map[string]string) []PortPublisher {
+func formatPublishers(portMappings []cni.PortMapping) []PortPublisher {
 	mapper := func(pm cni.PortMapping) PortPublisher {
 		return PortPublisher{
 			URL:           pm.HostIP,
@@ -332,12 +356,8 @@ func formatPublishers(labelMap map[string]string) []PortPublisher {
 	}
 
 	var dockerPorts []PortPublisher
-	if portMappings, err := portutil.ParsePortsLabel(labelMap); err == nil {
-		for _, p := range portMappings {
-			dockerPorts = append(dockerPorts, mapper(p))
-		}
-	} else {
-		log.L.Error(err.Error())
+	for _, p := range portMappings {
+		dockerPorts = append(dockerPorts, mapper(p))
 	}
 	return dockerPorts
 }

cmd/nerdctl/container/container_port.go

@@ -29,6 +29,7 @@ import (
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
 	"github.com/containerd/nerdctl/v2/pkg/containerutil"
 	"github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker"
+	"github.com/containerd/nerdctl/v2/pkg/portutil"
 )
 
 func PortCommand() *cobra.Command {
@@ -81,13 +82,26 @@ func portAction(cmd *cobra.Command, args []string) error {
 	}
 	defer cancel()
 
+	dataStore, err := clientutil.DataStore(globalOptions.DataRoot, globalOptions.Address)
+	if err != nil {
+		return err
+	}
+
 	walker := &containerwalker.ContainerWalker{
 		Client: client,
 		OnFound: func(ctx context.Context, found containerwalker.Found) error {
 			if found.MatchCount > 1 {
 				return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req)
 			}
-			return containerutil.PrintHostPort(ctx, cmd.OutOrStdout(), found.Container, argPort, argProto)
+			containerLabels, err := found.Container.Labels(ctx)
+			if err != nil {
+				return err
+			}
+			ports, err := portutil.LoadPortMappings(dataStore, globalOptions.Namespace, found.Container.ID(), containerLabels)
+			if err != nil {
+				return err
+			}
+			return containerutil.PrintHostPort(ctx, cmd.OutOrStdout(), found.Container, argPort, argProto, ports)
 		},
 	}
 	req := args[0]

cmd/nerdctl/container/container_run_network_linux_test.go

@@ -36,7 +36,6 @@ import (
 
 	"github.com/containerd/containerd/v2/defaults"
 	"github.com/containerd/containerd/v2/pkg/netns"
-	"github.com/containerd/errdefs"
 	"github.com/containerd/nerdctl/mod/tigron/expect"
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
@@ -409,21 +408,21 @@ func TestRunPort(t *testing.T) {
 	baseTestRunPort(t, testutil.NginxAlpineImage, testutil.NginxAlpineIndexHTMLSnippet, true)
 }
 
-func TestRunWithInvalidPortThenCleanUp(t *testing.T) {
+func TestRunWithManyPortsThenCleanUp(t *testing.T) {
 	testCase := nerdtest.Setup()
 	// docker does not set label restriction to 4096 bytes
 	testCase.Require = require.Not(nerdtest.Docker)
 
 	testCase.SubTests = []*test.Case{
 		{
-			Description: "Run a container with invalid ports, and then clean up.",
+			Description: "Run a container with many ports, and then clean up.",
 			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
 				return helpers.Command("run", "--data-root", data.Temp().Path(), "--rm", "-p", "22200-22299:22200-22299", testutil.CommonImage)
 			},
 			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
 				return &test.Expected{
-					ExitCode: 1,
-					Errors:   []error{errdefs.ErrInvalidArgument},
+					ExitCode: 0,
+					Errors:   []error{},
 					Output: func(stdout string, t tig.T) {
 						getAddrHash := func(addr string) string {
 							const addrHashLen = 8

docs/dir.md

@@ -35,6 +35,7 @@ Files:
 - `<CID>-json.log`: used by `nerdctl logs`
 - `oci-hook.*.log`: logs of the OCI hook
 - `lifecycle.json`: used to store stateful information about the container that can only be retrieved through OCI hooks
+- `network-config.json`: used to store port mapping information for containers run with the `-p` option.
 
 ### `<DATAROOT>/<ADDRHASH>/names/<NAMESPACE>`
 e.g. `/var/lib/nerdctl/1935db59/names/default`

pkg/cmd/container/create.go

@@ -37,7 +37,6 @@ import (
 	"github.com/containerd/containerd/v2/core/containers"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/oci"
-	"github.com/containerd/go-cni"
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/pkg/annotations"
@@ -61,6 +60,7 @@ import (
 	"github.com/containerd/nerdctl/v2/pkg/mountutil"
 	"github.com/containerd/nerdctl/v2/pkg/namestore"
 	"github.com/containerd/nerdctl/v2/pkg/platformutil"
+	"github.com/containerd/nerdctl/v2/pkg/portutil"
 	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/store"
@@ -390,6 +390,11 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa
 	}
 	cOpts = append(cOpts, ilOpt)
 
+	err = portutil.GeneratePortMappingsConfig(dataStore, options.GOptions.Namespace, id, netLabelOpts.PortMappings)
+	if err != nil {
+		return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), fmt.Errorf("Error writing to network-config.json: %v", err)
+	}
+
 	opts = append(opts, propagateInternalContainerdLabelsToOCIAnnotations(),
 		oci.WithAnnotations(strutil.ConvertKVStringsToMap(options.Annotations)))
 
@@ -689,7 +694,6 @@ type internalLabels struct {
 	networks             []string
 	ipAddress            string
 	ip6Address           string
-	ports                []cni.PortMapping
 	macAddress           string
 	dnsServers           []string
 	dnsSearchDomains     []string
@@ -741,13 +745,6 @@ func withInternalLabels(internalLabels internalLabels) (containerd.NewContainerO
 		return nil, err
 	}
 	m[labels.Networks] = string(networksJSON)
-	if len(internalLabels.ports) > 0 {
-		portsJSON, err := json.Marshal(internalLabels.ports)
-		if err != nil {
-			return nil, err
-		}
-		m[labels.Ports] = string(portsJSON)
-	}
 	if internalLabels.logURI != "" {
 		m[labels.LogURI] = internalLabels.logURI
 		logConfigJSON, err := json.Marshal(internalLabels.logConfig)
@@ -909,7 +906,6 @@ func withHealthcheck(options types.ContainerCreateOptions, ensuredImage *imgutil
 func (il *internalLabels) loadNetOpts(opts types.NetworkOptions) {
 	il.hostname = opts.Hostname
 	il.domainname = opts.Domainname
-	il.ports = opts.PortMappings
 	il.ipAddress = opts.IPAddress
 	il.ip6Address = opts.IP6Address
 	il.networks = opts.NetworkSlice

pkg/cmd/container/inspect.go

@@ -25,27 +25,36 @@ import (
 	"github.com/containerd/containerd/v2/core/snapshots"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
+	"github.com/containerd/nerdctl/v2/pkg/clientutil"
 	"github.com/containerd/nerdctl/v2/pkg/containerdutil"
 	"github.com/containerd/nerdctl/v2/pkg/containerinspector"
 	"github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker"
 	"github.com/containerd/nerdctl/v2/pkg/imgutil"
 	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat"
+	"github.com/containerd/nerdctl/v2/pkg/portutil"
 )
 
 // Inspect prints detailed information for each container in `containers`.
 func Inspect(ctx context.Context, client *containerd.Client, containers []string, options types.ContainerInspectOptions) ([]any, error) {
+	dataStore, err := clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address)
+	if err != nil {
+		return []any{}, err
+	}
+
 	f := &containerInspector{
 		mode:        options.Mode,
 		size:        options.Size,
 		snapshotter: containerdutil.SnapshotService(client, options.GOptions.Snapshotter),
+		dataStore:   dataStore,
+		namespace:   options.GOptions.Namespace,
 	}
 
 	walker := &containerwalker.ContainerWalker{
 		Client:  client,
 		OnFound: f.Handler,
 	}
 
-	err := walker.WalkAll(ctx, containers, true)
+	err = walker.WalkAll(ctx, containers, true)
 	if err != nil {
 		return []any{}, err
 	}
@@ -58,6 +67,8 @@ type containerInspector struct {
 	size        bool
 	snapshotter snapshots.Snapshotter
 	entries     []interface{}
+	dataStore   string
+	namespace   string
 }
 
 func (x *containerInspector) Handler(ctx context.Context, found containerwalker.Found) error {
@@ -68,6 +79,19 @@ func (x *containerInspector) Handler(ctx context.Context, found containerwalker.
 	if err != nil {
 		return err
 	}
+
+	containerLabels, err := found.Container.Labels(ctx)
+	if err != nil {
+		return err
+	}
+	ports, err := portutil.LoadPortMappings(x.dataStore, x.namespace, n.ID, containerLabels)
+	if err != nil {
+		return err
+	}
+	if n.Process != nil && n.Process.NetNS != nil && len(ports) > 0 {
+		n.Process.NetNS.PortMappings = ports
+	}
+
 	switch x.mode {
 	case "native":
 		x.entries = append(x.entries, n)