add soci convert feature

Signed-off-by: Arjun Raja Yogidas <[email protected]>

Author: coderbirju

cmd/nerdctl/image/image_convert.go

@@ -89,6 +89,12 @@ func convertCommand() *cobra.Command {
 	cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd")
 	// #endregion
 
+	// #region soci flags
+	cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.")
+	cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format")
+	cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans")
+	// #endregion
+
 	// #region generic flags
 	cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
 	cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 	}
 	// #endregion
 
+	// #region soci flags
+	soci, err := cmd.Flags().GetBool("soci")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	// #endregion
+
 	// #region generic flags
 	uncompress, err := cmd.Flags().GetBool("uncompress")
 	if err != nil {
@@ -268,6 +289,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 		OverlayFsType:  overlaybdFsType,
 		OverlaydbDBStr: overlaybdDbstr,
 		// #endregion
+		// #region soci flags
+		Soci: soci,
+		SociOptions: types.SociOptions{
+			SpanSize:     sociSpanSize,
+			MinLayerSize: sociMinLayerSize,
+		},
+		// #endregion
 		// #region generic flags
 		Uncompress: uncompress,
 		Oci:        oci,

cmd/nerdctl/image/image_convert_linux_test.go

@@ -88,6 +88,24 @@ func TestImageConvert(t *testing.T) {
 				},
 				Expected: test.Expects(0, nil, nil),
 			},
+			{
+				Description: "soci",
+				Require: require.All(
+					require.Not(nerdtest.Docker),
+					nerdtest.Soci,
+					nerdtest.SociVersion("0.10.0"),
+				),
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					helpers.Anyhow("rmi", "-f", data.Identifier("converted-image"))
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("image", "convert", "--soci",
+						"--soci-span-size", "2097152",
+						"--soci-min-layer-size", "20971520",
+						testutil.CommonImage, data.Identifier("converted-image"))
+				},
+				Expected: test.Expects(0, nil, nil),
+			},
 		},
 	}
 

docs/command-reference.md

@@ -960,6 +960,11 @@ Flags:
 - `--oci`                              : convert Docker media types to OCI media types
 - `--platform=<PLATFORM>`              : convert content for a specific platform
 - `--all-platforms`                    : convert content for all platforms (default: false)
+- `--soci`: generate SOCI v2 Indices to oci images.
+*[**Note**: content is converted for all platforms by default when using this flag, use the `--platorm` flag to limit this behavior]*
+- `--soci-min-layer-size` : Span size in bytes that soci index uses to segment layer data. Default is 4 MiB.
+- `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB.
+
 
 ### :nerd_face: nerdctl image encrypt
 

docs/soci.md

@@ -45,3 +45,18 @@ For images that already have SOCI indices, see https://gallery.ecr.aws/soci-work
 nerdctl push --snapshotter=soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest
 ```
 --soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.
+
+
+## Enable SOCI for `nerdctl image convert`
+
+| :zap: Requirement | nerdctl >= 2.2.0 |
+| ----------------- | ---------------- |
+
+| :zap: Requirement | soci-snapshotter >= 0.10.0 |
+| ----------------- | ---------------- |
+
+- Convert an image to generate SOCI Index artifacts v2. Running the `nerdctl image convert` with the `--soci` flag and a `srcImg` and `dstImg`, `nerdctl` will create the SOCI v2 indices and the new image will be present in the `dstImg` address.
+```console
+nerdctl image convert --soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest public.ecr.aws/my-registry/my-repo:soci
+```
+--soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.

pkg/api/types/image_types.go

@@ -19,7 +19,7 @@ package types
 import (
 	"io"
 
-	"github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageListOptions specifies options for `nerdctl image list`.
@@ -124,6 +124,12 @@ type ImageConvertOptions struct {
 	OverlaydbDBStr string
 	// #endregion
 
+	// #region soci flags
+	// Soci convert image to SOCI format.eiifc
+	Soci bool
+	// SociOptions contains SOCI-specific options
+	SociOptions SociOptions
+	// #endregion
 }
 
 // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`.
@@ -200,7 +206,7 @@ type ImagePullOptions struct {
 	// If nil, it will unpack automatically if only 1 platform is specified.
 	Unpack *bool
 	// Content for specific platforms. Empty if `--all-platforms` is true
-	OCISpecPlatform []v1.Platform
+	OCISpecPlatform []ocispec.Platform
 	// Pull mode
 	Mode string
 	// Suppress verbose output

pkg/cmd/image/convert.go

@@ -47,6 +47,7 @@ import (
 	converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter"
 	"github.com/containerd/nerdctl/v2/pkg/platformutil"
 	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
+	"github.com/containerd/nerdctl/v2/pkg/snapshotterutil"
 )
 
 func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error {
@@ -86,8 +87,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 	zstdchunked := options.ZstdChunked
 	overlaybd := options.Overlaybd
 	nydus := options.Nydus
+	soci := options.Soci
 	var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
-	if estargz || zstd || zstdchunked || overlaybd || nydus {
+	if estargz || zstd || zstdchunked || overlaybd || nydus || soci {
 		convertCount := 0
 		if estargz {
 			convertCount++
@@ -104,9 +106,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 		if nydus {
 			convertCount++
 		}
+		if soci {
+			convertCount++
+		}
 
 		if convertCount > 1 {
-			return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used")
+			return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used")
 		}
 
 		var convertFunc converter.ConvertFunc
@@ -164,6 +169,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 				)),
 			)
 			convertType = "nydus"
+		case soci:
+			// Convert image to SOCI format
+			convertedRef, err := snapshotterutil.ConvertSociIndexV2(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions)
+			if err != nil {
+				return fmt.Errorf("failed to convert image to SOCI format: %w", err)
+			}
+			res := converterutil.ConvertedImageInfo{
+				Image: convertedRef,
+			}
+			return printConvertedImage(options.Stdout, options, res)
 		}
 
 		if convertType != "overlaybd" {

pkg/cmd/image/push.go

@@ -209,7 +209,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		return err
 	}
 	if options.GOptions.Snapshotter == "soci" {
-		if err = snapshotterutil.CreateSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil {
+		if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil {
 			return err
 		}
 		if err = snapshotterutil.PushSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms); err != nil {

pkg/snapshotterutil/sociutil.go

@@ -18,23 +18,26 @@ package snapshotterutil
 
 import (
 	"bufio"
+	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"strconv"
 	"strings"
 
+	"github.com/containerd/containerd/v2/client"
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
 )
 
-// CreateSoci creates a SOCI index(`rawRef`)
-func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error {
+// setupSociCommand creates and sets up a SOCI command with common configuration
+func setupSociCommand(gOpts types.GlobalCommandOptions) (*exec.Cmd, error) {
 	sociExecutable, err := exec.LookPath("soci")
 	if err != nil {
 		log.L.WithError(err).Error("soci executable not found in path $PATH")
 		log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md")
-		return err
+		return nil, err
 	}
 
 	sociCmd := exec.Command(sociExecutable)
@@ -47,7 +50,65 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 	if gOpts.Namespace != "" {
 		sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace)
 	}
-	// #endregion
+
+	return sociCmd, nil
+}
+
+// ConvertSociIndexV2 converts an image to SOCI format and returns the converted image reference with digest
+func ConvertSociIndexV2(ctx context.Context, client *client.Client, srcRef string, destRef string, gOpts types.GlobalCommandOptions, platforms []string, sOpts types.SociOptions) (string, error) {
+	sociCmd, err := setupSociCommand(gOpts)
+	if err != nil {
+		return "", err
+	}
+
+	// TODO: Implement conversion logic
+	sociCmd.Args = append(sociCmd.Args, "convert")
+
+	if len(platforms) > 0 {
+		// multiple values need to be passed as separate, repeating flags in soci as it uses urfave
+		// https://github.com/urfave/cli/blob/main/docs/v2/examples/flags.md#multiple-values-per-single-flag
+		for _, p := range platforms {
+			sociCmd.Args = append(sociCmd.Args, "--platform", p)
+		}
+	}
+
+	if sOpts.SpanSize != -1 {
+		sociCmd.Args = append(sociCmd.Args, "--span-size", strconv.FormatInt(sOpts.SpanSize, 10))
+	}
+
+	if sOpts.MinLayerSize != -1 {
+		sociCmd.Args = append(sociCmd.Args, "--min-layer-size", strconv.FormatInt(sOpts.MinLayerSize, 10))
+	}
+
+	sociCmd.Args = append(sociCmd.Args, srcRef, destRef)
+
+	log.L.Infof("Converting image from %s to %s using SOCI format", srcRef, destRef)
+
+	err = processSociIO(sociCmd)
+	if err != nil {
+		return "", err
+	}
+	err = sociCmd.Wait()
+	if err != nil {
+		return "", err
+	}
+
+	// Get the converted image's digest
+	img, err := client.GetImage(ctx, destRef)
+	if err != nil {
+		return "", fmt.Errorf("failed to get converted image: %w", err)
+	}
+
+	// Return the full reference with digest
+	return fmt.Sprintf("%s@%s", destRef, img.Target().Digest), nil
+}
+
+// CreateSociIndexV1 creates a SOCI index(`rawRef`)
+func CreateSociIndexV1(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error {
+	sociCmd, err := setupSociCommand(gOpts)
+	if err != nil {
+		return err
+	}
 
 	// Global flags have to be put before subcommand before soci upgrades to urfave v3.
 	// https://github.com/urfave/cli/issues/1113
@@ -73,7 +134,7 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 	// --timeout, --debug, --content-store
 	sociCmd.Args = append(sociCmd.Args, rawRef)
 
-	log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args)
+	log.L.Debugf("running soci %v", sociCmd.Args)
 
 	err = processSociIO(sociCmd)
 	if err != nil {
@@ -88,25 +149,11 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string) error {
 	log.L.Debugf("pushing SOCI index: %s", rawRef)
 
-	sociExecutable, err := exec.LookPath("soci")
+	sociCmd, err := setupSociCommand(gOpts)
 	if err != nil {
-		log.L.WithError(err).Error("soci executable not found in path $PATH")
-		log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md")
 		return err
 	}
 
-	sociCmd := exec.Command(sociExecutable)
-	sociCmd.Env = os.Environ()
-
-	// #region for global flags.
-	if gOpts.Address != "" {
-		sociCmd.Args = append(sociCmd.Args, "--address", gOpts.Address)
-	}
-	if gOpts.Namespace != "" {
-		sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace)
-	}
-	// #endregion
-
 	// Global flags have to be put before subcommand before soci upgrades to urfave v3.
 	// https://github.com/urfave/cli/issues/1113
 	sociCmd.Args = append(sociCmd.Args, "push")
@@ -131,7 +178,7 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool,
 	}
 	sociCmd.Args = append(sociCmd.Args, rawRef)
 
-	log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args)
+	log.L.Debugf("running soci %v", sociCmd.Args)
 
 	err = processSociIO(sociCmd)
 	if err != nil {

pkg/testutil/nerdtest/requirements.go

@@ -419,6 +419,53 @@ var RemapIDs = &test.Requirement{
 	},
 }
 
+// SociVersion returns a requirement that checks if the installed SOCI version
+// meets the minimum required version
+func SociVersion(minVersion string) *test.Requirement {
+	return &test.Requirement{
+		Check: func(data test.Data, helpers test.Helpers) (bool, string) {
+			sociExecutable, err := exec.LookPath("soci")
+			if err != nil {
+				return false, fmt.Sprintf("soci executable not found in path $PATH: %v", err)
+			}
+
+			cmd := exec.Command(sociExecutable, "--version")
+			output, err := cmd.Output()
+			if err != nil {
+				return false, fmt.Sprintf("failed to get soci version: %v", err)
+			}
+
+			// Parse version from output
+			// Example output format: "soci version v0.9.0 737f61a3db40c386f997c1f126344158aa3ad43c"
+			versionStr := strings.TrimSpace(string(output))
+			parts := strings.Fields(versionStr)
+			if len(parts) < 3 {
+				return false, fmt.Sprintf("unexpected soci version output format: %s", versionStr)
+			}
+
+			// Extract version number without 'v' prefix
+			installedVersion := strings.TrimPrefix(parts[2], "v")
+
+			// Compare versions
+			v1, err := semver.NewVersion(installedVersion)
+			if err != nil {
+				return false, fmt.Sprintf("failed to parse installed version %s: %v", installedVersion, err)
+			}
+
+			v2, err := semver.NewVersion(minVersion)
+			if err != nil {
+				return false, fmt.Sprintf("failed to parse minimum required version %s: %v", minVersion, err)
+			}
+
+			if v1.LessThan(v2) {
+				return false, fmt.Sprintf("installed soci version %s is older than required version %s", installedVersion, minVersion)
+			}
+
+			return true, fmt.Sprintf("soci version %s meets minimum requirement %s", installedVersion, minVersion)
+		},
+	}
+}
+
 func ContainerdVersion(v string) *test.Requirement {
 	return &test.Requirement{
 		Check: func(data test.Data, helpers test.Helpers) (bool, string) {