add soci convert feature

Signed-off-by: Arjun Raja Yogidas <[email protected]>

Author: coderbirju

cmd/nerdctl/image/image_convert.go

@@ -89,6 +89,12 @@ func convertCommand() *cobra.Command {
 	cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd")
 	// #endregion
 
+	// #region soci flags
+	cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.")
+	cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format")
+	cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans")
+	// #endregion
+
 	// #region generic flags
 	cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
 	cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 	}
 	// #endregion
 
+	// #region soci flags
+	soci, err := cmd.Flags().GetBool("soci")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	// #endregion
+
 	// #region generic flags
 	uncompress, err := cmd.Flags().GetBool("uncompress")
 	if err != nil {
@@ -277,6 +298,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 			OverlayFsType:  overlaybdFsType,
 			OverlaydbDBStr: overlaybdDbstr,
 		},
+		SociConvertOptions: types.SociConvertOptions{
+			Soci: soci,
+			SociOptions: types.SociOptions{
+				SpanSize:     sociSpanSize,
+				MinLayerSize: sociMinLayerSize,
+			},
+		},
 		Stdout: cmd.OutOrStdout(),
 	}, nil
 }

cmd/nerdctl/image/image_convert_linux_test.go

@@ -89,6 +89,24 @@ func TestImageConvert(t *testing.T) {
 				},
 				Expected: test.Expects(0, nil, nil),
 			},
+			{
+				Description: "soci",
+				Require: require.All(
+					require.Not(nerdtest.Docker),
+					nerdtest.Soci,
+					nerdtest.SociVersion("0.10.0"),
+				),
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					helpers.Anyhow("rmi", "-f", data.Identifier("converted-image"))
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("image", "convert", "--soci",
+						"--soci-span-size", "2097152",
+						"--soci-min-layer-size", "20971520",
+						testutil.CommonImage, data.Identifier("converted-image"))
+				},
+				Expected: test.Expects(0, nil, nil),
+			},
 		},
 	}
 

docs/command-reference.md

@@ -979,6 +979,11 @@ Flags:
 - `--oci`                              : convert Docker media types to OCI media types
 - `--platform=<PLATFORM>`              : convert content for a specific platform
 - `--all-platforms`                    : convert content for all platforms (default: false)
+- `--soci`                             : generate SOCI v2 Indices to oci images.
+*[**Note**: content is converted for all platforms by default when using this flag, use the `--platorm` flag to limit this behavior]*
+- `--soci-span-size` : Span size in bytes that soci index uses to segment layer data. Default is 4 MiB.
+- `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB.
+
 
 ### :nerd_face: nerdctl image encrypt
 

docs/soci.md

@@ -45,3 +45,18 @@ For images that already have SOCI indices, see https://gallery.ecr.aws/soci-work
 nerdctl push --snapshotter=soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest
 ```
 --soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.
+
+
+## Enable SOCI for `nerdctl image convert`
+
+| :zap: Requirement | nerdctl >= 2.2.0 |
+| ----------------- | ---------------- |
+
+| :zap: Requirement | soci-snapshotter >= 0.10.0 |
+| ----------------- | ---------------- |
+
+- Convert an image to generate SOCI Index artifacts v2. Running the `nerdctl image convert` with the `--soci` flag and a `srcImg` and `dstImg`, `nerdctl` will create the SOCI v2 indices and the new image will be present in the `dstImg` address.
+```console
+nerdctl image convert --soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest public.ecr.aws/my-registry/my-repo:soci
+```
+--soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.

pkg/api/types/image_types.go

@@ -19,7 +19,7 @@ package types
 import (
 	"io"
 
-	"github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageListOptions specifies options for `nerdctl image list`.
@@ -73,6 +73,7 @@ type ImageConvertOptions struct {
 	ZstdChunkedOptions
 	NydusOptions
 	OverlaybdOptions
+	SociConvertOptions
 }
 
 // EstargzOptions contains eStargz conversion options
@@ -135,6 +136,15 @@ type OverlaybdOptions struct {
 	OverlayFsType string
 	// OverlaydbDBStr database config string for overlaybd
 	OverlaydbDBStr string
+	// #endregion
+}
+
+type SociConvertOptions struct {
+	// Soci convert image to SOCI format.
+	Soci bool
+	// SociOptions contains SOCI-specific options
+	SociOptions SociOptions
+	// #endregion
 }
 
 // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`.
@@ -211,7 +221,7 @@ type ImagePullOptions struct {
 	// If nil, it will unpack automatically if only 1 platform is specified.
 	Unpack *bool
 	// Content for specific platforms. Empty if `--all-platforms` is true
-	OCISpecPlatform []v1.Platform
+	OCISpecPlatform []ocispec.Platform
 	// Pull mode
 	Mode string
 	// Suppress verbose output

pkg/cmd/image/convert.go

@@ -47,6 +47,7 @@ import (
 	converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter"
 	"github.com/containerd/nerdctl/v2/pkg/platformutil"
 	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
+	"github.com/containerd/nerdctl/v2/pkg/snapshotterutil"
 )
 
 func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error {
@@ -86,8 +87,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 	zstdchunked := options.ZstdChunked
 	overlaybd := options.Overlaybd
 	nydus := options.Nydus
+	soci := options.Soci
 	var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
-	if estargz || zstd || zstdchunked || overlaybd || nydus {
+	if estargz || zstd || zstdchunked || overlaybd || nydus || soci {
 		convertCount := 0
 		if estargz {
 			convertCount++
@@ -104,9 +106,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 		if nydus {
 			convertCount++
 		}
+		if soci {
+			convertCount++
+		}
 
 		if convertCount > 1 {
-			return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used")
+			return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used")
 		}
 
 		var convertFunc converter.ConvertFunc
@@ -164,6 +169,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 				)),
 			)
 			convertType = "nydus"
+		case soci:
+			// Convert image to SOCI format
+			convertedRef, err := snapshotterutil.ConvertSociIndexV2(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions)
+			if err != nil {
+				return fmt.Errorf("failed to convert image to SOCI format: %w", err)
+			}
+			res := converterutil.ConvertedImageInfo{
+				Image: convertedRef,
+			}
+			return printConvertedImage(options.Stdout, options, res)
 		}
 
 		if convertType != "overlaybd" {

pkg/cmd/image/push.go

@@ -210,7 +210,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		return err
 	}
 	if options.GOptions.Snapshotter == "soci" {
-		if err = snapshotterutil.CreateSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil {
+		if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil {
 			return err
 		}
 		if err = snapshotterutil.PushSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms); err != nil {

pkg/snapshotterutil/sociutil.go

@@ -18,23 +18,26 @@ package snapshotterutil
 
 import (
 	"bufio"
+	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"strconv"
 	"strings"
 
+	"github.com/containerd/containerd/v2/client"
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
 )
 
-// CreateSoci creates a SOCI index(`rawRef`)
-func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error {
+// setupSociCommand creates and sets up a SOCI command with common configuration
+func setupSociCommand(gOpts types.GlobalCommandOptions) (*exec.Cmd, error) {
 	sociExecutable, err := exec.LookPath("soci")
 	if err != nil {
 		log.L.WithError(err).Error("soci executable not found in path $PATH")
 		log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md")
-		return err
+		return nil, err
 	}
 
 	sociCmd := exec.Command(sociExecutable)
@@ -47,7 +50,64 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 	if gOpts.Namespace != "" {
 		sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace)
 	}
-	// #endregion
+
+	return sociCmd, nil
+}
+
+// ConvertSociIndexV2 converts an image to SOCI format and returns the converted image reference with digest
+func ConvertSociIndexV2(ctx context.Context, client *client.Client, srcRef string, destRef string, gOpts types.GlobalCommandOptions, platforms []string, sOpts types.SociOptions) (string, error) {
+	sociCmd, err := setupSociCommand(gOpts)
+	if err != nil {
+		return "", err
+	}
+
+	sociCmd.Args = append(sociCmd.Args, "convert")
+
+	if len(platforms) > 0 {
+		// multiple values need to be passed as separate, repeating flags in soci as it uses urfave
+		// https://github.com/urfave/cli/blob/main/docs/v2/examples/flags.md#multiple-values-per-single-flag
+		for _, p := range platforms {
+			sociCmd.Args = append(sociCmd.Args, "--platform", p)
+		}
+	}
+
+	if sOpts.SpanSize != -1 {
+		sociCmd.Args = append(sociCmd.Args, "--span-size", strconv.FormatInt(sOpts.SpanSize, 10))
+	}
+
+	if sOpts.MinLayerSize != -1 {
+		sociCmd.Args = append(sociCmd.Args, "--min-layer-size", strconv.FormatInt(sOpts.MinLayerSize, 10))
+	}
+
+	sociCmd.Args = append(sociCmd.Args, srcRef, destRef)
+
+	log.L.Infof("Converting image from %s to %s using SOCI format", srcRef, destRef)
+
+	err = processSociIO(sociCmd)
+	if err != nil {
+		return "", err
+	}
+	err = sociCmd.Wait()
+	if err != nil {
+		return "", err
+	}
+
+	// Get the converted image's digest
+	img, err := client.GetImage(ctx, destRef)
+	if err != nil {
+		return "", fmt.Errorf("failed to get converted image: %w", err)
+	}
+
+	// Return the full reference with digest
+	return fmt.Sprintf("%s@%s", destRef, img.Target().Digest), nil
+}
+
+// CreateSociIndexV1 creates a SOCI index(`rawRef`)
+func CreateSociIndexV1(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error {
+	sociCmd, err := setupSociCommand(gOpts)
+	if err != nil {
+		return err
+	}
 
 	// Global flags have to be put before subcommand before soci upgrades to urfave v3.
 	// https://github.com/urfave/cli/issues/1113
@@ -73,7 +133,7 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 	// --timeout, --debug, --content-store
 	sociCmd.Args = append(sociCmd.Args, rawRef)
 
-	log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args)
+	log.L.Debugf("running soci %v", sociCmd.Args)
 
 	err = processSociIO(sociCmd)
 	if err != nil {
@@ -88,25 +148,11 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string) error {
 	log.L.Debugf("pushing SOCI index: %s", rawRef)
 
-	sociExecutable, err := exec.LookPath("soci")
+	sociCmd, err := setupSociCommand(gOpts)
 	if err != nil {
-		log.L.WithError(err).Error("soci executable not found in path $PATH")
-		log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md")
 		return err
 	}
 
-	sociCmd := exec.Command(sociExecutable)
-	sociCmd.Env = os.Environ()
-
-	// #region for global flags.
-	if gOpts.Address != "" {
-		sociCmd.Args = append(sociCmd.Args, "--address", gOpts.Address)
-	}
-	if gOpts.Namespace != "" {
-		sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace)
-	}
-	// #endregion
-
 	// Global flags have to be put before subcommand before soci upgrades to urfave v3.
 	// https://github.com/urfave/cli/issues/1113
 	sociCmd.Args = append(sociCmd.Args, "push")
@@ -131,7 +177,7 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool,
 	}
 	sociCmd.Args = append(sociCmd.Args, rawRef)
 
-	log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args)
+	log.L.Debugf("running soci %v", sociCmd.Args)
 
 	err = processSociIO(sociCmd)
 	if err != nil {