Merge pull request #4300 from coderbirju/add-soci-convert

add soci to nerdctl image convert

Author: AkihiroSuda

cmd/nerdctl/image/image_convert.go

@@ -89,6 +89,12 @@ func convertCommand() *cobra.Command {
 	cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd")
 	// #endregion
 
+	// #region soci flags
+	cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.")
+	cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format")
+	cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans")
+	// #endregion
+
 	// #region generic flags
 	cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
 	cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 	}
 	// #endregion
 
+	// #region soci flags
+	soci, err := cmd.Flags().GetBool("soci")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	// #endregion
+
 	// #region generic flags
 	uncompress, err := cmd.Flags().GetBool("uncompress")
 	if err != nil {
@@ -277,6 +298,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 			OverlayFsType:  overlaybdFsType,
 			OverlaydbDBStr: overlaybdDbstr,
 		},
+		SociConvertOptions: types.SociConvertOptions{
+			Soci: soci,
+			SociOptions: types.SociOptions{
+				SpanSize:     sociSpanSize,
+				MinLayerSize: sociMinLayerSize,
+			},
+		},
 		Stdout: cmd.OutOrStdout(),
 	}, nil
 }

cmd/nerdctl/image/image_convert_linux_test.go

@@ -89,6 +89,24 @@ func TestImageConvert(t *testing.T) {
 				},
 				Expected: test.Expects(0, nil, nil),
 			},
+			{
+				Description: "soci",
+				Require: require.All(
+					require.Not(nerdtest.Docker),
+					nerdtest.Soci,
+					nerdtest.SociVersion("0.10.0"),
+				),
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					helpers.Anyhow("rmi", "-f", data.Identifier("converted-image"))
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("image", "convert", "--soci",
+						"--soci-span-size", "2097152",
+						"--soci-min-layer-size", "0",
+						testutil.CommonImage, data.Identifier("converted-image"))
+				},
+				Expected: test.Expects(0, nil, nil),
+			},
 		},
 	}
 

docs/command-reference.md

@@ -979,6 +979,11 @@ Flags:
 - `--oci`                              : convert Docker media types to OCI media types
 - `--platform=<PLATFORM>`              : convert content for a specific platform
 - `--all-platforms`                    : convert content for all platforms (default: false)
+- `--soci`                             : convert content to SOCI image manifest v2
+*[**Note**: soci convert uses the default platform if nothing is specified. --platform flag can be used to specify a platform]*
+- `--soci-span-size` : Span size in bytes that soci index uses to segment layer data. Default is 4 MiB.
+- `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB.
+
 
 ### :nerd_face: nerdctl image encrypt
 

docs/soci.md

@@ -4,6 +4,22 @@ SOCI Snapshotter is a containerd snapshotter plugin. It enables standard OCI ima
 
 See https://github.com/awslabs/soci-snapshotter to learn further information.
 
+## SOCI Index Manifest Versions
+
+SOCI supports two index manifest versions:
+
+- **v1**: Original format using OCI Referrers API (disabled by default in SOCI v0.10.0+)
+- **v2**: New format that packages SOCI index with the image (default in SOCI v0.10.0+)
+
+To enable v1 indices in SOCI v0.10.0+, add to `/etc/soci-snapshotter-grpc/config.toml`:
+```toml
+[pull_modes]
+  [pull_modes.soci_v1]
+    enable = true
+```
+
+For detailed information about the differences between v1 and v2, see the [SOCI Index Manifest v2 documentation](https://github.com/awslabs/soci-snapshotter/blob/main/docs/soci-index-manifest-v2.md).
+
 ## Prerequisites
 
 - Install containerd remote snapshotter plugin (`soci-snapshotter-grpc`) from https://github.com/awslabs/soci-snapshotter/blob/main/docs/getting-started.md
@@ -45,3 +61,21 @@ For images that already have SOCI indices, see https://gallery.ecr.aws/soci-work
 nerdctl push --snapshotter=soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest
 ```
 --soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.
+
+> **Note**: With SOCI v0.10.0+, When using `nerdctl push --snapshotter=soci`, it creates and pushes v1 indices. When pushing a converted image (created with `nerdctl image convert --soci`), it will push v2 indices.
+
+## Enable SOCI for `nerdctl image convert`
+
+| :zap: Requirement | nerdctl >= 2.1.3 |
+| ----------------- | ---------------- |
+
+| :zap: Requirement | soci-snapshotter >= 0.10.0 |
+| ----------------- | ---------------- |
+
+- Convert an image to generate SOCI Index artifacts v2. Running the `nerdctl image convert` with the `--soci` flag and a `srcImg` and `dstImg`, `nerdctl` will create the SOCI v2 indices and the new image will be present in the `dstImg` address.
+```console
+nerdctl image convert --soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest public.ecr.aws/my-registry/my-repo:soci
+```
+--soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.
+
+The `image convert` command with `--soci` flag creates SOCI-enabled images using SOCI Index Manifest v2, which combines the SOCI index and the original image into a single artifact.

pkg/api/types/image_types.go

@@ -19,7 +19,7 @@ package types
 import (
 	"io"
 
-	"github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageListOptions specifies options for `nerdctl image list`.
@@ -73,6 +73,7 @@ type ImageConvertOptions struct {
 	ZstdChunkedOptions
 	NydusOptions
 	OverlaybdOptions
+	SociConvertOptions
 }
 
 // EstargzOptions contains eStargz conversion options
@@ -135,6 +136,15 @@ type OverlaybdOptions struct {
 	OverlayFsType string
 	// OverlaydbDBStr database config string for overlaybd
 	OverlaydbDBStr string
+	// #endregion
+}
+
+type SociConvertOptions struct {
+	// Soci convert image to SOCI format.
+	Soci bool
+	// SociOptions contains SOCI-specific options
+	SociOptions SociOptions
+	// #endregion
 }
 
 // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`.
@@ -211,7 +221,7 @@ type ImagePullOptions struct {
 	// If nil, it will unpack automatically if only 1 platform is specified.
 	Unpack *bool
 	// Content for specific platforms. Empty if `--all-platforms` is true
-	OCISpecPlatform []v1.Platform
+	OCISpecPlatform []ocispec.Platform
 	// Pull mode
 	Mode string
 	// Suppress verbose output

pkg/cmd/image/convert.go

@@ -47,6 +47,7 @@ import (
 	converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter"
 	"github.com/containerd/nerdctl/v2/pkg/platformutil"
 	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
+	"github.com/containerd/nerdctl/v2/pkg/snapshotterutil"
 )
 
 func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error {
@@ -86,8 +87,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 	zstdchunked := options.ZstdChunked
 	overlaybd := options.Overlaybd
 	nydus := options.Nydus
+	soci := options.Soci
 	var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
-	if estargz || zstd || zstdchunked || overlaybd || nydus {
+	if estargz || zstd || zstdchunked || overlaybd || nydus || soci {
 		convertCount := 0
 		if estargz {
 			convertCount++
@@ -104,9 +106,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 		if nydus {
 			convertCount++
 		}
+		if soci {
+			convertCount++
+		}
 
 		if convertCount > 1 {
-			return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used")
+			return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used")
 		}
 
 		var convertFunc converter.ConvertFunc
@@ -164,6 +169,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 				)),
 			)
 			convertType = "nydus"
+		case soci:
+			// Convert image to SOCI format
+			convertedRef, err := snapshotterutil.ConvertSociIndexV2(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions)
+			if err != nil {
+				return fmt.Errorf("failed to convert image to SOCI format: %w", err)
+			}
+			res := converterutil.ConvertedImageInfo{
+				Image: convertedRef,
+			}
+			return printConvertedImage(options.Stdout, options, res)
 		}
 
 		if convertType != "overlaybd" {

pkg/cmd/image/push.go

@@ -210,7 +210,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		return err
 	}
 	if options.GOptions.Snapshotter == "soci" {
-		if err = snapshotterutil.CreateSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil {
+		if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil {
 			return err
 		}
 		if err = snapshotterutil.PushSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms); err != nil {