diff --git a/cmd/nerdctl/image/image_convert.go b/cmd/nerdctl/image/image_convert.go index ff7caade58d..49496f09ee7 100644 --- a/cmd/nerdctl/image/image_convert.go +++ b/cmd/nerdctl/image/image_convert.go @@ -89,6 +89,12 @@ func convertCommand() *cobra.Command { cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd") // #endregion + // #region soci flags + cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.") + cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format") + cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans") + // #endregion + // #region generic flags cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers") cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types") @@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) { } // #endregion + // #region soci flags + soci, err := cmd.Flags().GetBool("soci") + if err != nil { + return types.ImageConvertOptions{}, err + } + sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size") + if err != nil { + return types.ImageConvertOptions{}, err + } + sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size") + if err != nil { + return types.ImageConvertOptions{}, err + } + // #endregion + // #region generic flags uncompress, err := cmd.Flags().GetBool("uncompress") if err != nil { @@ -277,6 +298,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) { OverlayFsType: overlaybdFsType, OverlaydbDBStr: overlaybdDbstr, }, + SociConvertOptions: types.SociConvertOptions{ + Soci: soci, + SociOptions: types.SociOptions{ + SpanSize: sociSpanSize, + MinLayerSize: sociMinLayerSize, + }, + }, Stdout: cmd.OutOrStdout(), }, nil } diff --git a/cmd/nerdctl/image/image_convert_linux_test.go b/cmd/nerdctl/image/image_convert_linux_test.go index e514300aede..be24918fb31 100644 --- a/cmd/nerdctl/image/image_convert_linux_test.go +++ b/cmd/nerdctl/image/image_convert_linux_test.go @@ -89,6 +89,24 @@ func TestImageConvert(t *testing.T) { }, Expected: test.Expects(0, nil, nil), }, + { + Description: "soci", + Require: require.All( + require.Not(nerdtest.Docker), + nerdtest.Soci, + nerdtest.SociVersion("0.10.0"), + ), + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rmi", "-f", data.Identifier("converted-image")) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("image", "convert", "--soci", + "--soci-span-size", "2097152", + "--soci-min-layer-size", "0", + testutil.CommonImage, data.Identifier("converted-image")) + }, + Expected: test.Expects(0, nil, nil), + }, }, } diff --git a/docs/command-reference.md b/docs/command-reference.md index 09bbef2fb89..3019be9d541 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -979,6 +979,11 @@ Flags: - `--oci` : convert Docker media types to OCI media types - `--platform=` : convert content for a specific platform - `--all-platforms` : convert content for all platforms (default: false) +- `--soci` : generate SOCI v2 Indices to oci images. +*[**Note**: content is converted for all platforms by default when using this flag, use the `--platorm` flag to limit this behavior]* +- `--soci-span-size` : Span size in bytes that soci index uses to segment layer data. Default is 4 MiB. +- `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB. + ### :nerd_face: nerdctl image encrypt diff --git a/docs/soci.md b/docs/soci.md index 67fbe92f584..209494566ff 100644 --- a/docs/soci.md +++ b/docs/soci.md @@ -4,6 +4,22 @@ SOCI Snapshotter is a containerd snapshotter plugin. It enables standard OCI ima See https://github.com/awslabs/soci-snapshotter to learn further information. +## SOCI Index Manifest Versions + +SOCI supports two index manifest versions: + +- **v1**: Original format using OCI Referrers API (disabled by default in SOCI v0.10.0+) +- **v2**: New format that packages SOCI index with the image (default in SOCI v0.10.0+) + +To enable v1 indices in SOCI v0.10.0+, add to `/etc/soci-snapshotter-grpc/config.toml`: +```toml +[pull_modes] + [pull_modes.soci_v1] + enable = true +``` + +For detailed information about the differences between v1 and v2, see the [SOCI Index Manifest v2 documentation](https://github.com/awslabs/soci-snapshotter/blob/main/docs/soci-index-manifest-v2.md). + ## Prerequisites - Install containerd remote snapshotter plugin (`soci-snapshotter-grpc`) from https://github.com/awslabs/soci-snapshotter/blob/main/docs/getting-started.md @@ -45,3 +61,22 @@ For images that already have SOCI indices, see https://gallery.ecr.aws/soci-work nerdctl push --snapshotter=soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest ``` --soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details. + +> **Note**: With SOCI v0.10.0+, `nerdctl push` creates and pushes v2 indices by default. For v1 indices, enable them in the snapshotter config as described in the section above. + + +## Enable SOCI for `nerdctl image convert` + +| :zap: Requirement | nerdctl >= 2.1.3 | +| ----------------- | ---------------- | + +| :zap: Requirement | soci-snapshotter >= 0.10.0 | +| ----------------- | ---------------- | + +- Convert an image to generate SOCI Index artifacts v2. Running the `nerdctl image convert` with the `--soci` flag and a `srcImg` and `dstImg`, `nerdctl` will create the SOCI v2 indices and the new image will be present in the `dstImg` address. +```console +nerdctl image convert --soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest public.ecr.aws/my-registry/my-repo:soci +``` +--soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details. + +The `image convert` command with `--soci` flag creates SOCI-enabled images using SOCI Index Manifest v2, which combines the SOCI index and the original image into a single artifact. diff --git a/pkg/api/types/image_types.go b/pkg/api/types/image_types.go index ddc08facf68..5ff507ccc7c 100644 --- a/pkg/api/types/image_types.go +++ b/pkg/api/types/image_types.go @@ -19,7 +19,7 @@ package types import ( "io" - "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // ImageListOptions specifies options for `nerdctl image list`. @@ -73,6 +73,7 @@ type ImageConvertOptions struct { ZstdChunkedOptions NydusOptions OverlaybdOptions + SociConvertOptions } // EstargzOptions contains eStargz conversion options @@ -135,6 +136,15 @@ type OverlaybdOptions struct { OverlayFsType string // OverlaydbDBStr database config string for overlaybd OverlaydbDBStr string + // #endregion +} + +type SociConvertOptions struct { + // Soci convert image to SOCI format. + Soci bool + // SociOptions contains SOCI-specific options + SociOptions SociOptions + // #endregion } // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`. @@ -211,7 +221,7 @@ type ImagePullOptions struct { // If nil, it will unpack automatically if only 1 platform is specified. Unpack *bool // Content for specific platforms. Empty if `--all-platforms` is true - OCISpecPlatform []v1.Platform + OCISpecPlatform []ocispec.Platform // Pull mode Mode string // Suppress verbose output diff --git a/pkg/cmd/image/convert.go b/pkg/cmd/image/convert.go index b7963ea2702..12a2040d598 100644 --- a/pkg/cmd/image/convert.go +++ b/pkg/cmd/image/convert.go @@ -47,6 +47,7 @@ import ( converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter" "github.com/containerd/nerdctl/v2/pkg/platformutil" "github.com/containerd/nerdctl/v2/pkg/referenceutil" + "github.com/containerd/nerdctl/v2/pkg/snapshotterutil" ) func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error { @@ -86,8 +87,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa zstdchunked := options.ZstdChunked overlaybd := options.Overlaybd nydus := options.Nydus + soci := options.Soci var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error) - if estargz || zstd || zstdchunked || overlaybd || nydus { + if estargz || zstd || zstdchunked || overlaybd || nydus || soci { convertCount := 0 if estargz { convertCount++ @@ -104,9 +106,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa if nydus { convertCount++ } + if soci { + convertCount++ + } if convertCount > 1 { - return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used") + return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used") } var convertFunc converter.ConvertFunc @@ -164,6 +169,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa )), ) convertType = "nydus" + case soci: + // Convert image to SOCI format + convertedRef, err := snapshotterutil.ConvertSociIndexV2(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions) + if err != nil { + return fmt.Errorf("failed to convert image to SOCI format: %w", err) + } + res := converterutil.ConvertedImageInfo{ + Image: convertedRef, + } + return printConvertedImage(options.Stdout, options, res) } if convertType != "overlaybd" { diff --git a/pkg/cmd/image/push.go b/pkg/cmd/image/push.go index 5c4b9d1272e..8731b0cfc94 100644 --- a/pkg/cmd/image/push.go +++ b/pkg/cmd/image/push.go @@ -210,7 +210,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options return err } if options.GOptions.Snapshotter == "soci" { - if err = snapshotterutil.CreateSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { + if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { return err } if err = snapshotterutil.PushSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms); err != nil { diff --git a/pkg/snapshotterutil/sociutil.go b/pkg/snapshotterutil/sociutil.go index a2148de027c..240ef54737e 100644 --- a/pkg/snapshotterutil/sociutil.go +++ b/pkg/snapshotterutil/sociutil.go @@ -18,23 +18,29 @@ package snapshotterutil import ( "bufio" + "context" + "fmt" "os" "os/exec" + "regexp" "strconv" "strings" + "github.com/Masterminds/semver/v3" + + "github.com/containerd/containerd/v2/client" "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/api/types" ) -// CreateSoci creates a SOCI index(`rawRef`) -func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error { +// setupSociCommand creates and sets up a SOCI command with common configuration +func setupSociCommand(gOpts types.GlobalCommandOptions) (*exec.Cmd, error) { sociExecutable, err := exec.LookPath("soci") if err != nil { log.L.WithError(err).Error("soci executable not found in path $PATH") log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") - return err + return nil, err } sociCmd := exec.Command(sociExecutable) @@ -47,7 +53,115 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo if gOpts.Namespace != "" { sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace) } - // #endregion + + return sociCmd, nil +} + +// CheckSociVersion checks if the SOCI binary version is at least the required version +func CheckSociVersion(requiredVersion string) error { + sociExecutable, err := exec.LookPath("soci") + if err != nil { + log.L.WithError(err).Error("soci executable not found in path $PATH") + log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") + return err + } + + cmd := exec.Command(sociExecutable, "--version") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to get SOCI version: %w", err) + } + + // Parse the version string + versionStr := string(output) + // Handle format like "soci version v0.10.0 8bbfe951bbb411798ee85dbd908544df4a1619a8.m" + re := regexp.MustCompile(`v?(\d+\.\d+\.\d+)`) + matches := re.FindStringSubmatch(versionStr) + if len(matches) < 2 { + return fmt.Errorf("failed to parse SOCI version from output: %s", versionStr) + } + + // Extract version number + installedVersionStr := matches[1] + + // Parse versions using semver package + installedVersion, err := semver.NewVersion(installedVersionStr) + if err != nil { + return fmt.Errorf("failed to parse installed SOCI version: %w", err) + } + + reqVersion, err := semver.NewVersion(requiredVersion) + if err != nil { + return fmt.Errorf("failed to parse required SOCI version: %w", err) + } + + // Compare versions + if installedVersion.LessThan(reqVersion) { + return fmt.Errorf("SOCI version %s is lower than the required version %s for the convert operation", installedVersion.String(), reqVersion.String()) + } + + return nil +} + +// ConvertSociIndexV2 converts an image to SOCI format and returns the converted image reference with digest +func ConvertSociIndexV2(ctx context.Context, client *client.Client, srcRef string, destRef string, gOpts types.GlobalCommandOptions, platforms []string, sOpts types.SociOptions) (string, error) { + // Check if SOCI version is at least 0.10.0 which is required for the convert operation + if err := CheckSociVersion("0.10.0"); err != nil { + return "", err + } + + sociCmd, err := setupSociCommand(gOpts) + if err != nil { + return "", err + } + + sociCmd.Args = append(sociCmd.Args, "convert") + + if len(platforms) > 0 { + // multiple values need to be passed as separate, repeating flags in soci as it uses urfave + // https://github.com/urfave/cli/blob/main/docs/v2/examples/flags.md#multiple-values-per-single-flag + for _, p := range platforms { + sociCmd.Args = append(sociCmd.Args, "--platform", p) + } + } + + if sOpts.SpanSize != -1 { + sociCmd.Args = append(sociCmd.Args, "--span-size", strconv.FormatInt(sOpts.SpanSize, 10)) + } + + if sOpts.MinLayerSize != -1 { + sociCmd.Args = append(sociCmd.Args, "--min-layer-size", strconv.FormatInt(sOpts.MinLayerSize, 10)) + } + + sociCmd.Args = append(sociCmd.Args, srcRef, destRef) + + log.L.Infof("Converting image from %s to %s using SOCI format", srcRef, destRef) + + err = processSociIO(sociCmd) + if err != nil { + return "", err + } + err = sociCmd.Wait() + if err != nil { + return "", err + } + + // Get the converted image's digest + img, err := client.GetImage(ctx, destRef) + if err != nil { + return "", fmt.Errorf("failed to get converted image: %w", err) + } + + // Return the full reference with digest + return fmt.Sprintf("%s@%s", destRef, img.Target().Digest), nil +} + +// CreateSociIndexV1 creates a SOCI index(`rawRef`) +func CreateSociIndexV1(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error { + sociCmd, err := setupSociCommand(gOpts) + if err != nil { + return err + } // Global flags have to be put before subcommand before soci upgrades to urfave v3. // https://github.com/urfave/cli/issues/1113 @@ -73,7 +187,7 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo // --timeout, --debug, --content-store sociCmd.Args = append(sociCmd.Args, rawRef) - log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args) + log.L.Debugf("running soci %v", sociCmd.Args) err = processSociIO(sociCmd) if err != nil { @@ -88,25 +202,11 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string) error { log.L.Debugf("pushing SOCI index: %s", rawRef) - sociExecutable, err := exec.LookPath("soci") + sociCmd, err := setupSociCommand(gOpts) if err != nil { - log.L.WithError(err).Error("soci executable not found in path $PATH") - log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") return err } - sociCmd := exec.Command(sociExecutable) - sociCmd.Env = os.Environ() - - // #region for global flags. - if gOpts.Address != "" { - sociCmd.Args = append(sociCmd.Args, "--address", gOpts.Address) - } - if gOpts.Namespace != "" { - sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace) - } - // #endregion - // Global flags have to be put before subcommand before soci upgrades to urfave v3. // https://github.com/urfave/cli/issues/1113 sociCmd.Args = append(sociCmd.Args, "push") @@ -131,7 +231,7 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, } sociCmd.Args = append(sociCmd.Args, rawRef) - log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args) + log.L.Debugf("running soci %v", sociCmd.Args) err = processSociIO(sociCmd) if err != nil { diff --git a/pkg/testutil/nerdtest/requirements.go b/pkg/testutil/nerdtest/requirements.go index 3cc9390996a..46bbeee675d 100644 --- a/pkg/testutil/nerdtest/requirements.go +++ b/pkg/testutil/nerdtest/requirements.go @@ -35,6 +35,7 @@ import ( "github.com/containerd/nerdctl/v2/pkg/infoutil" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" "github.com/containerd/nerdctl/v2/pkg/rootlessutil" + "github.com/containerd/nerdctl/v2/pkg/snapshotterutil" "github.com/containerd/nerdctl/v2/pkg/testutil" "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest/platform" ) @@ -411,6 +412,21 @@ var RemapIDs = &test.Requirement{ }, } +// SociVersion returns a requirement that checks if the installed SOCI version +// meets the minimum required version +func SociVersion(minVersion string) *test.Requirement { + return &test.Requirement{ + Check: func(data test.Data, helpers test.Helpers) (bool, string) { + // Use the common CheckSociVersion function from snapshotterutil + err := snapshotterutil.CheckSociVersion(minVersion) + if err != nil { + return false, err.Error() + } + return true, fmt.Sprintf("soci version meets minimum requirement %s", minVersion) + }, + } +} + func ContainerdVersion(v string) *test.Requirement { return &test.Requirement{ Check: func(data test.Data, helpers test.Helpers) (bool, string) {