diff --git a/cmd/nerdctl/image/image_convert.go b/cmd/nerdctl/image/image_convert.go index 871d9c97d81..eb3585099b2 100644 --- a/cmd/nerdctl/image/image_convert.go +++ b/cmd/nerdctl/image/image_convert.go @@ -89,6 +89,12 @@ func convertCommand() *cobra.Command { cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd") // #endregion + // #region soci flags + cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.") + cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format") + cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans") + // #endregion + // #region generic flags cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers") cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types") @@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) { } // #endregion + // #region soci flags + soci, err := cmd.Flags().GetBool("soci") + if err != nil { + return types.ImageConvertOptions{}, err + } + sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size") + if err != nil { + return types.ImageConvertOptions{}, err + } + sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size") + if err != nil { + return types.ImageConvertOptions{}, err + } + // #endregion + // #region generic flags uncompress, err := cmd.Flags().GetBool("uncompress") if err != nil { @@ -268,6 +289,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) { OverlayFsType: overlaybdFsType, OverlaydbDBStr: overlaybdDbstr, // #endregion + // #region soci flags + Soci: soci, + SociOptions: types.SociOptions{ + SpanSize: sociSpanSize, + MinLayerSize: sociMinLayerSize, + }, + // #endregion // #region generic flags Uncompress: uncompress, Oci: oci, diff --git a/cmd/nerdctl/image/image_convert_linux_test.go b/cmd/nerdctl/image/image_convert_linux_test.go index b26358ec8b9..fd4a8606324 100644 --- a/cmd/nerdctl/image/image_convert_linux_test.go +++ b/cmd/nerdctl/image/image_convert_linux_test.go @@ -88,6 +88,24 @@ func TestImageConvert(t *testing.T) { }, Expected: test.Expects(0, nil, nil), }, + { + Description: "soci", + Require: require.All( + require.Not(nerdtest.Docker), + nerdtest.Soci, + nerdtest.SociVersion("0.10.0"), + ), + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rmi", "-f", data.Identifier("converted-image")) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("image", "convert", "--soci", + "--soci-span-size", "2097152", + "--soci-min-layer-size", "20971520", + testutil.CommonImage, data.Identifier("converted-image")) + }, + Expected: test.Expects(0, nil, nil), + }, }, } diff --git a/docs/command-reference.md b/docs/command-reference.md index a4173d8f93a..154cfabc336 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -960,6 +960,10 @@ Flags: - `--oci` : convert Docker media types to OCI media types - `--platform=` : convert content for a specific platform - `--all-platforms` : convert content for all platforms (default: false) +- `--soci`: generate SOCI v2 Indices to oci images. *[**Note**: content is converted for all platforms by default when using this flag, use the `--platorm` flag to limit this behavior]* +- `--soci-min-layer-size` : Span size in bytes that soci index uses to segment layer data. Default is 4 MiB. +- `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB. + ### :nerd_face: nerdctl image encrypt diff --git a/docs/soci.md b/docs/soci.md index 67fbe92f584..d2dc84645df 100644 --- a/docs/soci.md +++ b/docs/soci.md @@ -45,3 +45,18 @@ For images that already have SOCI indices, see https://gallery.ecr.aws/soci-work nerdctl push --snapshotter=soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest ``` --soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details. + + +## Enable SOCI for `nerdctl image convert` + +| :zap: Requirement | nerdctl >= 2.2.0 | +| ----------------- | ---------------- | + +| :zap: Requirement | soci-snapshotter >= 0.10.0 | +| ----------------- | ---------------- | + +- Convert an image to generate SOCI Index artifacts v2. Running the `nerdctl image convert` with the `--soci` flag and a `srcImg` and `dstImg`, `nerdctl` will create the SOCI v2 indices and the new image will be present in the `dstImg` address. +```console +nerdctl image convert --soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest public.ecr.aws/my-registry/my-repo:soci +``` +--soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details. \ No newline at end of file diff --git a/pkg/api/types/image_types.go b/pkg/api/types/image_types.go index d48e6318026..b2a8316c890 100644 --- a/pkg/api/types/image_types.go +++ b/pkg/api/types/image_types.go @@ -19,7 +19,7 @@ package types import ( "io" - "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // ImageListOptions specifies options for `nerdctl image list`. @@ -124,6 +124,12 @@ type ImageConvertOptions struct { OverlaydbDBStr string // #endregion + // #region soci flags + // Soci convert image to SOCI format.eiifc + Soci bool + // SociOptions contains SOCI-specific options + SociOptions SociOptions + // #endregion } // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`. @@ -200,7 +206,7 @@ type ImagePullOptions struct { // If nil, it will unpack automatically if only 1 platform is specified. Unpack *bool // Content for specific platforms. Empty if `--all-platforms` is true - OCISpecPlatform []v1.Platform + OCISpecPlatform []ocispec.Platform // Pull mode Mode string // Suppress verbose output diff --git a/pkg/cmd/image/convert.go b/pkg/cmd/image/convert.go index b7963ea2702..12a2040d598 100644 --- a/pkg/cmd/image/convert.go +++ b/pkg/cmd/image/convert.go @@ -47,6 +47,7 @@ import ( converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter" "github.com/containerd/nerdctl/v2/pkg/platformutil" "github.com/containerd/nerdctl/v2/pkg/referenceutil" + "github.com/containerd/nerdctl/v2/pkg/snapshotterutil" ) func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error { @@ -86,8 +87,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa zstdchunked := options.ZstdChunked overlaybd := options.Overlaybd nydus := options.Nydus + soci := options.Soci var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error) - if estargz || zstd || zstdchunked || overlaybd || nydus { + if estargz || zstd || zstdchunked || overlaybd || nydus || soci { convertCount := 0 if estargz { convertCount++ @@ -104,9 +106,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa if nydus { convertCount++ } + if soci { + convertCount++ + } if convertCount > 1 { - return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used") + return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used") } var convertFunc converter.ConvertFunc @@ -164,6 +169,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa )), ) convertType = "nydus" + case soci: + // Convert image to SOCI format + convertedRef, err := snapshotterutil.ConvertSociIndexV2(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions) + if err != nil { + return fmt.Errorf("failed to convert image to SOCI format: %w", err) + } + res := converterutil.ConvertedImageInfo{ + Image: convertedRef, + } + return printConvertedImage(options.Stdout, options, res) } if convertType != "overlaybd" { diff --git a/pkg/cmd/image/push.go b/pkg/cmd/image/push.go index 0c463e76f02..0519d919282 100644 --- a/pkg/cmd/image/push.go +++ b/pkg/cmd/image/push.go @@ -209,7 +209,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options return err } if options.GOptions.Snapshotter == "soci" { - if err = snapshotterutil.CreateSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { + if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { return err } if err = snapshotterutil.PushSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms); err != nil { diff --git a/pkg/snapshotterutil/sociutil.go b/pkg/snapshotterutil/sociutil.go index a2148de027c..3f6371b6181 100644 --- a/pkg/snapshotterutil/sociutil.go +++ b/pkg/snapshotterutil/sociutil.go @@ -18,23 +18,26 @@ package snapshotterutil import ( "bufio" + "context" + "fmt" "os" "os/exec" "strconv" "strings" + "github.com/containerd/containerd/v2/client" "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/api/types" ) -// CreateSoci creates a SOCI index(`rawRef`) -func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error { +// setupSociCommand creates and sets up a SOCI command with common configuration +func setupSociCommand(gOpts types.GlobalCommandOptions) (*exec.Cmd, error) { sociExecutable, err := exec.LookPath("soci") if err != nil { log.L.WithError(err).Error("soci executable not found in path $PATH") log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") - return err + return nil, err } sociCmd := exec.Command(sociExecutable) @@ -47,7 +50,77 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo if gOpts.Namespace != "" { sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace) } - // #endregion + + return sociCmd, nil +} + +// ConvertSociIndexV2 converts an image to SOCI format and returns the converted image reference with digest +func ConvertSociIndexV2(ctx context.Context, client *client.Client, srcRef string, destRef string, gOpts types.GlobalCommandOptions, platforms []string, sOpts types.SociOptions) (string, error) { + // Check minimum required version + // const minRequiredVersion = "0.9.0" + // if err := checkSociVersion(minRequiredVersion); err != nil { + // return err + // } + + sociCmd, err := setupSociCommand(gOpts) + if err != nil { + return "", err + } + + // TODO: Implement conversion logic + sociCmd.Args = append(sociCmd.Args, "convert") + + if len(platforms) > 0 { + // multiple values need to be passed as separate, repeating flags in soci as it uses urfave + // https://github.com/urfave/cli/blob/main/docs/v2/examples/flags.md#multiple-values-per-single-flag + for _, p := range platforms { + sociCmd.Args = append(sociCmd.Args, "--platform", p) + } + } + + if sOpts.SpanSize != -1 { + sociCmd.Args = append(sociCmd.Args, "--span-size", strconv.FormatInt(sOpts.SpanSize, 10)) + } + + if sOpts.MinLayerSize != -1 { + sociCmd.Args = append(sociCmd.Args, "--min-layer-size", strconv.FormatInt(sOpts.MinLayerSize, 10)) + } + + sociCmd.Args = append(sociCmd.Args, srcRef, destRef) + + log.L.Infof("Converting image from %s to %s using SOCI format", srcRef, destRef) + + err = processSociIO(sociCmd) + if err != nil { + return "", err + } + err = sociCmd.Wait() + if err != nil { + return "", err + } + + // Get the converted image's digest + img, err := client.GetImage(ctx, destRef) + if err != nil { + return "", fmt.Errorf("failed to get converted image: %w", err) + } + + // Return the full reference with digest + return fmt.Sprintf("%s@%s", destRef, img.Target().Digest), nil +} + +// CreateSociIndexV1 creates a SOCI index(`rawRef`) +func CreateSociIndexV1(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error { + // Check minimum required version + // const minRequiredVersion = "0.9.0" + // if err := checkSociVersion(minRequiredVersion); err != nil { + // return err + // } + + sociCmd, err := setupSociCommand(gOpts) + if err != nil { + return err + } // Global flags have to be put before subcommand before soci upgrades to urfave v3. // https://github.com/urfave/cli/issues/1113 @@ -73,7 +146,7 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo // --timeout, --debug, --content-store sociCmd.Args = append(sociCmd.Args, rawRef) - log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args) + log.L.Debugf("running soci %v", sociCmd.Args) err = processSociIO(sociCmd) if err != nil { @@ -88,25 +161,11 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string) error { log.L.Debugf("pushing SOCI index: %s", rawRef) - sociExecutable, err := exec.LookPath("soci") + sociCmd, err := setupSociCommand(gOpts) if err != nil { - log.L.WithError(err).Error("soci executable not found in path $PATH") - log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") return err } - sociCmd := exec.Command(sociExecutable) - sociCmd.Env = os.Environ() - - // #region for global flags. - if gOpts.Address != "" { - sociCmd.Args = append(sociCmd.Args, "--address", gOpts.Address) - } - if gOpts.Namespace != "" { - sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace) - } - // #endregion - // Global flags have to be put before subcommand before soci upgrades to urfave v3. // https://github.com/urfave/cli/issues/1113 sociCmd.Args = append(sociCmd.Args, "push") @@ -131,7 +190,7 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, } sociCmd.Args = append(sociCmd.Args, rawRef) - log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args) + log.L.Debugf("running soci %v", sociCmd.Args) err = processSociIO(sociCmd) if err != nil { @@ -140,6 +199,55 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, return sociCmd.Wait() } +// checkSociVersion checks if the installed SOCI version meets the minimum required version +// func checkSociVersion(minVersion string) error { +// sociExecutable, err := exec.LookPath("soci") +// if err != nil { +// log.L.WithError(err).Error("soci executable not found in path $PATH") +// log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") +// return err +// } + +// cmd := exec.Command(sociExecutable, "--version") +// output, err := cmd.Output() +// if err != nil { +// return fmt.Errorf("failed to get soci version: %w", err) +// } + +// // Parse version from output +// // Example output format: "soci version v0.9.0 737f61a3db40c386f997c1f126344158aa3ad43c" +// versionStr := strings.TrimSpace(string(output)) +// parts := strings.Fields(versionStr) +// if len(parts) < 3 { +// return fmt.Errorf("unexpected soci version output format: %s", versionStr) +// } + +// // Extract version number without 'v' prefix +// installedVersion := strings.TrimPrefix(parts[2], "v") + +// // Compare versions +// v1, err := semver.NewVersion(installedVersion) +// if err != nil { +// return fmt.Errorf("failed to parse installed version %s: %w", installedVersion, err) +// } + +// v2, err := semver.NewVersion(minVersion) +// if err != nil { +// return fmt.Errorf("failed to parse minimum required version %s: %w", minVersion, err) +// } + +// if v1.LessThan(v2) { +// return fmt.Errorf("installed soci version %s is older than required version %s", installedVersion, minVersion) +// } + +// // Log the full version info including commit hash for debugging purposes +// if len(parts) > 3 { +// log.L.Debugf("soci version: %s (commit: %s)", installedVersion, parts[3]) +// } + +// return nil +// } + func processSociIO(sociCmd *exec.Cmd) error { stdout, err := sociCmd.StdoutPipe() if err != nil { diff --git a/pkg/testutil/nerdtest/requirements.go b/pkg/testutil/nerdtest/requirements.go index e0e621501ee..8a791df720f 100644 --- a/pkg/testutil/nerdtest/requirements.go +++ b/pkg/testutil/nerdtest/requirements.go @@ -419,6 +419,53 @@ var RemapIDs = &test.Requirement{ }, } +// SociVersion returns a requirement that checks if the installed SOCI version +// meets the minimum required version +func SociVersion(minVersion string) *test.Requirement { + return &test.Requirement{ + Check: func(data test.Data, helpers test.Helpers) (bool, string) { + sociExecutable, err := exec.LookPath("soci") + if err != nil { + return false, fmt.Sprintf("soci executable not found in path $PATH: %v", err) + } + + cmd := exec.Command(sociExecutable, "--version") + output, err := cmd.Output() + if err != nil { + return false, fmt.Sprintf("failed to get soci version: %v", err) + } + + // Parse version from output + // Example output format: "soci version v0.9.0 737f61a3db40c386f997c1f126344158aa3ad43c" + versionStr := strings.TrimSpace(string(output)) + parts := strings.Fields(versionStr) + if len(parts) < 3 { + return false, fmt.Sprintf("unexpected soci version output format: %s", versionStr) + } + + // Extract version number without 'v' prefix + installedVersion := strings.TrimPrefix(parts[2], "v") + + // Compare versions + v1, err := semver.NewVersion(installedVersion) + if err != nil { + return false, fmt.Sprintf("failed to parse installed version %s: %v", installedVersion, err) + } + + v2, err := semver.NewVersion(minVersion) + if err != nil { + return false, fmt.Sprintf("failed to parse minimum required version %s: %v", minVersion, err) + } + + if v1.LessThan(v2) { + return false, fmt.Sprintf("installed soci version %s is older than required version %s", installedVersion, minVersion) + } + + return true, fmt.Sprintf("soci version %s meets minimum requirement %s", installedVersion, minVersion) + }, + } +} + func ContainerdVersion(v string) *test.Requirement { return &test.Requirement{ Check: func(data test.Data, helpers test.Helpers) (bool, string) {