Merge pull request #4352 from ChengyuZhu6/estargz

commit: Add support for converting writable layers to eStargz blobs

Author: AkihiroSuda

cmd/nerdctl/container/container_commit.go

@@ -44,6 +44,10 @@ func CommitCommand() *cobra.Command {
 	cmd.Flags().BoolP("pause", "p", true, "Pause container during commit")
 	cmd.Flags().StringP("compression", "", "gzip", "commit compression algorithm (zstd or gzip)")
 	cmd.Flags().String("format", "docker", "Format of the committed image (docker or oci)")
+	cmd.Flags().Bool("estargz", false, "Convert the committed layer to eStargz for lazy pulling")
+	cmd.Flags().Int("estargz-compression-level", 9, "eStargz compression level (1-9)")
+	cmd.Flags().Int("estargz-chunk-size", 0, "eStargz chunk size")
+	cmd.Flags().Int("estargz-min-chunk-size", 0, "The minimal number of bytes of data must be written in one gzip stream")
 	return cmd
 }
 
@@ -86,6 +90,23 @@ func commitOptions(cmd *cobra.Command) (types.ContainerCommitOptions, error) {
 		return types.ContainerCommitOptions{}, errors.New("--format param only supports docker or oci")
 	}
 
+	estargz, err := cmd.Flags().GetBool("estargz")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+	estargzCompressionLevel, err := cmd.Flags().GetInt("estargz-compression-level")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+	estargzChunkSize, err := cmd.Flags().GetInt("estargz-chunk-size")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+	estargzMinChunkSize, err := cmd.Flags().GetInt("estargz-min-chunk-size")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+
 	return types.ContainerCommitOptions{
 		Stdout:      cmd.OutOrStdout(),
 		GOptions:    globalOptions,
@@ -95,6 +116,12 @@ func commitOptions(cmd *cobra.Command) (types.ContainerCommitOptions, error) {
 		Change:      change,
 		Compression: types.CompressionType(com),
 		Format:      types.ImageFormat(format),
+		EstargzOptions: types.EstargzOptions{
+			Estargz:                 estargz,
+			EstargzCompressionLevel: estargzCompressionLevel,
+			EstargzChunkSize:        estargzChunkSize,
+			EstargzMinChunkSize:     estargzMinChunkSize,
+		},
 	}, nil
 }
 

cmd/nerdctl/image/image_convert.go

@@ -237,37 +237,6 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 	return types.ImageConvertOptions{
 		GOptions: globalOptions,
 		Format:   format,
-		// #region estargz flags
-		Estargz:                 estargz,
-		EstargzRecordIn:         estargzRecordIn,
-		EstargzCompressionLevel: estargzCompressionLevel,
-		EstargzChunkSize:        estargzChunkSize,
-		EstargzMinChunkSize:     estargzMinChunkSize,
-		EstargzExternalToc:      estargzExternalTOC,
-		EstargzKeepDiffID:       estargzKeepDiffID,
-		// #endregion
-		// #region zstd flags
-		Zstd:                 zstd,
-		ZstdCompressionLevel: zstdCompressionLevel,
-		// #endregion
-		// #region zstd:chunked flags
-		ZstdChunked:                 zstdchunked,
-		ZstdChunkedCompressionLevel: zstdChunkedCompressionLevel,
-		ZstdChunkedChunkSize:        zstdChunkedChunkSize,
-		ZstdChunkedRecordIn:         zstdChunkedRecordIn,
-		// #endregion
-		// #region nydus flags
-		Nydus:                 nydus,
-		NydusBuilderPath:      nydusBuilderPath,
-		NydusWorkDir:          nydusWorkDir,
-		NydusPrefetchPatterns: nydusPrefetchPatterns,
-		NydusCompressor:       nydusCompressor,
-		// #endregion
-		// #region overlaybd flags
-		Overlaybd:      overlaybd,
-		OverlayFsType:  overlaybdFsType,
-		OverlaydbDBStr: overlaybdDbstr,
-		// #endregion
 		// #region generic flags
 		Uncompress: uncompress,
 		Oci:        oci,
@@ -276,6 +245,38 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 		Platforms:    platforms,
 		AllPlatforms: allPlatforms,
 		// #endregion
+		// Embed image format options
+		EstargzOptions: types.EstargzOptions{
+			Estargz:                 estargz,
+			EstargzRecordIn:         estargzRecordIn,
+			EstargzCompressionLevel: estargzCompressionLevel,
+			EstargzChunkSize:        estargzChunkSize,
+			EstargzMinChunkSize:     estargzMinChunkSize,
+			EstargzExternalToc:      estargzExternalTOC,
+			EstargzKeepDiffID:       estargzKeepDiffID,
+		},
+		ZstdOptions: types.ZstdOptions{
+			Zstd:                 zstd,
+			ZstdCompressionLevel: zstdCompressionLevel,
+		},
+		ZstdChunkedOptions: types.ZstdChunkedOptions{
+			ZstdChunked:                 zstdchunked,
+			ZstdChunkedCompressionLevel: zstdChunkedCompressionLevel,
+			ZstdChunkedChunkSize:        zstdChunkedChunkSize,
+			ZstdChunkedRecordIn:         zstdChunkedRecordIn,
+		},
+		NydusOptions: types.NydusOptions{
+			Nydus:                 nydus,
+			NydusBuilderPath:      nydusBuilderPath,
+			NydusWorkDir:          nydusWorkDir,
+			NydusPrefetchPatterns: nydusPrefetchPatterns,
+			NydusCompressor:       nydusCompressor,
+		},
+		OverlaybdOptions: types.OverlaybdOptions{
+			Overlaybd:      overlaybd,
+			OverlayFsType:  overlaybdFsType,
+			OverlaydbDBStr: overlaybdDbstr,
+		},
 		Stdout: cmd.OutOrStdout(),
 	}, nil
 }

docs/command-reference.md

@@ -778,6 +778,10 @@ Flags:
 - :whale: `-p, --pause`: Pause container during commit (default: true)
 - :nerd_face: `--compression`: Commit compression algorithm (supported values: zstd or gzip) (default: gzip) (zstd is generally better for compression ratio but might not be as widely supported)
 - :nerd_face: `--format`: Format of the committed image (supported values: docker or oci) (default: docker) (docker uses Docker Schema2 media types for compatibility, oci uses OCI image format media types)
+- :nerd_face: `--estargz`: Convert the committed layer to eStargz for lazy pulling
+- :nerd_face: `--estargz-compression-level`: eStargz compression level (1-9) (default: 9)
+- :nerd_face: `--estargz-chunk-size`: eStargz chunk size
+- :nerd_face: `--estargz-min-chunk-size`: The minimal number of bytes of data must be written in one gzip stream
 
 ## Image management
 

pkg/api/types/container_types.go

@@ -396,6 +396,8 @@ type ContainerCommitOptions struct {
 	Compression CompressionType
 	// Format specifies the image format for the committed image (docker or oci)
 	Format ImageFormat
+	// Embed EstargzOptions for eStargz conversion options
+	EstargzOptions
 }
 
 type CompressionType string

pkg/api/types/image_types.go

@@ -67,7 +67,16 @@ type ImageConvertOptions struct {
 	// Format the output using the given Go template, e.g, 'json'
 	Format string
 
-	// #region estargz flags
+	// Embed image format options
+	EstargzOptions
+	ZstdOptions
+	ZstdChunkedOptions
+	NydusOptions
+	OverlaybdOptions
+}
+
+// EstargzOptions contains eStargz conversion options
+type EstargzOptions struct {
 	// Estargz convert legacy tar(.gz) layers to eStargz for lazy pulling. Should be used in conjunction with '--oci'
 	Estargz bool
 	// EstargzRecordIn read 'ctr-remote optimize --record-out=<FILE>' record file (EXPERIMENTAL)
@@ -82,16 +91,18 @@ type ImageConvertOptions struct {
 	EstargzExternalToc bool
 	// EstargzKeepDiffID convert to esgz without changing diffID (cannot be used in conjunction with '--estargz-record-in'. must be specified with '--estargz-external-toc')
 	EstargzKeepDiffID bool
-	// #endregion
+}
 
-	// #region zstd flags
+// ZstdOptions contains zstd conversion options
+type ZstdOptions struct {
 	// Zstd convert legacy tar(.gz) layers to zstd. Should be used in conjunction with '--oci'
 	Zstd bool
 	// ZstdCompressionLevel zstd compression level
 	ZstdCompressionLevel int
-	// #endregion
+}
 
-	// #region zstd:chunked flags
+// ZstdChunkedOptions contains zstd:chunked conversion options
+type ZstdChunkedOptions struct {
 	// ZstdChunked convert legacy tar(.gz) layers to zstd:chunked for lazy pulling. Should be used in conjunction with '--oci'
 	ZstdChunked bool
 	// ZstdChunkedCompressionLevel zstd compression level
@@ -100,9 +111,10 @@ type ImageConvertOptions struct {
 	ZstdChunkedChunkSize int
 	// ZstdChunkedRecordIn read 'ctr-remote optimize --record-out=<FILE>' record file (EXPERIMENTAL)
 	ZstdChunkedRecordIn string
-	// #endregion
+}
 
-	// #region nydus flags
+// NydusOptions contains nydus conversion options
+type NydusOptions struct {
 	// Nydus convert legacy tar(.gz) layers to nydus for lazy pulling. Should be used in conjunction with '--oci'
 	Nydus bool
 	// NydusBuilderPath the nydus-image binary path, if unset, search in PATH environment
@@ -113,17 +125,16 @@ type ImageConvertOptions struct {
 	NydusPrefetchPatterns string
 	// NydusCompressor nydus blob compression algorithm, possible values: `none`, `lz4_block`, `zstd`, default is `lz4_block`
 	NydusCompressor string
-	// #endregion
+}
 
-	// #region overlaybd flags
+// OverlaybdOptions contains overlaybd conversion options
+type OverlaybdOptions struct {
 	// Overlaybd convert tar.gz layers to overlaybd layers
 	Overlaybd bool
 	// OverlayFsType filesystem type for overlaybd
 	OverlayFsType string
 	// OverlaydbDBStr database config string for overlaybd
 	OverlaydbDBStr string
-	// #endregion
-
 }
 
 // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`.

pkg/cmd/container/commit.go

@@ -44,13 +44,14 @@ func Commit(ctx context.Context, client *containerd.Client, rawRef string, req s
 	}
 
 	opts := &commit.Opts{
-		Author:      options.Author,
-		Message:     options.Message,
-		Ref:         parsedReference.String(),
-		Pause:       options.Pause,
-		Changes:     changes,
-		Compression: options.Compression,
-		Format:      options.Format,
+		Author:         options.Author,
+		Message:        options.Message,
+		Ref:            parsedReference.String(),
+		Pause:          options.Pause,
+		Changes:        changes,
+		Compression:    options.Compression,
+		Format:         options.Format,
+		EstargzOptions: options.EstargzOptions,
 	}
 
 	walker := &containerwalker.ContainerWalker{

pkg/imgutil/commit/commit.go

@@ -43,6 +43,8 @@ import (
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	"github.com/containerd/platforms"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
@@ -64,6 +66,7 @@ type Opts struct {
 	Changes     Changes
 	Compression types.CompressionType
 	Format      types.ImageFormat
+	types.EstargzOptions
 }
 
 var (
@@ -431,6 +434,50 @@ func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs c
 		return ocispec.Descriptor{}, digest.Digest(""), err
 	}
 
+	// Convert to eStargz if requested
+	if opts.Estargz {
+		log.G(ctx).Infof("Converting diff layer to eStargz format")
+
+		esgzOpts := []estargz.Option{
+			estargz.WithCompressionLevel(opts.EstargzCompressionLevel),
+		}
+		if opts.EstargzChunkSize > 0 {
+			esgzOpts = append(esgzOpts, estargz.WithChunkSize(opts.EstargzChunkSize))
+		}
+		if opts.EstargzMinChunkSize > 0 {
+			esgzOpts = append(esgzOpts, estargz.WithMinChunkSize(opts.EstargzMinChunkSize))
+		}
+
+		convertFunc := estargzconvert.LayerConvertFunc(esgzOpts...)
+
+		esgzDesc, err := convertFunc(ctx, cs, newDesc)
+		if err != nil {
+			return ocispec.Descriptor{}, digest.Digest(""), fmt.Errorf("failed to convert diff layer to eStargz: %w", err)
+		} else if esgzDesc != nil {
+			esgzDesc.MediaType = mediaType
+			esgzInfo, err := cs.Info(ctx, esgzDesc.Digest)
+			if err != nil {
+				return ocispec.Descriptor{}, digest.Digest(""), err
+			}
+
+			esgzDiffIDStr, ok := esgzInfo.Labels["containerd.io/uncompressed"]
+			if !ok {
+				return ocispec.Descriptor{}, digest.Digest(""), fmt.Errorf("invalid differ response with no diffID")
+			}
+
+			esgzDiffID, err := digest.Parse(esgzDiffIDStr)
+			if err != nil {
+				return ocispec.Descriptor{}, digest.Digest(""), err
+			}
+			return ocispec.Descriptor{
+				MediaType:   esgzDesc.MediaType,
+				Digest:      esgzDesc.Digest,
+				Size:        esgzDesc.Size,
+				Annotations: esgzDesc.Annotations,
+			}, esgzDiffID, nil
+		}
+	}
+
 	return ocispec.Descriptor{
 		MediaType: mediaType,
 		Digest:    newDesc.Digest,