Merge pull request containerd#4388 from ChengyuZhu6/zstd

commit: Add support for converting writable layers to zstdchunked blobs

Author: AkihiroSuda

cmd/nerdctl/container/container_commit.go

@@ -48,6 +48,9 @@ func CommitCommand() *cobra.Command {
 	cmd.Flags().Int("estargz-compression-level", 9, "eStargz compression level (1-9)")
 	cmd.Flags().Int("estargz-chunk-size", 0, "eStargz chunk size")
 	cmd.Flags().Int("estargz-min-chunk-size", 0, "The minimal number of bytes of data must be written in one gzip stream")
+	cmd.Flags().Bool("zstdchunked", false, "Convert the committed layer to zstd:chunked for lazy pulling")
+	cmd.Flags().Int("zstdchunked-compression-level", 3, "zstd:chunked compression level")
+	cmd.Flags().Int("zstdchunked-chunk-size", 0, "zstd:chunked chunk size")
 	return cmd
 }
 
@@ -107,6 +110,24 @@ func commitOptions(cmd *cobra.Command) (types.ContainerCommitOptions, error) {
 		return types.ContainerCommitOptions{}, err
 	}
 
+	zstdchunked, err := cmd.Flags().GetBool("zstdchunked")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+	zstdchunkedCompressionLevel, err := cmd.Flags().GetInt("zstdchunked-compression-level")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+	zstdchunkedChunkSize, err := cmd.Flags().GetInt("zstdchunked-chunk-size")
+	if err != nil {
+		return types.ContainerCommitOptions{}, err
+	}
+
+	// estargz and zstdchunked are mutually exclusive
+	if estargz && zstdchunked {
+		return types.ContainerCommitOptions{}, errors.New("options --estargz and --zstdchunked lead to conflict, only one of them can be used")
+	}
+
 	return types.ContainerCommitOptions{
 		Stdout:      cmd.OutOrStdout(),
 		GOptions:    globalOptions,
@@ -122,6 +143,11 @@ func commitOptions(cmd *cobra.Command) (types.ContainerCommitOptions, error) {
 			EstargzChunkSize:        estargzChunkSize,
 			EstargzMinChunkSize:     estargzMinChunkSize,
 		},
+		ZstdChunkedOptions: types.ZstdChunkedOptions{
+			ZstdChunked:                 zstdchunked,
+			ZstdChunkedCompressionLevel: zstdchunkedCompressionLevel,
+			ZstdChunkedChunkSize:        zstdchunkedChunkSize,
+		},
 	}, nil
 }
 

docs/command-reference.md

@@ -782,6 +782,10 @@ Flags:
 - :nerd_face: `--estargz-compression-level`: eStargz compression level (1-9) (default: 9)
 - :nerd_face: `--estargz-chunk-size`: eStargz chunk size
 - :nerd_face: `--estargz-min-chunk-size`: The minimal number of bytes of data must be written in one gzip stream
+- :nerd_face: `--zstdchunked`: Convert the committed layer to zstd:chunked for lazy pulling
+support zstdchunked convert
+- :nerd_face: `--zstdchunked-compression-level`: zstd:chunked compression level (default: 3)
+- :nerd_face: `--zstdchunked-chunk-size`: zstd:chunked chunk size
 
 ## Image management
 

pkg/api/types/container_types.go

@@ -398,6 +398,8 @@ type ContainerCommitOptions struct {
 	Format ImageFormat
 	// Embed EstargzOptions for eStargz conversion options
 	EstargzOptions
+	// Embed ZstdChunkedOptions for zstd:chunked conversion options
+	ZstdChunkedOptions
 }
 
 type CompressionType string

pkg/cmd/container/commit.go

@@ -44,14 +44,15 @@ func Commit(ctx context.Context, client *containerd.Client, rawRef string, req s
 	}
 
 	opts := &commit.Opts{
-		Author:         options.Author,
-		Message:        options.Message,
-		Ref:            parsedReference.String(),
-		Pause:          options.Pause,
-		Changes:        changes,
-		Compression:    options.Compression,
-		Format:         options.Format,
-		EstargzOptions: options.EstargzOptions,
+		Author:             options.Author,
+		Message:            options.Message,
+		Ref:                parsedReference.String(),
+		Pause:              options.Pause,
+		Changes:            changes,
+		Compression:        options.Compression,
+		Format:             options.Format,
+		EstargzOptions:     options.EstargzOptions,
+		ZstdChunkedOptions: options.ZstdChunkedOptions,
 	}
 
 	walker := &containerwalker.ContainerWalker{

pkg/imgutil/commit/commit.go

@@ -27,6 +27,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
 	"github.com/opencontainers/image-spec/specs-go"
@@ -45,6 +46,7 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
@@ -67,6 +69,7 @@ type Opts struct {
 	Compression types.CompressionType
 	Format      types.ImageFormat
 	types.EstargzOptions
+	types.ZstdChunkedOptions
 }
 
 var (
@@ -181,6 +184,9 @@ func Commit(ctx context.Context, client *containerd.Client, container containerd
 	// Sync filesystem to make sure that all the data writes in container could be persisted to disk.
 	Sync()
 
+	if opts.ZstdChunked {
+		opts.Compression = types.Zstd
+	}
 	diffLayerDesc, diffID, err := createDiff(ctx, id, sn, client.ContentStore(), differ, opts.Compression, opts)
 	if err != nil {
 		return emptyDigest, fmt.Errorf("failed to export layer: %w", err)
@@ -478,6 +484,44 @@ func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs c
 		}
 	}
 
+	// Convert to zstd:chunked if requested
+	if opts.ZstdChunked {
+		log.G(ctx).Infof("Converting diff layer to zstd:chunked format")
+
+		esgzOpts := []estargz.Option{
+			estargz.WithChunkSize(opts.ZstdChunkedChunkSize),
+		}
+
+		convertFunc := zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(zstd.EncoderLevelFromZstd(opts.ZstdChunkedCompressionLevel), esgzOpts...)
+
+		zstdchunkedDesc, err := convertFunc(ctx, cs, newDesc)
+		if err != nil {
+			return ocispec.Descriptor{}, digest.Digest(""), fmt.Errorf("failed to convert diff layer to zstd:chunked: %w", err)
+		} else if zstdchunkedDesc != nil {
+			zstdchunkedDesc.MediaType = mediaType
+			zstdchunkedInfo, err := cs.Info(ctx, zstdchunkedDesc.Digest)
+			if err != nil {
+				return ocispec.Descriptor{}, digest.Digest(""), err
+			}
+
+			zstdchunkedDiffIDStr, ok := zstdchunkedInfo.Labels["containerd.io/uncompressed"]
+			if !ok {
+				return ocispec.Descriptor{}, digest.Digest(""), fmt.Errorf("invalid differ response with no diffID")
+			}
+
+			zstdchunkedDiffID, err := digest.Parse(zstdchunkedDiffIDStr)
+			if err != nil {
+				return ocispec.Descriptor{}, digest.Digest(""), err
+			}
+			return ocispec.Descriptor{
+				MediaType:   zstdchunkedDesc.MediaType,
+				Digest:      zstdchunkedDesc.Digest,
+				Size:        zstdchunkedDesc.Size,
+				Annotations: zstdchunkedDesc.Annotations,
+			}, zstdchunkedDiffID, nil
+		}
+	}
+
 	return ocispec.Descriptor{
 		MediaType: mediaType,
 		Digest:    newDesc.Digest,