Skip to content

Commit 31a9120

Browse files
tonistiigitonistiigi
authored
Merge pull request moby#3968 from sipsma/docker-zstd-apply
compression: register docker zstd stream processor
2 parents f21a96c + 6ce499c commit 31a9120

File tree

4 files changed

+86
-52
lines changed

4 files changed

+86
-52
lines changed

‎client/client_test.go

Lines changed: 75 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -3765,66 +3765,95 @@ func testBuildExportZstd(t *testing.T, sb integration.Sandbox) {
37653765

37663766
func testPullZstdImage(t *testing.T, sb integration.Sandbox) {
37673767
integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush)
3768-
c, err := New(sb.Context(), sb.Address())
3769-
require.NoError(t, err)
3770-
defer c.Close()
3768+
for _, ociMediaTypes := range []bool{true, false} {
3769+
ociMediaTypes := ociMediaTypes
3770+
t.Run(t.Name()+fmt.Sprintf("/ociMediaTypes=%t", ociMediaTypes), func(t *testing.T) {
3771+
c, err := New(sb.Context(), sb.Address())
3772+
require.NoError(t, err)
3773+
defer c.Close()
37713774

3772-
busybox := llb.Image("busybox:latest")
3773-
cmd := `sh -e -c "echo -n zstd > data"`
3775+
busybox := llb.Image("busybox:latest")
3776+
cmd := `sh -e -c "echo -n zstd > data"`
37743777

3775-
st := llb.Scratch()
3776-
st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
3778+
st := llb.Scratch()
3779+
st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
37773780

3778-
def, err := st.Marshal(sb.Context())
3779-
require.NoError(t, err)
3781+
def, err := st.Marshal(sb.Context())
3782+
require.NoError(t, err)
37803783

3781-
registry, err := sb.NewRegistry()
3782-
if errors.Is(err, integration.ErrRequirements) {
3783-
t.Skip(err.Error())
3784-
}
3785-
require.NoError(t, err)
3784+
registry, err := sb.NewRegistry()
3785+
if errors.Is(err, integration.ErrRequirements) {
3786+
t.Skip(err.Error())
3787+
}
3788+
require.NoError(t, err)
37863789

3787-
target := registry + "/buildkit/build/exporter:zstd"
3790+
target := registry + "/buildkit/build/exporter:zstd"
37883791

3789-
_, err = c.Solve(sb.Context(), def, SolveOpt{
3790-
Exports: []ExportEntry{
3791-
{
3792-
Type: ExporterImage,
3793-
Attrs: map[string]string{
3794-
"name": target,
3795-
"push": "true",
3796-
"compression": "zstd",
3792+
_, err = c.Solve(sb.Context(), def, SolveOpt{
3793+
Exports: []ExportEntry{
3794+
{
3795+
Type: ExporterImage,
3796+
Attrs: map[string]string{
3797+
"name": target,
3798+
"push": "true",
3799+
"compression": "zstd",
3800+
"oci-mediatypes": strconv.FormatBool(ociMediaTypes),
3801+
},
3802+
},
3803+
},
3804+
}, nil)
3805+
require.NoError(t, err)
37973806

3798-
// containerd applier supports only zstd with oci-mediatype.
3799-
"oci-mediatypes": "true",
3807+
ensurePruneAll(t, c, sb)
3808+
3809+
st = llb.Image(target).File(llb.Copy(llb.Image(target), "/data", "/zdata"))
3810+
def, err = st.Marshal(sb.Context())
3811+
require.NoError(t, err)
3812+
3813+
destDir := t.TempDir()
3814+
3815+
out := filepath.Join(destDir, "out.tar")
3816+
outW, err := os.Create(out)
3817+
require.NoError(t, err)
3818+
3819+
_, err = c.Solve(sb.Context(), def, SolveOpt{
3820+
Exports: []ExportEntry{
3821+
{
3822+
Type: ExporterOCI,
3823+
Output: fixedWriteCloser(outW),
3824+
Attrs: map[string]string{
3825+
"oci-mediatypes": strconv.FormatBool(ociMediaTypes),
3826+
},
3827+
},
38003828
},
3801-
},
3802-
},
3803-
}, nil)
3804-
require.NoError(t, err)
3829+
}, nil)
3830+
require.NoError(t, err)
38053831

3806-
ensurePruneAll(t, c, sb)
3832+
dt, err := os.ReadFile(out)
3833+
require.NoError(t, err)
38073834

3808-
st = llb.Scratch().File(llb.Copy(llb.Image(target), "/data", "/zdata"))
3835+
m, err := testutil.ReadTarToMap(dt, false)
3836+
require.NoError(t, err)
38093837

3810-
def, err = st.Marshal(sb.Context())
3811-
require.NoError(t, err)
3838+
var index ocispecs.Index
3839+
err = json.Unmarshal(m["index.json"].Data, &index)
3840+
require.NoError(t, err)
38123841

3813-
destDir := t.TempDir()
3842+
var mfst ocispecs.Manifest
3843+
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].Data, &mfst)
3844+
require.NoError(t, err)
38143845

3815-
_, err = c.Solve(sb.Context(), def, SolveOpt{
3816-
Exports: []ExportEntry{
3817-
{
3818-
Type: ExporterLocal,
3819-
OutputDir: destDir,
3820-
},
3821-
},
3822-
}, nil)
3823-
require.NoError(t, err)
3846+
firstLayer := mfst.Layers[0]
3847+
if ociMediaTypes {
3848+
require.Equal(t, ocispecs.MediaTypeImageLayer+"+zstd", firstLayer.MediaType)
3849+
} else {
3850+
require.Equal(t, images.MediaTypeDockerSchema2Layer+".zstd", firstLayer.MediaType)
3851+
}
38243852

3825-
dt, err := os.ReadFile(filepath.Join(destDir, "zdata"))
3826-
require.NoError(t, err)
3827-
require.Equal(t, dt, []byte("zstd"))
3853+
zstdLayerDigest := firstLayer.Digest.Hex()
3854+
require.Equal(t, m["blobs/sha256/"+zstdLayerDigest].Data[:4], []byte{0x28, 0xb5, 0x2f, 0xfd})
3855+
})
3856+
}
38283857
}
38293858
func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
38303859
integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush)

‎util/compression/compression.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ func (c Config) SetLevel(l int) Config {
7878

7979
const (
8080
mediaTypeDockerSchema2LayerZstd = images.MediaTypeDockerSchema2Layer + ".zstd"
81-
mediaTypeImageLayerZstd = ocispecs.MediaTypeImageLayer + "+zstd" // unreleased image-spec#790
8281
)
8382

8483
var Default = Gzip
@@ -104,7 +103,7 @@ func fromMediaType(mediaType string) (Type, error) {
104103
return Uncompressed, nil
105104
case ocispecs.MediaTypeImageLayerGzip, ocispecs.MediaTypeImageLayerNonDistributableGzip: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
106105
return Gzip, nil
107-
case mediaTypeImageLayerZstd, ocispecs.MediaTypeImageLayerNonDistributableZstd: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
106+
case ocispecs.MediaTypeImageLayerZstd, ocispecs.MediaTypeImageLayerNonDistributableZstd: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
108107
return Zstd, nil
109108
default:
110109
return nil, errors.Errorf("unsupported media type %s", mediaType)
@@ -193,7 +192,7 @@ var toDockerLayerType = map[string]string{
193192
images.MediaTypeDockerSchema2LayerForeignGzip: images.MediaTypeDockerSchema2LayerForeignGzip,
194193
ocispecs.MediaTypeImageLayerNonDistributable: images.MediaTypeDockerSchema2LayerForeign, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
195194
ocispecs.MediaTypeImageLayerNonDistributableGzip: images.MediaTypeDockerSchema2LayerForeignGzip, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
196-
mediaTypeImageLayerZstd: mediaTypeDockerSchema2LayerZstd,
195+
ocispecs.MediaTypeImageLayerZstd: mediaTypeDockerSchema2LayerZstd,
197196
mediaTypeDockerSchema2LayerZstd: mediaTypeDockerSchema2LayerZstd,
198197
}
199198

@@ -207,8 +206,8 @@ var toOCILayerType = map[string]string{
207206
images.MediaTypeDockerSchema2LayerGzip: ocispecs.MediaTypeImageLayerGzip,
208207
images.MediaTypeDockerSchema2LayerForeign: ocispecs.MediaTypeImageLayerNonDistributable, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
209208
images.MediaTypeDockerSchema2LayerForeignGzip: ocispecs.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
210-
mediaTypeImageLayerZstd: mediaTypeImageLayerZstd,
211-
mediaTypeDockerSchema2LayerZstd: mediaTypeImageLayerZstd,
209+
ocispecs.MediaTypeImageLayerZstd: ocispecs.MediaTypeImageLayerZstd,
210+
mediaTypeDockerSchema2LayerZstd: ocispecs.MediaTypeImageLayerZstd,
212211
}
213212

214213
func convertLayerMediaType(ctx context.Context, mediaType string, oci bool) string {

‎util/compression/zstd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func (c zstdType) NeedsForceCompression() bool {
4747
}
4848

4949
func (c zstdType) MediaType() string {
50-
return mediaTypeImageLayerZstd
50+
return ocispecs.MediaTypeImageLayerZstd
5151
}
5252

5353
func (c zstdType) String() string {

‎util/winlayers/applier.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,12 @@ type winApplier struct {
3737
}
3838

3939
func (s *winApplier) Apply(ctx context.Context, desc ocispecs.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispecs.Descriptor, err error) {
40+
// HACK:, containerd doesn't know about vnd.docker.image.rootfs.diff.tar.zstd, but that
41+
// media type is compatible w/ the oci type, so just lie and say it's the oci type
42+
if desc.MediaType == images.MediaTypeDockerSchema2Layer+".zstd" {
43+
desc.MediaType = ocispecs.MediaTypeImageLayerZstd
44+
}
45+
4046
if !hasWindowsLayerMode(ctx) {
4147
return s.apply(ctx, desc, mounts, opts...)
4248
}

0 commit comments

Comments
 (0)