applier: add hack to support docker zstd layers

Before this, buildkit was able to create and push layers of type
vnd.docker.image.rootfs.diff.tar.zstd but if you tried to then pull and
unpack those layers in buildkit, you'd get an error from containerd:
`failed to get stream processor for application/vnd.docker.image.rootfs.diff.tar.zstd: no processor for media-type`

While that media type is not official, support for exporting it was
added to buildkit in the past anyways since it works in practice and is
accepted by many registries. It thus seems logical that buildkit should
also support pulling and unpacking those layers too.

There is support for registering stream processors in containerd, but
those only work when using the OCI worker since it relies on the apply
implementation being in the same process as buildkitd.

As a fallback, we instead just implement a hack that swaps out the
docker zstd media type for the oci one before sending it to containerd.
This works in practice because the two types are compatible.

Signed-off-by: Erik Sipsma <[email protected]>

Author: sipsma

client/client_test.go

@@ -3765,66 +3765,95 @@ func testBuildExportZstd(t *testing.T, sb integration.Sandbox) {
 
 func testPullZstdImage(t *testing.T, sb integration.Sandbox) {
 	integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush)
-	c, err := New(sb.Context(), sb.Address())
-	require.NoError(t, err)
-	defer c.Close()
+	for _, ociMediaTypes := range []bool{true, false} {
+		ociMediaTypes := ociMediaTypes
+		t.Run(t.Name()+fmt.Sprintf("/ociMediaTypes=%t", ociMediaTypes), func(t *testing.T) {
+			c, err := New(sb.Context(), sb.Address())
+			require.NoError(t, err)
+			defer c.Close()
 
-	busybox := llb.Image("busybox:latest")
-	cmd := `sh -e -c "echo -n zstd > data"`
+			busybox := llb.Image("busybox:latest")
+			cmd := `sh -e -c "echo -n zstd > data"`
 
-	st := llb.Scratch()
-	st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
+			st := llb.Scratch()
+			st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
 
-	def, err := st.Marshal(sb.Context())
-	require.NoError(t, err)
+			def, err := st.Marshal(sb.Context())
+			require.NoError(t, err)
 
-	registry, err := sb.NewRegistry()
-	if errors.Is(err, integration.ErrRequirements) {
-		t.Skip(err.Error())
-	}
-	require.NoError(t, err)
+			registry, err := sb.NewRegistry()
+			if errors.Is(err, integration.ErrRequirements) {
+				t.Skip(err.Error())
+			}
+			require.NoError(t, err)
 
-	target := registry + "/buildkit/build/exporter:zstd"
+			target := registry + "/buildkit/build/exporter:zstd"
 
-	_, err = c.Solve(sb.Context(), def, SolveOpt{
-		Exports: []ExportEntry{
-			{
-				Type: ExporterImage,
-				Attrs: map[string]string{
-					"name":        target,
-					"push":        "true",
-					"compression": "zstd",
+			_, err = c.Solve(sb.Context(), def, SolveOpt{
+				Exports: []ExportEntry{
+					{
+						Type: ExporterImage,
+						Attrs: map[string]string{
+							"name":           target,
+							"push":           "true",
+							"compression":    "zstd",
+							"oci-mediatypes": strconv.FormatBool(ociMediaTypes),
+						},
+					},
+				},
+			}, nil)
+			require.NoError(t, err)
 
-					// containerd applier supports only zstd with oci-mediatype.
-					"oci-mediatypes": "true",
+			ensurePruneAll(t, c, sb)
+
+			st = llb.Image(target).File(llb.Copy(llb.Image(target), "/data", "/zdata"))
+			def, err = st.Marshal(sb.Context())
+			require.NoError(t, err)
+
+			destDir := t.TempDir()
+
+			out := filepath.Join(destDir, "out.tar")
+			outW, err := os.Create(out)
+			require.NoError(t, err)
+
+			_, err = c.Solve(sb.Context(), def, SolveOpt{
+				Exports: []ExportEntry{
+					{
+						Type:   ExporterOCI,
+						Output: fixedWriteCloser(outW),
+						Attrs: map[string]string{
+							"oci-mediatypes": strconv.FormatBool(ociMediaTypes),
+						},
+					},
 				},
-			},
-		},
-	}, nil)
-	require.NoError(t, err)
+			}, nil)
+			require.NoError(t, err)
 
-	ensurePruneAll(t, c, sb)
+			dt, err := os.ReadFile(out)
+			require.NoError(t, err)
 
-	st = llb.Scratch().File(llb.Copy(llb.Image(target), "/data", "/zdata"))
+			m, err := testutil.ReadTarToMap(dt, false)
+			require.NoError(t, err)
 
-	def, err = st.Marshal(sb.Context())
-	require.NoError(t, err)
+			var index ocispecs.Index
+			err = json.Unmarshal(m["index.json"].Data, &index)
+			require.NoError(t, err)
 
-	destDir := t.TempDir()
+			var mfst ocispecs.Manifest
+			err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].Data, &mfst)
+			require.NoError(t, err)
 
-	_, err = c.Solve(sb.Context(), def, SolveOpt{
-		Exports: []ExportEntry{
-			{
-				Type:      ExporterLocal,
-				OutputDir: destDir,
-			},
-		},
-	}, nil)
-	require.NoError(t, err)
+			firstLayer := mfst.Layers[0]
+			if ociMediaTypes {
+				require.Equal(t, ocispecs.MediaTypeImageLayer+"+zstd", firstLayer.MediaType)
+			} else {
+				require.Equal(t, images.MediaTypeDockerSchema2Layer+".zstd", firstLayer.MediaType)
+			}
 
-	dt, err := os.ReadFile(filepath.Join(destDir, "zdata"))
-	require.NoError(t, err)
-	require.Equal(t, dt, []byte("zstd"))
+			zstdLayerDigest := firstLayer.Digest.Hex()
+			require.Equal(t, m["blobs/sha256/"+zstdLayerDigest].Data[:4], []byte{0x28, 0xb5, 0x2f, 0xfd})
+		})
+	}
 }
 func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush)

util/compression/compression.go

@@ -78,7 +78,6 @@ func (c Config) SetLevel(l int) Config {
 
 const (
 	mediaTypeDockerSchema2LayerZstd = images.MediaTypeDockerSchema2Layer + ".zstd"
-	mediaTypeImageLayerZstd         = ocispecs.MediaTypeImageLayer + "+zstd" // unreleased image-spec#790
 )
 
 var Default = Gzip
@@ -104,7 +103,7 @@ func fromMediaType(mediaType string) (Type, error) {
 		return Uncompressed, nil
 	case ocispecs.MediaTypeImageLayerGzip, ocispecs.MediaTypeImageLayerNonDistributableGzip: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 		return Gzip, nil
-	case mediaTypeImageLayerZstd, ocispecs.MediaTypeImageLayerNonDistributableZstd: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
+	case ocispecs.MediaTypeImageLayerZstd, ocispecs.MediaTypeImageLayerNonDistributableZstd: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 		return Zstd, nil
 	default:
 		return nil, errors.Errorf("unsupported media type %s", mediaType)
@@ -193,7 +192,7 @@ var toDockerLayerType = map[string]string{
 	images.MediaTypeDockerSchema2LayerForeignGzip:    images.MediaTypeDockerSchema2LayerForeignGzip,
 	ocispecs.MediaTypeImageLayerNonDistributable:     images.MediaTypeDockerSchema2LayerForeign,     //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 	ocispecs.MediaTypeImageLayerNonDistributableGzip: images.MediaTypeDockerSchema2LayerForeignGzip, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
-	mediaTypeImageLayerZstd:                          mediaTypeDockerSchema2LayerZstd,
+	ocispecs.MediaTypeImageLayerZstd:                 mediaTypeDockerSchema2LayerZstd,
 	mediaTypeDockerSchema2LayerZstd:                  mediaTypeDockerSchema2LayerZstd,
 }
 
@@ -207,8 +206,8 @@ var toOCILayerType = map[string]string{
 	images.MediaTypeDockerSchema2LayerGzip:           ocispecs.MediaTypeImageLayerGzip,
 	images.MediaTypeDockerSchema2LayerForeign:        ocispecs.MediaTypeImageLayerNonDistributable,     //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 	images.MediaTypeDockerSchema2LayerForeignGzip:    ocispecs.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
-	mediaTypeImageLayerZstd:                          mediaTypeImageLayerZstd,
-	mediaTypeDockerSchema2LayerZstd:                  mediaTypeImageLayerZstd,
+	ocispecs.MediaTypeImageLayerZstd:                 ocispecs.MediaTypeImageLayerZstd,
+	mediaTypeDockerSchema2LayerZstd:                  ocispecs.MediaTypeImageLayerZstd,
 }
 
 func convertLayerMediaType(ctx context.Context, mediaType string, oci bool) string {

util/compression/zstd.go

@@ -47,7 +47,7 @@ func (c zstdType) NeedsForceCompression() bool {
 }
 
 func (c zstdType) MediaType() string {
-	return mediaTypeImageLayerZstd
+	return ocispecs.MediaTypeImageLayerZstd
 }
 
 func (c zstdType) String() string {

util/winlayers/applier.go

@@ -37,6 +37,12 @@ type winApplier struct {
 }
 
 func (s *winApplier) Apply(ctx context.Context, desc ocispecs.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispecs.Descriptor, err error) {
+	// HACK:, containerd doesn't know about vnd.docker.image.rootfs.diff.tar.zstd, but that
+	// media type is compatible w/ the oci type, so just lie and say it's the oci type
+	if desc.MediaType == images.MediaTypeDockerSchema2Layer+".zstd" {
+		desc.MediaType = ocispecs.MediaTypeImageLayerZstd
+	}
+
 	if !hasWindowsLayerMode(ctx) {
 		return s.apply(ctx, desc, mounts, opts...)
 	}