added import/export support for OCI compatible image manifest version of cache manifest (opt-in on export, inferred on import) moby/buildkit moby#2251

Signed-off-by: Kang, Matthew <[email protected]>

Author: kattmang

README.md

@@ -388,6 +388,7 @@ buildctl build ... \
   * `min`: only export layers for the resulting image
   * `max`: export all the layers of all intermediate steps
 * `ref=<ref>`: specify repository reference to store cache, e.g. `docker.io/user/image:tag`
+* `image-manifest=<true|false>`: whether to export cache manifest as an OCI-compatible image manifest rather than a manifest list/index (default: `false`)
 * `oci-mediatypes=<true|false>`: whether to use OCI mediatypes in exported manifests (default: `true`, since BuildKit `v0.8`)
 * `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`
 * `compression-level=<value>`: choose compression level for gzip, estargz (0-9) and zstd (0-22)
@@ -414,6 +415,7 @@ The directory layout conforms to OCI Image Spec v1.0.
   * `max`: export all the layers of all intermediate steps
 * `dest=<path>`: destination directory for cache exporter
 * `tag=<tag>`: specify custom tag of image to write to local index (default: `latest`)
+* `image-manifest=<true|false>`: whether to export cache manifest as an OCI-compatible image manifest rather than a manifest list/index (default: `false`)
 * `oci-mediatypes=<true|false>`: whether to use OCI mediatypes in exported manifests (default `true`, since BuildKit `v0.8`)
 * `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`.
 * `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)

cache/remotecache/export.go

@@ -43,18 +43,19 @@ const (
 	ExporterResponseManifestDesc = "cache.manifest"
 )
 
-type contentCacheExporter struct {
-	solver.CacheExporterTarget
-	chains   *v1.CacheChains
-	ingester content.Ingester
-	oci      bool
-	ref      string
-	comp     compression.Config
+func NewExporter(ingester content.Ingester, ref string, oci bool, imageManifest bool, compressionConfig compression.Config) Exporter {
+	cc := v1.NewCacheChains()
+	return &contentCacheExporter{CacheExporterTarget: cc, chains: cc, ingester: ingester, oci: oci, imageManifest: imageManifest, ref: ref, comp: compressionConfig}
 }
 
-func NewExporter(ingester content.Ingester, ref string, oci bool, compressionConfig compression.Config) Exporter {
-	cc := v1.NewCacheChains()
-	return &contentCacheExporter{CacheExporterTarget: cc, chains: cc, ingester: ingester, oci: oci, ref: ref, comp: compressionConfig}
+type contentCacheExporter struct {
+	solver.CacheExporterTarget
+	chains        *v1.CacheChains
+	ingester      content.Ingester
+	oci           bool
+	imageManifest bool
+	ref           string
+	comp          compression.Config
 }
 
 func (ce *contentCacheExporter) Name() string {
@@ -75,20 +76,23 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string
 	}
 
 	// own type because oci type can't be pushed and docker type doesn't have annotations
-	type manifestList struct {
+	type abstractManifest struct {
 		specs.Versioned
 
-		MediaType string `json:"mediaType,omitempty"`
-
+		MediaType string               `json:"mediaType,omitempty"`
+		Config    *ocispecs.Descriptor `json:"config,omitempty"`
 		// Manifests references platform specific manifests.
-		Manifests []ocispecs.Descriptor `json:"manifests"`
+		Manifests []ocispecs.Descriptor `json:"manifests,omitempty"`
+		Layers    []ocispecs.Descriptor `json:"layers,omitempty"`
 	}
 
-	var mfst manifestList
+	var mfst abstractManifest
 	mfst.SchemaVersion = 2
 	mfst.MediaType = images.MediaTypeDockerSchema2ManifestList
-	if ce.oci {
+	if ce.oci && !ce.imageManifest {
 		mfst.MediaType = ocispecs.MediaTypeImageIndex
+	} else if ce.imageManifest {
+		mfst.MediaType = ocispecs.MediaTypeImageManifest
 	}
 
 	for _, l := range config.Layers {
@@ -101,10 +105,16 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string
 			return nil, layerDone(errors.Wrap(err, "error writing layer blob"))
 		}
 		layerDone(nil)
-		mfst.Manifests = append(mfst.Manifests, dgstPair.Descriptor)
+		if ce.imageManifest {
+			mfst.Layers = append(mfst.Layers, dgstPair.Descriptor)
+		} else {
+			mfst.Manifests = append(mfst.Manifests, dgstPair.Descriptor)
+		}
 	}
 
-	mfst.Manifests = compression.ConvertAllLayerMediaTypes(ctx, ce.oci, mfst.Manifests...)
+	if !ce.imageManifest {
+		mfst.Manifests = compression.ConvertAllLayerMediaTypes(ctx, ce.oci, mfst.Manifests...)
+	}
 
 	dt, err := json.Marshal(config)
 	if err != nil {
@@ -122,7 +132,11 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string
 	}
 	configDone(nil)
 
-	mfst.Manifests = append(mfst.Manifests, desc)
+	if ce.imageManifest {
+		mfst.Config = &desc
+	} else {
+		mfst.Manifests = append(mfst.Manifests, desc)
+	}
 
 	dt, err = json.Marshal(mfst)
 	if err != nil {
@@ -135,7 +149,12 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string
 		Size:      int64(len(dt)),
 		MediaType: mfst.MediaType,
 	}
-	mfstDone := progress.OneOff(ctx, fmt.Sprintf("writing manifest %s", dgst))
+
+	mfstLog := fmt.Sprintf("writing cache manifest %s", dgst)
+	if ce.imageManifest {
+		mfstLog = fmt.Sprintf("writing cache image manifest %s", dgst)
+	}
+	mfstDone := progress.OneOff(ctx, mfstLog)
 	if err := content.WriteBlob(ctx, ce.ingester, dgst.String(), bytes.NewReader(dt), desc); err != nil {
 		return nil, mfstDone(errors.Wrap(err, "error writing manifest blob"))
 	}
@@ -145,5 +164,6 @@ func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string
 	}
 	res[ExporterResponseManifestDesc] = string(descJSON)
 	mfstDone(nil)
+
 	return res, nil
 }

cache/remotecache/import.go

@@ -3,10 +3,14 @@ package remotecache
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"sync"
 	"time"
 
+	"github.com/moby/buildkit/util/progress"
+	"github.com/opencontainers/image-spec/specs-go"
+
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
@@ -21,6 +25,27 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
+type ManifestType int
+
+const (
+	NotInferred ManifestType = iota
+	ManifestList
+	ImageManifest
+)
+
+func (data ManifestType) String() string {
+	switch data {
+	case NotInferred:
+		return "Not Inferred"
+	case ManifestList:
+		return "Manifest List"
+	case ImageManifest:
+		return "Image Manifest"
+	default:
+		return "Not Inferred"
+	}
+}
+
 // ResolveCacheImporterFunc returns importer and descriptor.
 type ResolveCacheImporterFunc func(ctx context.Context, g session.Group, attrs map[string]string) (Importer, ocispecs.Descriptor, error)
 
@@ -47,24 +72,52 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
 		return nil, err
 	}
 
-	var mfst ocispecs.Index
-	if err := json.Unmarshal(dt, &mfst); err != nil {
+	manifestType, err := inferManifestType(ctx, dt)
+	if err != nil {
 		return nil, err
 	}
 
-	allLayers := v1.DescriptorProvider{}
+	layerDone := progress.OneOff(ctx, fmt.Sprintf("inferred cache manifest type: %s", manifestType))
+	layerDone(nil)
 
+	allLayers := v1.DescriptorProvider{}
 	var configDesc ocispecs.Descriptor
 
-	for _, m := range mfst.Manifests {
-		if m.MediaType == v1.CacheConfigMediaTypeV0 {
-			configDesc = m
-			continue
+	if manifestType == ManifestList {
+		var mfst ocispecs.Index
+		if err := json.Unmarshal(dt, &mfst); err != nil {
+			return nil, err
+		}
+
+		for _, m := range mfst.Manifests {
+			if m.MediaType == v1.CacheConfigMediaTypeV0 {
+				configDesc = m
+				continue
+			}
+			allLayers[m.Digest] = v1.DescriptorProviderPair{
+				Descriptor: m,
+				Provider:   ci.provider,
+			}
 		}
-		allLayers[m.Digest] = v1.DescriptorProviderPair{
-			Descriptor: m,
-			Provider:   ci.provider,
+	} else if manifestType == ImageManifest {
+		var mfst ocispecs.Manifest
+		if err := json.Unmarshal(dt, &mfst); err != nil {
+			return nil, err
 		}
+
+		for _, m := range mfst.Layers {
+			if m.MediaType == v1.CacheConfigMediaTypeV0 {
+				configDesc = m
+				continue
+			}
+			allLayers[m.Digest] = v1.DescriptorProviderPair{
+				Descriptor: m,
+				Provider:   ci.provider,
+			}
+		}
+	} else {
+		err = errors.Wrapf(err, "Unsupported or uninferrable manifest type")
+		return nil, err
 	}
 
 	if dsls, ok := ci.provider.(DistributionSourceLabelSetter); ok {
@@ -97,6 +150,37 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispecs.Descr
 	return solver.NewCacheManager(ctx, id, keysStorage, resultStorage), nil
 }
 
+// extends support for "new"-style image-manifest style remote cache manifests and determining downstream
+// handling based on inference of document structure (is this a new or old cache manifest type?)
+func inferManifestType(ctx context.Context, dt []byte) (ManifestType, error) {
+	// this is a loose schema superset of both OCI Index and Manifest in order to
+	// be able to poke at the structure of the imported cache manifest
+	type OpenManifest struct {
+		specs.Versioned
+
+		MediaType string                 `json:"mediaType,omitempty"`
+		Config    map[string]interface{} `json:"config,omitempty"`
+		// Manifests references platform specific manifests.
+		Manifests []map[string]interface{} `json:"manifests,omitempty"`
+		Layers    []map[string]interface{} `json:"layers,omitempty"`
+	}
+
+	var openManifest OpenManifest
+	if err := json.Unmarshal(dt, &openManifest); err != nil {
+		return NotInferred, err
+	}
+
+	if len(openManifest.Manifests) == 0 && len(openManifest.Layers) > 0 {
+		return ImageManifest, nil
+	}
+
+	if len(openManifest.Layers) == 0 && len(openManifest.Manifests) > 0 {
+		return ManifestList, nil
+	}
+
+	return NotInferred, nil
+}
+
 func readBlob(ctx context.Context, provider content.Provider, desc ocispecs.Descriptor) ([]byte, error) {
 	maxBlobSize := int64(1 << 20)
 	if desc.Size > maxBlobSize {

cache/remotecache/local/local.go

@@ -19,6 +19,7 @@ const (
 	attrDigest           = "digest"
 	attrSrc              = "src"
 	attrDest             = "dest"
+	attrImageManifest    = "image-manifest"
 	attrOCIMediatypes    = "oci-mediatypes"
 	contentStoreIDPrefix = "local:"
 )
@@ -50,12 +51,20 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor
 			}
 			ociMediatypes = b
 		}
+		imageManifest := false
+		if v, ok := attrs[attrImageManifest]; ok {
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to parse %s", attrImageManifest)
+			}
+			imageManifest = b
+		}
 		csID := contentStoreIDPrefix + store
 		cs, err := getContentStore(ctx, sm, g, csID)
 		if err != nil {
 			return nil, err
 		}
-		return &exporter{remotecache.NewExporter(cs, "", ociMediatypes, compressionConfig)}, nil
+		return &exporter{remotecache.NewExporter(cs, "", ociMediatypes, imageManifest, compressionConfig)}, nil
 	}
 }
 

cache/remotecache/registry/registry.go

@@ -36,6 +36,7 @@ func canonicalizeRef(rawRef string) (reference.Named, error) {
 
 const (
 	attrRef           = "ref"
+	attrImageManifest = "image-manifest"
 	attrOCIMediatypes = "oci-mediatypes"
 	attrInsecure      = "registry.insecure"
 )
@@ -67,6 +68,14 @@ func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) r
 			}
 			ociMediatypes = b
 		}
+		imageManifest := false
+		if v, ok := attrs[attrImageManifest]; ok {
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to parse %s", attrImageManifest)
+			}
+			imageManifest = b
+		}
 		insecure := false
 		if v, ok := attrs[attrInsecure]; ok {
 			b, err := strconv.ParseBool(v)
@@ -82,7 +91,7 @@ func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) r
 		if err != nil {
 			return nil, err
 		}
-		return &exporter{remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, compressionConfig)}, nil
+		return &exporter{remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, imageManifest, compressionConfig)}, nil
 	}
 }