Merge pull request containerd#9554 from vvoland/archive-sparse

import/export: Support references to missing content

Author: mxpv

client/import.go

@@ -37,6 +37,7 @@ type importOpts struct {
 	platformMatcher platforms.MatchComparer
 	compress        bool
 	discardLayers   bool
+	skipMissing     bool
 }
 
 // ImportOpt allows the caller to specify import specific options
@@ -113,6 +114,15 @@ func WithDiscardUnpackedLayers() ImportOpt {
 	}
 }
 
+// WithSkipMissing allows to import an archive which doesn't contain all the
+// referenced blobs.
+func WithSkipMissing() ImportOpt {
+	return func(c *importOpts) error {
+		c.skipMissing = true
+		return nil
+	}
+}
+
 // Import imports an image from a Tar stream using reader.
 // Caller needs to specify importer. Future version may use oci.v1 as the default.
 // Note that unreferenced blobs may be imported to the content store as well.
@@ -162,7 +172,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
 	var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		// Only save images at top level
 		if desc.Digest != index.Digest {
-			return images.Children(ctx, cs, desc)
+			// Don't set labels on missing content.
+			children, err := images.Children(ctx, cs, desc)
+			if iopts.skipMissing && errdefs.IsNotFound(err) {
+				return nil, images.ErrSkipDesc
+			}
+			return children, err
 		}
 
 		idx, err := decodeIndex(ctx, cs, desc)

content/helpers.go

@@ -320,3 +320,14 @@ func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) {
 	}
 	return
 }
+
+// Exists returns whether an attempt to access the content would not error out
+// with an ErrNotFound error. It will return an encountered error if it was
+// different than ErrNotFound.
+func Exists(ctx context.Context, provider InfoProvider, desc ocispec.Descriptor) (bool, error) {
+	_, err := provider.Info(ctx, desc.Digest)
+	if errdefs.IsNotFound(err) {
+		return false, nil
+	}
+	return err == nil, err
+}

images/archive/exporter.go

@@ -24,11 +24,14 @@ import (
 	"io"
 	"path"
 	"sort"
+	"strings"
 
 	"github.com/containerd/containerd/v2/content"
 	"github.com/containerd/containerd/v2/errdefs"
 	"github.com/containerd/containerd/v2/images"
+	"github.com/containerd/containerd/v2/labels"
 	"github.com/containerd/containerd/v2/platforms"
+	"github.com/containerd/log"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -140,6 +143,45 @@ func WithSkipNonDistributableBlobs() ExportOpt {
 	return WithBlobFilter(f)
 }
 
+// WithSkipMissing excludes blobs referenced by manifests if not all blobs
+// would be included in the archive.
+// The manifest itself is excluded only if it's not present locally.
+// This allows to export multi-platform images if not all platforms are present
+// while still persisting the multi-platform index.
+func WithSkipMissing(store ContentProvider) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.blobRecordOptions.childrenHandler = images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
+			children, err := images.Children(ctx, store, desc)
+			if !images.IsManifestType(desc.MediaType) {
+				return children, err
+			}
+
+			if err != nil {
+				// If manifest itself is missing, skip it from export.
+				if errdefs.IsNotFound(err) {
+					return nil, images.ErrSkipDesc
+				}
+				return nil, err
+			}
+
+			// Don't export manifest descendants if any of them doesn't exist.
+			for _, child := range children {
+				exists, err := content.Exists(ctx, store, child)
+				if err != nil {
+					return nil, err
+				}
+
+				// If any child is missing, only export the manifest, but don't export its descendants.
+				if !exists {
+					return nil, nil
+				}
+			}
+			return children, nil
+		})
+		return nil
+	}
+}
+
 func addNameAnnotation(name string, base map[string]string) map[string]string {
 	annotations := map[string]string{}
 	for k, v := range base {
@@ -152,8 +194,31 @@ func addNameAnnotation(name string, base map[string]string) map[string]string {
 	return annotations
 }
 
+func copySourceLabels(ctx context.Context, infoProvider content.InfoProvider, desc ocispec.Descriptor) (ocispec.Descriptor, error) {
+	info, err := infoProvider.Info(ctx, desc.Digest)
+	if err != nil {
+		return desc, err
+	}
+	for k, v := range info.Labels {
+		if strings.HasPrefix(k, labels.LabelDistributionSource) {
+			if desc.Annotations == nil {
+				desc.Annotations = map[string]string{k: v}
+			} else {
+				desc.Annotations[k] = v
+			}
+		}
+	}
+	return desc, nil
+}
+
+// ContentProvider provides both content and info about content
+type ContentProvider interface {
+	content.Provider
+	content.InfoProvider
+}
+
 // Export implements Exporter.
-func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error {
+func Export(ctx context.Context, store ContentProvider, writer io.Writer, opts ...ExportOpt) error {
 	var eo exportOptions
 	for _, opt := range opts {
 		if err := opt(ctx, &eo); err != nil {
@@ -163,13 +228,22 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts
 
 	records := []tarRecord{
 		ociLayoutFile(""),
-		ociIndexRecord(eo.manifests),
+	}
+
+	manifests := make([]ocispec.Descriptor, 0, len(eo.manifests))
+	for _, desc := range eo.manifests {
+		d, err := copySourceLabels(ctx, store, desc)
+		if err != nil {
+			log.G(ctx).WithError(err).WithField("desc", desc).Warn("failed to copy distribution.source labels")
+			continue
+		}
+		manifests = append(manifests, d)
 	}
 
 	algorithms := map[string]struct{}{}
 	dManifests := map[digest.Digest]*exportManifest{}
 	resolvedIndex := map[digest.Digest]digest.Digest{}
-	for _, desc := range eo.manifests {
+	for _, desc := range manifests {
 		if images.IsManifestType(desc.MediaType) {
 			mt, ok := dManifests[desc.Digest]
 			if !ok {
@@ -259,6 +333,8 @@ func Export(ctx context.Context, store content.Provider, writer io.Writer, opts
 		}
 	}
 
+	records = append(records, ociIndexRecord(manifests))
+
 	if !eo.skipDockerManifest && len(dManifests) > 0 {
 		tr, err := manifestsRecord(ctx, store, dManifests)
 		if err != nil {
@@ -291,7 +367,10 @@ func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descri
 		return nil, nil
 	}
 
-	childrenHandler := images.ChildrenHandler(store)
+	childrenHandler := brOpts.childrenHandler
+	if childrenHandler == nil {
+		childrenHandler = images.ChildrenHandler(store)
+	}
 
 	handlers := images.Handlers(
 		childrenHandler,
@@ -313,7 +392,8 @@ type tarRecord struct {
 }
 
 type blobRecordOptions struct {
-	blobFilter BlobFilter
+	blobFilter      BlobFilter
+	childrenHandler images.HandlerFunc
 }
 
 func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOptions) tarRecord {