Merge pull request #3430 from tonistiigi/imagetools-create-platforms

imagetools: add platforms filter to imagetools create

Author: tonistiigi

build/build.go

@@ -723,7 +723,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
 								return err
 							}
 
-							dt, desc, err := itpull.Combine(ctx, srcs, indexAnnotations, false)
+							dt, desc, _, err := itpull.Combine(ctx, srcs, indexAnnotations, false)
 							if err != nil {
 								return err
 							}

commands/imagetools/create.go

@@ -7,13 +7,16 @@ import (
 	"os"
 	"strings"
 
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
 	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/cobrautil/completion"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/cli/cli/command"
+	"github.com/moby/buildkit/util/attestation"
 	"github.com/moby/buildkit/util/progress/progressui"
 	"github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -31,6 +34,7 @@ type createOptions struct {
 	actionAppend bool
 	progress     string
 	preferIndex  bool
+	platforms    []string
 }
 
 func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, args []string) error {
@@ -67,6 +71,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
 		return err
 	}
 
+	platforms, err := parsePlatforms(in.platforms)
+	if err != nil {
+		return err
+	}
+
 	repos := map[string]struct{}{}
 
 	for _, t := range tags {
@@ -160,7 +169,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
 		return errors.Wrapf(err, "failed to parse annotations")
 	}
 
-	dt, desc, err := r.Combine(ctx, srcs, annotations, in.preferIndex)
+	dt, desc, srcMap, err := r.Combine(ctx, srcs, annotations, in.preferIndex)
+	if err != nil {
+		return err
+	}
+
+	dt, desc, manifests, err := filterPlatforms(dt, desc, srcMap, platforms)
 	if err != nil {
 		return err
 	}
@@ -170,6 +184,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
 		return nil
 	}
 
+	// manifests can be nil only if pushing one single-platform desc directly
+	if manifests == nil {
+		manifests = []descWithSource{{Descriptor: desc, Source: srcs[0]}}
+	}
+
 	// new resolver cause need new auth
 	r = imagetools.New(imageopt)
 
@@ -187,17 +206,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
 		eg.Go(func() error {
 			return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error {
 				eg2, _ := errgroup.WithContext(ctx)
-				for _, s := range srcs {
-					if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) {
-						continue
-					}
-					s := s
+				for _, desc := range manifests {
 					eg2.Go(func() error {
-						sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String()))
-						return r.Copy(ctx, s, t)
+						sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", desc.Digest.String(), desc.Source.Ref.String(), t.String()))
+						return r.Copy(ctx, desc.Source, t)
 					})
 				}
-
 				if err := eg2.Wait(); err != nil {
 					return err
 				}
@@ -216,6 +230,107 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
 	return err
 }
 
+type descWithSource struct {
+	ocispecs.Descriptor
+	Source *imagetools.Source
+}
+
+func filterPlatforms(dt []byte, desc ocispecs.Descriptor, srcMap map[digest.Digest]*imagetools.Source, plats []ocispecs.Platform) ([]byte, ocispecs.Descriptor, []descWithSource, error) {
+	if len(plats) == 0 {
+		return dt, desc, nil, nil
+	}
+
+	matcher := platforms.Any(plats...)
+
+	if !images.IsIndexType(desc.MediaType) {
+		var mfst ocispecs.Manifest
+		if err := json.Unmarshal(dt, &mfst); err != nil {
+			return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse manifest")
+		}
+		if desc.Platform == nil {
+			return nil, ocispecs.Descriptor{}, nil, errors.Errorf("cannot filter platforms from a manifest without platform information")
+		}
+		if !matcher.Match(*desc.Platform) {
+			return nil, ocispecs.Descriptor{}, nil, errors.Errorf("input platform %s does not match any of the provided platforms", platforms.Format(*desc.Platform))
+		}
+		return dt, desc, nil, nil
+	}
+
+	var idx ocispecs.Index
+	if err := json.Unmarshal(dt, &idx); err != nil {
+		return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse index")
+	}
+
+	manifestMap := map[digest.Digest]ocispecs.Descriptor{}
+	for _, m := range idx.Manifests {
+		manifestMap[m.Digest] = m
+	}
+	references := map[digest.Digest]struct{}{}
+	for _, m := range idx.Manifests {
+		if refType, ok := m.Annotations[attestation.DockerAnnotationReferenceType]; ok && refType == attestation.DockerAnnotationReferenceTypeDefault {
+			dgstStr, ok := m.Annotations[attestation.DockerAnnotationReferenceDigest]
+			if !ok {
+				continue
+			}
+			dgst, err := digest.Parse(dgstStr)
+			if err != nil {
+				continue
+			}
+			subject, ok := manifestMap[dgst]
+			if !ok {
+				continue
+			}
+			if subject.Platform == nil || matcher.Match(*subject.Platform) {
+				references[m.Digest] = struct{}{}
+			}
+		}
+	}
+
+	var mfsts []ocispecs.Descriptor
+	var mfstsWithSource []descWithSource
+
+	for _, m := range idx.Manifests {
+		if _, isRef := references[m.Digest]; isRef || m.Platform == nil || matcher.Match(*m.Platform) {
+			src, ok := srcMap[m.Digest]
+			if !ok {
+				defaultSource, ok := srcMap[desc.Digest]
+				if !ok {
+					return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", m.Digest)
+				}
+				src = defaultSource
+			}
+			mfsts = append(mfsts, m)
+			mfstsWithSource = append(mfstsWithSource, descWithSource{
+				Descriptor: m,
+				Source:     src,
+			})
+		}
+	}
+	if len(mfsts) == len(idx.Manifests) {
+		// all platforms matched, no need to rewrite index
+		return dt, desc, mfstsWithSource, nil
+	}
+
+	if len(mfsts) == 0 {
+		return nil, ocispecs.Descriptor{}, nil, errors.Errorf("none of the manifests match the provided platforms")
+	}
+
+	idx.Manifests = mfsts
+	idxBytes, err := json.MarshalIndent(&idx, "", "  ")
+	if err != nil {
+		return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index")
+	}
+
+	desc = ocispecs.Descriptor{
+		MediaType:   desc.MediaType,
+		Size:        int64(len(idxBytes)),
+		Digest:      digest.FromBytes(idxBytes),
+		Annotations: desc.Annotations,
+	}
+
+	return idxBytes, desc, mfstsWithSource, nil
+}
+
 func parseSources(in []string) ([]*imagetools.Source, error) {
 	out := make([]*imagetools.Source, len(in))
 	for i, in := range in {
@@ -228,6 +343,26 @@ func parseSources(in []string) ([]*imagetools.Source, error) {
 	return out, nil
 }
 
+func parsePlatforms(in []string) ([]ocispecs.Platform, error) {
+	out := make([]ocispecs.Platform, 0, len(in))
+	for _, p := range in {
+		if arr := strings.Split(p, ","); len(arr) > 1 {
+			v, err := parsePlatforms(arr)
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, v...)
+			continue
+		}
+		plat, err := platforms.Parse(p)
+		if err != nil {
+			return nil, errors.Wrapf(err, "invalid platform %q", p)
+		}
+		out = append(out, plat)
+	}
+	return out, nil
+}
+
 func parseRefs(in []string) ([]reference.Named, error) {
 	refs := make([]reference.Named, len(in))
 	for i, in := range in {
@@ -291,6 +426,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
 	flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty", "rawjson"). Use plain to show container output`)
 	flags.StringArrayVarP(&options.annotations, "annotation", "", []string{}, "Add annotation to the image")
 	flags.BoolVar(&options.preferIndex, "prefer-index", true, "When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy")
+	flags.StringArrayVarP(&options.platforms, "platform", "p", []string{}, "Filter specified platforms of target image")
 
 	return cmd
 }

docs/reference/buildx_imagetools_create.md

@@ -17,6 +17,7 @@ Create a new image based on source images
 | `-D`, `--debug`                  | `bool`        |         | Enable debug logging                                                                                                          |
 | [`--dry-run`](#dry-run)          | `bool`        |         | Show final image instead of pushing                                                                                           |
 | [`-f`](#file), [`--file`](#file) | `stringArray` |         | Read source descriptor from file                                                                                              |
+| `-p`, `--platform`               | `stringArray` |         | Filter specified platforms of target image                                                                                    |
 | `--prefer-index`                 | `bool`        | `true`  | When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy |
 | `--progress`                     | `string`      | `auto`  | Set type of progress output (`auto`, `plain`, `tty`, `rawjson`). Use plain to show container output                           |
 | [`-t`](#tag), [`--tag`](#tag)    | `stringArray` |         | Set reference for new image                                                                                                   |

util/imagetools/create.go

@@ -28,7 +28,7 @@ type Source struct {
 	Ref  reference.Named
 }
 
-func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, error) {
+func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, map[digest.Digest]*Source, error) {
 	eg, ctx := errgroup.WithContext(ctx)
 
 	dts := make([][]byte, len(srcs))
@@ -73,7 +73,7 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
 	}
 
 	if err := eg.Wait(); err != nil {
-		return nil, ocispecs.Descriptor{}, err
+		return nil, ocispecs.Descriptor{}, nil, err
 	}
 
 	// on single source, return original bytes
@@ -83,21 +83,25 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
 		// of preferIndex since if set to true then the source is already in the preferred format, and if false
 		// it doesn't matter since we're not going to split it into separate manifests
 		case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
-			return dts[0], srcs[0].Desc, nil
+			srcMap := map[digest.Digest]*Source{
+				srcs[0].Desc.Digest: srcs[0],
+			}
+			return dts[0], srcs[0].Desc, srcMap, nil
 		default:
 			if !preferIndex {
-				return dts[0], srcs[0].Desc, nil
+				return dts[0], srcs[0].Desc, nil, nil
 			}
 		}
 	}
 
-	m := map[digest.Digest]int{}
-	newDescs := make([]ocispecs.Descriptor, 0, len(srcs))
+	indexes := map[digest.Digest]int{}
+	sources := map[digest.Digest]*Source{}
+	descs := make([]ocispecs.Descriptor, 0, len(srcs))
 
-	addDesc := func(d ocispecs.Descriptor) {
-		idx, ok := m[d.Digest]
+	addDesc := func(d ocispecs.Descriptor, src *Source) {
+		idx, ok := indexes[d.Digest]
 		if ok {
-			old := newDescs[idx]
+			old := descs[idx]
 			if old.MediaType == "" {
 				old.MediaType = d.MediaType
 			}
@@ -108,37 +112,38 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
 				old.Annotations = map[string]string{}
 			}
 			maps.Copy(old.Annotations, d.Annotations)
-			newDescs[idx] = old
+			descs[idx] = old
 		} else {
-			m[d.Digest] = len(newDescs)
-			newDescs = append(newDescs, d)
+			indexes[d.Digest] = len(descs)
+			descs = append(descs, d)
 		}
+		sources[d.Digest] = src
 	}
 
 	for i, src := range srcs {
 		switch src.Desc.MediaType {
 		case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
 			var mfst ocispecs.Index
 			if err := json.Unmarshal(dts[i], &mfst); err != nil {
-				return nil, ocispecs.Descriptor{}, errors.WithStack(err)
+				return nil, ocispecs.Descriptor{}, nil, errors.WithStack(err)
 			}
 			for _, d := range mfst.Manifests {
-				addDesc(d)
+				addDesc(d, src)
 			}
 		default:
-			addDesc(src.Desc)
+			addDesc(src.Desc, src)
 		}
 	}
 
 	dockerMfsts := 0
-	for _, desc := range newDescs {
+	for _, desc := range descs {
 		if strings.HasPrefix(desc.MediaType, "application/vnd.docker.") {
 			dockerMfsts++
 		}
 	}
 
 	var mt string
-	if dockerMfsts == len(newDescs) {
+	if dockerMfsts == len(descs) {
 		// all manifests are Docker types, use Docker manifest list
 		mt = images.MediaTypeDockerSchema2ManifestList
 	} else {
@@ -154,18 +159,18 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
 			case exptypes.AnnotationIndex:
 				indexAnnotation[k.Key] = v
 			case exptypes.AnnotationManifestDescriptor:
-				for i := range newDescs {
-					if newDescs[i].Annotations == nil {
-						newDescs[i].Annotations = map[string]string{}
+				for i := range descs {
+					if descs[i].Annotations == nil {
+						descs[i].Annotations = map[string]string{}
 					}
-					if k.Platform == nil || k.PlatformString() == platforms.Format(*newDescs[i].Platform) {
-						newDescs[i].Annotations[k.Key] = v
+					if k.Platform == nil || k.PlatformString() == platforms.Format(*descs[i].Platform) {
+						descs[i].Annotations[k.Key] = v
 					}
 				}
 			case exptypes.AnnotationManifest, "":
-				return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are not supported yet", k.Type)
+				return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are not supported yet", k.Type)
 			case exptypes.AnnotationIndexDescriptor:
-				return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are invalid while creating an image", k.Type)
+				return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are invalid while creating an image", k.Type)
 			}
 		}
 	}
@@ -175,18 +180,18 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
 		Versioned: specs.Versioned{
 			SchemaVersion: 2,
 		},
-		Manifests:   newDescs,
+		Manifests:   descs,
 		Annotations: indexAnnotation,
 	}, "", "  ")
 	if err != nil {
-		return nil, ocispecs.Descriptor{}, errors.Wrap(err, "failed to marshal index")
+		return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index")
 	}
 
 	return idxBytes, ocispecs.Descriptor{
 		MediaType: mt,
 		Size:      int64(len(idxBytes)),
 		Digest:    digest.FromBytes(idxBytes),
-	}, nil
+	}, sources, nil
 }
 
 func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispecs.Descriptor, dt []byte) error {