Merge pull request moby#3161 from crazy-max/local-exporter-wrap

export(local): split opt

Author: tonistiigi

README.md

@@ -293,6 +293,50 @@ COPY --from=builder /usr/src/app/testresult.xml .
 buildctl build ... --opt target=testresult --output type=local,dest=path/to/output-dir
 ```
 
+With a [multi-platform build](docs/multi-platform.md), a subfolder matching
+each target platform will be created in the destination directory:
+
+```dockerfile
+FROM busybox AS build
+ARG TARGETOS
+ARG TARGETARCH
+RUN mkdir /out && echo foo > /out/hello-$TARGETOS-$TARGETARCH
+
+FROM scratch
+COPY --from=build /out /
+```
+
+```bash
+$ buildctl build \
+  --frontend dockerfile.v0 \
+  --opt platform=linux/amd64,linux/arm64 \
+  --output type=local,dest=./bin/release
+
+$ tree ./bin
+./bin/
+└── release
+    ├── linux_amd64
+    │   └── hello-linux-amd64
+    └── linux_arm64
+        └── hello-linux-arm64
+```
+
+You can set `platform-split=false` to merge files from all platforms together
+into same directory:
+
+```bash
+$ buildctl build \
+  --frontend dockerfile.v0 \
+  --opt platform=linux/amd64,linux/arm64 \
+  --output type=local,dest=./bin/release,platform-split=false
+
+$ tree ./bin
+./bin/
+└── release
+    ├── hello-linux-amd64
+    └── hello-linux-arm64
+```
+
 Tar exporter is similar to local exporter but transfers the files through a tarball.
 
 ```bash

client/client_test.go

@@ -203,6 +203,8 @@ func TestIntegration(t *testing.T) {
 		testLLBMountPerformance,
 		testClientCustomGRPCOpts,
 		testMultipleRecordsWithSameLayersCacheImportExport,
+		testExportLocalNoPlatformSplit,
+		testExportLocalNoPlatformSplitOverwrite,
 	)
 }
 
@@ -5440,6 +5442,152 @@ func testMultipleRecordsWithSameLayersCacheImportExport(t *testing.T, sb integra
 	ensurePruneAll(t, c, sb)
 }
 
+func testExportLocalNoPlatformSplit(t *testing.T, sb integration.Sandbox) {
+	integration.CheckFeatureCompat(t, sb, integration.FeatureOCIExporter, integration.FeatureMultiPlatform)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	platformsToTest := []string{"linux/amd64", "linux/arm64"}
+	frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		res := gateway.NewResult()
+		expPlatforms := &exptypes.Platforms{
+			Platforms: make([]exptypes.Platform, len(platformsToTest)),
+		}
+		for i, platform := range platformsToTest {
+			st := llb.Scratch().File(
+				llb.Mkfile("hello-"+strings.ReplaceAll(platform, "/", "-"), 0600, []byte(platform)),
+			)
+
+			def, err := st.Marshal(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			r, err := c.Solve(ctx, gateway.SolveRequest{
+				Definition: def.ToPB(),
+			})
+			if err != nil {
+				return nil, err
+			}
+
+			ref, err := r.SingleRef()
+			if err != nil {
+				return nil, err
+			}
+
+			_, err = ref.ToState()
+			if err != nil {
+				return nil, err
+			}
+			res.AddRef(platform, ref)
+
+			expPlatforms.Platforms[i] = exptypes.Platform{
+				ID:       platform,
+				Platform: platforms.MustParse(platform),
+			}
+		}
+		dt, err := json.Marshal(expPlatforms)
+		if err != nil {
+			return nil, err
+		}
+		res.AddMeta(exptypes.ExporterPlatformsKey, dt)
+
+		return res, nil
+	}
+
+	destDir := t.TempDir()
+	_, err = c.Build(sb.Context(), SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+				Attrs: map[string]string{
+					"platform-split": "false",
+				},
+			},
+		},
+	}, "", frontend, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(filepath.Join(destDir, "hello-linux-amd64"))
+	require.NoError(t, err)
+	require.Equal(t, "linux/amd64", string(dt))
+
+	dt, err = os.ReadFile(filepath.Join(destDir, "hello-linux-arm64"))
+	require.NoError(t, err)
+	require.Equal(t, "linux/arm64", string(dt))
+}
+
+func testExportLocalNoPlatformSplitOverwrite(t *testing.T, sb integration.Sandbox) {
+	integration.CheckFeatureCompat(t, sb, integration.FeatureOCIExporter, integration.FeatureMultiPlatform)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	platformsToTest := []string{"linux/amd64", "linux/arm64"}
+	frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
+		res := gateway.NewResult()
+		expPlatforms := &exptypes.Platforms{
+			Platforms: make([]exptypes.Platform, len(platformsToTest)),
+		}
+		for i, platform := range platformsToTest {
+			st := llb.Scratch().File(
+				llb.Mkfile("hello-linux", 0600, []byte(platform)),
+			)
+
+			def, err := st.Marshal(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			r, err := c.Solve(ctx, gateway.SolveRequest{
+				Definition: def.ToPB(),
+			})
+			if err != nil {
+				return nil, err
+			}
+
+			ref, err := r.SingleRef()
+			if err != nil {
+				return nil, err
+			}
+
+			_, err = ref.ToState()
+			if err != nil {
+				return nil, err
+			}
+			res.AddRef(platform, ref)
+
+			expPlatforms.Platforms[i] = exptypes.Platform{
+				ID:       platform,
+				Platform: platforms.MustParse(platform),
+			}
+		}
+		dt, err := json.Marshal(expPlatforms)
+		if err != nil {
+			return nil, err
+		}
+		res.AddMeta(exptypes.ExporterPlatformsKey, dt)
+
+		return res, nil
+	}
+
+	destDir := t.TempDir()
+	_, err = c.Build(sb.Context(), SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+				Attrs: map[string]string{
+					"platform-split": "false",
+				},
+			},
+		},
+	}, "", frontend, nil)
+	require.Error(t, err)
+}
+
 func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) {
 	def, err := llb.Image(ref).Marshal(ctx)
 	if err != nil {

exporter/local/export.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/moby/buildkit/cache"
@@ -92,6 +93,9 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 
 	now := time.Now().Truncate(time.Second)
 
+	visitedPath := map[string]string{}
+	var visitedMu sync.Mutex
+
 	export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) func() error {
 		return func() error {
 			outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, attestations, now, e.opts)
@@ -102,20 +106,43 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 				defer cleanup()
 			}
 
+			if !e.opts.PlatformSplit {
+				// check for duplicate paths
+				err = outputFS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
+					if fi.IsDir() {
+						return nil
+					}
+					if err != nil && !errors.Is(err, os.ErrNotExist) {
+						return err
+					}
+					visitedMu.Lock()
+					defer visitedMu.Unlock()
+					if vp, ok := visitedPath[p]; ok {
+						return errors.Errorf("cannot overwrite %s from %s with %s when split option is disabled", p, vp, k)
+					}
+					visitedPath[p] = k
+					return nil
+				})
+				if err != nil {
+					return err
+				}
+			}
+
 			lbl := "copying files"
 			if isMap {
 				lbl += " " + k
-				st := fstypes.Stat{
-					Mode: uint32(os.ModeDir | 0755),
-					Path: strings.Replace(k, "/", "_", -1),
-				}
-				if e.opts.Epoch != nil {
-					st.ModTime = e.opts.Epoch.UnixNano()
-				}
-
-				outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}})
-				if err != nil {
-					return err
+				if e.opts.PlatformSplit {
+					st := fstypes.Stat{
+						Mode: uint32(os.ModeDir | 0755),
+						Path: strings.Replace(k, "/", "_", -1),
+					}
+					if e.opts.Epoch != nil {
+						st.ModTime = e.opts.Epoch.UnixNano()
+					}
+					outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}})
+					if err != nil {
+						return err
+					}
 				}
 			}
 

exporter/local/fs.go

@@ -3,11 +3,13 @@ package local
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/fs"
 	"os"
 	"path"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/docker/docker/pkg/idtools"
@@ -28,15 +30,20 @@ import (
 
 const (
 	keyAttestationPrefix = "attestation-prefix"
+	// keyPlatformSplit is an exporter option which can be used to split result
+	// in subfolders when multiple platform references are exported.
+	keyPlatformSplit = "platform-split"
 )
 
 type CreateFSOpts struct {
 	Epoch             *time.Time
 	AttestationPrefix string
+	PlatformSplit     bool
 }
 
 func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) {
 	rest := make(map[string]string)
+	c.PlatformSplit = true
 
 	var err error
 	c.Epoch, opt, err = epoch.ParseExporterAttrs(opt)
@@ -48,6 +55,12 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) {
 		switch k {
 		case keyAttestationPrefix:
 			c.AttestationPrefix = v
+		case keyPlatformSplit:
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "non-bool value for %s: %s", keyPlatformSplit, v)
+			}
+			c.PlatformSplit = b
 		default:
 			rest[k] = v
 		}
@@ -164,6 +177,11 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
 			}
 
 			name := opt.AttestationPrefix + path.Base(attestations[i].Path)
+			if !opt.PlatformSplit {
+				nameExt := path.Ext(name)
+				namBase := strings.TrimSuffix(name, nameExt)
+				name = fmt.Sprintf("%s.%s%s", namBase, strings.Replace(k, "/", "_", -1), nameExt)
+			}
 			if _, ok := names[name]; ok {
 				return nil, nil, errors.Errorf("duplicate attestation path name %s", name)
 			}