Skip to content

Commit 071d3cf

Browse files
crazy-maxcrazy-max
committed
exporter: add local exporter mode=mirror
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent f556f9c commit 071d3cf

File tree

8 files changed

+268
-15
lines changed

8 files changed

+268
-15
lines changed

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,15 @@ $ tree ./bin
369369
└── hello-linux-arm64
370370
```
371371

372+
Local output also supports `mode=<copy|mirror>`:
373+
374+
- `copy` (default) preserves existing files in destination that are not present in build result.
375+
- `mirror` removes destination files and directories that are not present in build result.
376+
377+
```bash
378+
buildctl build ... --output type=local,dest=./bin/release,mode=mirror
379+
```
380+
372381
Tar exporter is similar to local exporter but transfers the files through a tarball.
373382

374383
```bash

‎client/client_test.go‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,9 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
242242
testExportLocalNoPlatformSplit,
243243
testExportLocalNoPlatformSplitOverwrite,
244244
testExportLocalForcePlatformSplit,
245+
testExportLocalModeCopyKeepsStaleDestinationFiles,
246+
testExportLocalModeMirrorRemovesStaleDestinationFiles,
247+
testExportLocalModeInvalid,
245248
testSolverOptLocalDirsStillWorks,
246249
testOCIIndexMediatype,
247250
testLayerLimitOnMounts,
@@ -7453,6 +7456,114 @@ func testExportLocalForcePlatformSplit(t *testing.T, sb integration.Sandbox) {
74537456
require.Equal(t, "hello", string(dt))
74547457
}
74557458

7459+
func testExportLocalModeCopyKeepsStaleDestinationFiles(t *testing.T, sb integration.Sandbox) {
7460+
c, err := New(sb.Context(), sb.Address())
7461+
require.NoError(t, err)
7462+
defer c.Close()
7463+
7464+
st := llb.Scratch().File(
7465+
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
7466+
)
7467+
def, err := st.Marshal(sb.Context())
7468+
require.NoError(t, err)
7469+
7470+
destDir := t.TempDir()
7471+
err = os.WriteFile(filepath.Join(destDir, "stale.txt"), []byte("stale"), 0600)
7472+
require.NoError(t, err)
7473+
err = os.MkdirAll(filepath.Join(destDir, "stale-dir"), 0755)
7474+
require.NoError(t, err)
7475+
err = os.WriteFile(filepath.Join(destDir, "stale-dir", "old.txt"), []byte("stale"), 0600)
7476+
require.NoError(t, err)
7477+
7478+
_, err = c.Solve(sb.Context(), def, SolveOpt{
7479+
Exports: []ExportEntry{
7480+
{
7481+
Type: ExporterLocal,
7482+
OutputDir: destDir,
7483+
},
7484+
},
7485+
}, nil)
7486+
require.NoError(t, err)
7487+
7488+
dt, err := os.ReadFile(filepath.Join(destDir, "fresh.txt"))
7489+
require.NoError(t, err)
7490+
require.Equal(t, "fresh", string(dt))
7491+
7492+
_, err = os.Stat(filepath.Join(destDir, "stale.txt"))
7493+
require.NoError(t, err)
7494+
_, err = os.Stat(filepath.Join(destDir, "stale-dir", "old.txt"))
7495+
require.NoError(t, err)
7496+
}
7497+
7498+
func testExportLocalModeMirrorRemovesStaleDestinationFiles(t *testing.T, sb integration.Sandbox) {
7499+
c, err := New(sb.Context(), sb.Address())
7500+
require.NoError(t, err)
7501+
defer c.Close()
7502+
7503+
st := llb.Scratch().File(
7504+
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
7505+
)
7506+
def, err := st.Marshal(sb.Context())
7507+
require.NoError(t, err)
7508+
7509+
destDir := t.TempDir()
7510+
err = os.WriteFile(filepath.Join(destDir, "stale.txt"), []byte("stale"), 0600)
7511+
require.NoError(t, err)
7512+
err = os.MkdirAll(filepath.Join(destDir, "stale-dir"), 0755)
7513+
require.NoError(t, err)
7514+
err = os.WriteFile(filepath.Join(destDir, "stale-dir", "old.txt"), []byte("stale"), 0600)
7515+
require.NoError(t, err)
7516+
7517+
_, err = c.Solve(sb.Context(), def, SolveOpt{
7518+
Exports: []ExportEntry{
7519+
{
7520+
Type: ExporterLocal,
7521+
OutputDir: destDir,
7522+
Attrs: map[string]string{
7523+
"mode": "mirror",
7524+
},
7525+
},
7526+
},
7527+
}, nil)
7528+
require.NoError(t, err)
7529+
7530+
dt, err := os.ReadFile(filepath.Join(destDir, "fresh.txt"))
7531+
require.NoError(t, err)
7532+
require.Equal(t, "fresh", string(dt))
7533+
7534+
_, err = os.Stat(filepath.Join(destDir, "stale.txt"))
7535+
require.ErrorIs(t, err, os.ErrNotExist)
7536+
_, err = os.Stat(filepath.Join(destDir, "stale-dir"))
7537+
require.ErrorIs(t, err, os.ErrNotExist)
7538+
}
7539+
7540+
func testExportLocalModeInvalid(t *testing.T, sb integration.Sandbox) {
7541+
c, err := New(sb.Context(), sb.Address())
7542+
require.NoError(t, err)
7543+
defer c.Close()
7544+
7545+
st := llb.Scratch().File(
7546+
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
7547+
)
7548+
def, err := st.Marshal(sb.Context())
7549+
require.NoError(t, err)
7550+
7551+
destDir := t.TempDir()
7552+
_, err = c.Solve(sb.Context(), def, SolveOpt{
7553+
Exports: []ExportEntry{
7554+
{
7555+
Type: ExporterLocal,
7556+
OutputDir: destDir,
7557+
Attrs: map[string]string{
7558+
"mode": "backup",
7559+
},
7560+
},
7561+
},
7562+
}, nil)
7563+
require.Error(t, err)
7564+
require.ErrorContains(t, err, `invalid local exporter mode "backup"`)
7565+
}
7566+
74567567
func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) {
74577568
def, err := llb.Image(ref).Marshal(ctx)
74587569
if err != nil {

‎client/solve.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,15 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
188188
if ex.OutputDir == "" {
189189
return nil, errors.Errorf("output directory is required for %s exporter", ex.Type)
190190
}
191-
syncTargets = append(syncTargets, filesync.WithFSSyncDir(exID, ex.OutputDir))
191+
if ex.Type == ExporterLocal {
192+
mode, err := parseLocalExporterMode(ex.Attrs)
193+
if err != nil {
194+
return nil, err
195+
}
196+
syncTargets = append(syncTargets, filesync.WithFSSyncDirMode(exID, ex.OutputDir, mode))
197+
} else {
198+
syncTargets = append(syncTargets, filesync.WithFSSyncDir(exID, ex.OutputDir))
199+
}
192200
}
193201
if supportStore {
194202
store := ex.OutputStore
@@ -595,3 +603,11 @@ func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
595603
}
596604
return mounts, nil
597605
}
606+
607+
func parseLocalExporterMode(attrs map[string]string) (filesync.FSSyncDirMode, error) {
608+
if attrs == nil {
609+
return filesync.FSSyncDirModeCopy, nil
610+
}
611+
612+
return filesync.ParseFSSyncDirMode(attrs["mode"])
613+
}

‎exporter/local/export.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,9 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
162162
}
163163

164164
progress := NewProgressHandler(ctx, lbl)
165-
if err := filesync.CopyToCaller(ctx, outputFS, e.id, caller, progress); err != nil {
165+
if err := filesync.CopyToCaller(ctx, outputFS, e.id, caller, progress, map[string]string{
166+
filesync.ExporterMetaLocalDirMode: string(e.opts.Mode),
167+
}); err != nil {
166168
return err
167169
}
168170
return nil

‎exporter/local/fs.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/moby/buildkit/exporter/attestation"
1919
"github.com/moby/buildkit/exporter/util/epoch"
2020
"github.com/moby/buildkit/session"
21+
"github.com/moby/buildkit/session/filesync"
2122
"github.com/moby/buildkit/snapshot"
2223
"github.com/moby/buildkit/solver/result"
2324
"github.com/moby/buildkit/util/staticfs"
@@ -33,12 +34,14 @@ const (
3334
// keyPlatformSplit is an exporter option which can be used to split result
3435
// in subfolders when multiple platform references are exported.
3536
keyPlatformSplit = "platform-split"
37+
keyMode = "mode"
3638
)
3739

3840
type CreateFSOpts struct {
3941
Epoch *time.Time
4042
AttestationPrefix string
4143
PlatformSplit *bool
44+
Mode filesync.FSSyncDirMode
4245
}
4346

4447
func (c *CreateFSOpts) UsePlatformSplit(isMap bool) bool {
@@ -50,6 +53,7 @@ func (c *CreateFSOpts) UsePlatformSplit(isMap bool) bool {
5053

5154
func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) {
5255
rest := make(map[string]string)
56+
c.Mode = filesync.FSSyncDirModeCopy
5357

5458
var err error
5559
c.Epoch, opt, err = epoch.ParseExporterAttrs(opt)
@@ -67,6 +71,12 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) {
6771
return nil, errors.Wrapf(err, "non-bool value for %s: %s", keyPlatformSplit, v)
6872
}
6973
c.PlatformSplit = &b
74+
case keyMode:
75+
mode, err := filesync.ParseFSSyncDirMode(v)
76+
if err != nil {
77+
return nil, err
78+
}
79+
c.Mode = mode
7080
default:
7181
rest[k] = v
7282
}

‎exporter/local/fs_test.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package local
2+
3+
import (
4+
"testing"
5+
6+
"github.com/moby/buildkit/session/filesync"
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
func TestCreateFSOptsLoadModeDefault(t *testing.T) {
11+
var opts CreateFSOpts
12+
_, err := opts.Load(nil)
13+
require.NoError(t, err)
14+
require.Equal(t, filesync.FSSyncDirModeCopy, opts.Mode)
15+
}
16+
17+
func TestCreateFSOptsLoadModeMirror(t *testing.T) {
18+
var opts CreateFSOpts
19+
_, err := opts.Load(map[string]string{
20+
"mode": "mirror",
21+
})
22+
require.NoError(t, err)
23+
require.Equal(t, filesync.FSSyncDirModeMirror, opts.Mode)
24+
}
25+
26+
func TestCreateFSOptsLoadModeInvalid(t *testing.T) {
27+
var opts CreateFSOpts
28+
_, err := opts.Load(map[string]string{
29+
"mode": "backup",
30+
})
31+
require.Error(t, err)
32+
require.ErrorContains(t, err, `invalid local exporter mode "backup"`)
33+
}
34+

‎session/filesync/diffcopy.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,12 +111,22 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p
111111
}))
112112
}
113113

114-
func syncTargetDiffCopy(ds grpc.ServerStream, dest string) error {
114+
func syncTargetDiffCopy(ds grpc.ServerStream, dest string, mode FSSyncDirMode) error {
115+
var merge bool
116+
switch mode {
117+
case "", FSSyncDirModeCopy:
118+
merge = true
119+
case FSSyncDirModeMirror:
120+
merge = false
121+
default:
122+
return errors.Errorf("invalid local exporter mode %q", mode)
123+
}
124+
115125
if err := os.MkdirAll(dest, 0700); err != nil {
116126
return errors.Wrapf(err, "failed to create synctarget dest dir %s", dest)
117127
}
118128
return errors.WithStack(fsutil.Receive(ds.Context(), ds, dest, fsutil.ReceiveOpt{
119-
Merge: true,
129+
Merge: merge,
120130
Filter: func() func(string, *fstypes.Stat) bool {
121131
uid := os.Getuid()
122132
gid := os.Getgid()

0 commit comments

Comments
 (0)