Skip to content

Commit 48e766e

Browse files
crazy-maxcrazy-max
committed
exporter: add local exporter mode=mirror
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent f556f9c commit 48e766e

File tree

5 files changed

+198
-12
lines changed

5 files changed

+198
-12
lines changed

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,15 @@ $ tree ./bin
369369
└── hello-linux-arm64
370370
```
371371

372+
Local output also supports `mode=<copy|mirror>`:
373+
374+
- `copy` (default) preserves existing files in destination that are not present in build result.
375+
- `mirror` removes destination files and directories that are not present in build result.
376+
377+
```bash
378+
buildctl build ... --output type=local,dest=./bin/release,mode=mirror
379+
```
380+
372381
Tar exporter is similar to local exporter but transfers the files through a tarball.
373382

374383
```bash

‎client/client_test.go‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,9 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
242242
testExportLocalNoPlatformSplit,
243243
testExportLocalNoPlatformSplitOverwrite,
244244
testExportLocalForcePlatformSplit,
245+
testExportLocalModeCopyKeepsStaleDestinationFiles,
246+
testExportLocalModeMirrorRemovesStaleDestinationFiles,
247+
testExportLocalModeInvalid,
245248
testSolverOptLocalDirsStillWorks,
246249
testOCIIndexMediatype,
247250
testLayerLimitOnMounts,
@@ -7453,6 +7456,114 @@ func testExportLocalForcePlatformSplit(t *testing.T, sb integration.Sandbox) {
74537456
require.Equal(t, "hello", string(dt))
74547457
}
74557458

7459+
func testExportLocalModeCopyKeepsStaleDestinationFiles(t *testing.T, sb integration.Sandbox) {
7460+
c, err := New(sb.Context(), sb.Address())
7461+
require.NoError(t, err)
7462+
defer c.Close()
7463+
7464+
st := llb.Scratch().File(
7465+
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
7466+
)
7467+
def, err := st.Marshal(sb.Context())
7468+
require.NoError(t, err)
7469+
7470+
destDir := t.TempDir()
7471+
err = os.WriteFile(filepath.Join(destDir, "stale.txt"), []byte("stale"), 0600)
7472+
require.NoError(t, err)
7473+
err = os.MkdirAll(filepath.Join(destDir, "stale-dir"), 0755)
7474+
require.NoError(t, err)
7475+
err = os.WriteFile(filepath.Join(destDir, "stale-dir", "old.txt"), []byte("stale"), 0600)
7476+
require.NoError(t, err)
7477+
7478+
_, err = c.Solve(sb.Context(), def, SolveOpt{
7479+
Exports: []ExportEntry{
7480+
{
7481+
Type: ExporterLocal,
7482+
OutputDir: destDir,
7483+
},
7484+
},
7485+
}, nil)
7486+
require.NoError(t, err)
7487+
7488+
dt, err := os.ReadFile(filepath.Join(destDir, "fresh.txt"))
7489+
require.NoError(t, err)
7490+
require.Equal(t, "fresh", string(dt))
7491+
7492+
_, err = os.Stat(filepath.Join(destDir, "stale.txt"))
7493+
require.NoError(t, err)
7494+
_, err = os.Stat(filepath.Join(destDir, "stale-dir", "old.txt"))
7495+
require.NoError(t, err)
7496+
}
7497+
7498+
func testExportLocalModeMirrorRemovesStaleDestinationFiles(t *testing.T, sb integration.Sandbox) {
7499+
c, err := New(sb.Context(), sb.Address())
7500+
require.NoError(t, err)
7501+
defer c.Close()
7502+
7503+
st := llb.Scratch().File(
7504+
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
7505+
)
7506+
def, err := st.Marshal(sb.Context())
7507+
require.NoError(t, err)
7508+
7509+
destDir := t.TempDir()
7510+
err = os.WriteFile(filepath.Join(destDir, "stale.txt"), []byte("stale"), 0600)
7511+
require.NoError(t, err)
7512+
err = os.MkdirAll(filepath.Join(destDir, "stale-dir"), 0755)
7513+
require.NoError(t, err)
7514+
err = os.WriteFile(filepath.Join(destDir, "stale-dir", "old.txt"), []byte("stale"), 0600)
7515+
require.NoError(t, err)
7516+
7517+
_, err = c.Solve(sb.Context(), def, SolveOpt{
7518+
Exports: []ExportEntry{
7519+
{
7520+
Type: ExporterLocal,
7521+
OutputDir: destDir,
7522+
Attrs: map[string]string{
7523+
"mode": "mirror",
7524+
},
7525+
},
7526+
},
7527+
}, nil)
7528+
require.NoError(t, err)
7529+
7530+
dt, err := os.ReadFile(filepath.Join(destDir, "fresh.txt"))
7531+
require.NoError(t, err)
7532+
require.Equal(t, "fresh", string(dt))
7533+
7534+
_, err = os.Stat(filepath.Join(destDir, "stale.txt"))
7535+
require.ErrorIs(t, err, os.ErrNotExist)
7536+
_, err = os.Stat(filepath.Join(destDir, "stale-dir"))
7537+
require.ErrorIs(t, err, os.ErrNotExist)
7538+
}
7539+
7540+
func testExportLocalModeInvalid(t *testing.T, sb integration.Sandbox) {
7541+
c, err := New(sb.Context(), sb.Address())
7542+
require.NoError(t, err)
7543+
defer c.Close()
7544+
7545+
st := llb.Scratch().File(
7546+
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
7547+
)
7548+
def, err := st.Marshal(sb.Context())
7549+
require.NoError(t, err)
7550+
7551+
destDir := t.TempDir()
7552+
_, err = c.Solve(sb.Context(), def, SolveOpt{
7553+
Exports: []ExportEntry{
7554+
{
7555+
Type: ExporterLocal,
7556+
OutputDir: destDir,
7557+
Attrs: map[string]string{
7558+
"mode": "backup",
7559+
},
7560+
},
7561+
},
7562+
}, nil)
7563+
require.Error(t, err)
7564+
require.ErrorContains(t, err, `invalid local exporter mode "backup"`)
7565+
}
7566+
74567567
func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) {
74577568
def, err := llb.Image(ref).Marshal(ctx)
74587569
if err != nil {

‎client/solve.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,15 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
188188
if ex.OutputDir == "" {
189189
return nil, errors.Errorf("output directory is required for %s exporter", ex.Type)
190190
}
191-
syncTargets = append(syncTargets, filesync.WithFSSyncDir(exID, ex.OutputDir))
191+
if ex.Type == ExporterLocal {
192+
mode, err := parseLocalExporterMode(ex.Attrs)
193+
if err != nil {
194+
return nil, err
195+
}
196+
syncTargets = append(syncTargets, filesync.WithFSSyncDirMode(exID, ex.OutputDir, mode))
197+
} else {
198+
syncTargets = append(syncTargets, filesync.WithFSSyncDir(exID, ex.OutputDir))
199+
}
192200
}
193201
if supportStore {
194202
store := ex.OutputStore
@@ -595,3 +603,19 @@ func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
595603
}
596604
return mounts, nil
597605
}
606+
607+
func parseLocalExporterMode(attrs map[string]string) (filesync.FSSyncDirMode, error) {
608+
if attrs == nil {
609+
return filesync.FSSyncDirModeCopy, nil
610+
}
611+
612+
mode := strings.TrimSpace(strings.ToLower(attrs["mode"]))
613+
switch mode {
614+
case "", string(filesync.FSSyncDirModeCopy):
615+
return filesync.FSSyncDirModeCopy, nil
616+
case string(filesync.FSSyncDirModeMirror):
617+
return filesync.FSSyncDirModeMirror, nil
618+
default:
619+
return "", errors.Errorf("invalid local exporter mode %q", attrs["mode"])
620+
}
621+
}

‎session/filesync/diffcopy.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,12 +111,22 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p
111111
}))
112112
}
113113

114-
func syncTargetDiffCopy(ds grpc.ServerStream, dest string) error {
114+
func syncTargetDiffCopy(ds grpc.ServerStream, dest string, mode FSSyncDirMode) error {
115+
var merge bool
116+
switch mode {
117+
case "", FSSyncDirModeCopy:
118+
merge = true
119+
case FSSyncDirModeMirror:
120+
merge = false
121+
default:
122+
return errors.Errorf("invalid local exporter mode %q", mode)
123+
}
124+
115125
if err := os.MkdirAll(dest, 0700); err != nil {
116126
return errors.Wrapf(err, "failed to create synctarget dest dir %s", dest)
117127
}
118128
return errors.WithStack(fsutil.Receive(ds.Context(), ds, dest, fsutil.ReceiveOpt{
119-
Merge: true,
129+
Merge: merge,
120130
Filter: func() func(string, *fstypes.Stat) bool {
121131
uid := os.Getuid()
122132
gid := os.Getgid()

‎session/filesync/filesync.go‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -251,10 +251,18 @@ type FSSyncTarget interface {
251251
target() *fsSyncTarget
252252
}
253253

254+
type FSSyncDirMode string
255+
256+
const (
257+
FSSyncDirModeCopy FSSyncDirMode = "copy"
258+
FSSyncDirModeMirror FSSyncDirMode = "mirror"
259+
)
260+
254261
type fsSyncTarget struct {
255-
id int
256-
outdir string
257-
f FileOutputFunc
262+
id int
263+
outdir string
264+
outdirMode FSSyncDirMode
265+
f FileOutputFunc
258266
}
259267

260268
func (target *fsSyncTarget) target() *fsSyncTarget {
@@ -270,23 +278,40 @@ func WithFSSync(id int, f FileOutputFunc) FSSyncTarget {
270278

271279
func WithFSSyncDir(id int, outdir string) FSSyncTarget {
272280
return &fsSyncTarget{
273-
id: id,
274-
outdir: outdir,
281+
id: id,
282+
outdir: outdir,
283+
outdirMode: FSSyncDirModeCopy,
284+
}
285+
}
286+
287+
func WithFSSyncDirMode(id int, outdir string, mode FSSyncDirMode) FSSyncTarget {
288+
if mode == "" {
289+
mode = FSSyncDirModeCopy
290+
}
291+
return &fsSyncTarget{
292+
id: id,
293+
outdir: outdir,
294+
outdirMode: mode,
275295
}
276296
}
277297

278298
func NewFSSyncTarget(targets ...FSSyncTarget) *SyncTarget {
279299
st := &SyncTarget{
280300
fs: make(map[int]FileOutputFunc),
281-
outdirs: make(map[int]string),
301+
outdirs: make(map[int]syncTargetDir),
282302
}
283303
st.Add(targets...)
284304
return st
285305
}
286306

307+
type syncTargetDir struct {
308+
dir string
309+
mode FSSyncDirMode
310+
}
311+
287312
type SyncTarget struct {
288313
fs map[int]FileOutputFunc
289-
outdirs map[int]string
314+
outdirs map[int]syncTargetDir
290315
}
291316

292317
var _ session.Attachable = &SyncTarget{}
@@ -298,7 +323,14 @@ func (sp *SyncTarget) Add(targets ...FSSyncTarget) {
298323
sp.fs[t.id] = t.f
299324
}
300325
if t.outdir != "" {
301-
sp.outdirs[t.id] = t.outdir
326+
mode := t.outdirMode
327+
if mode == "" {
328+
mode = FSSyncDirModeCopy
329+
}
330+
sp.outdirs[t.id] = syncTargetDir{
331+
dir: t.outdir,
332+
mode: mode,
333+
}
302334
}
303335
}
304336
}
@@ -326,7 +358,7 @@ func (sp *SyncTarget) chooser(ctx context.Context) int {
326358
func (sp *SyncTarget) DiffCopy(stream FileSend_DiffCopyServer) (err error) {
327359
id := sp.chooser(stream.Context())
328360
if outdir, ok := sp.outdirs[id]; ok {
329-
return syncTargetDiffCopy(stream, outdir)
361+
return syncTargetDiffCopy(stream, outdir.dir, outdir.mode)
330362
}
331363
f, ok := sp.fs[id]
332364
if !ok {

0 commit comments

Comments
 (0)