Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,15 @@ $ tree ./bin
└── hello-linux-arm64
```

Local output also supports `mode=<copy|delete>`:

- `copy` (default) preserves existing files in destination that are not present in build result.
- `delete` removes destination files and directories that are not present in build result.

```bash
buildctl build ... --output type=local,dest=./bin/release,mode=delete
```

Tar exporter is similar to local exporter but transfers the files through a tarball.

```bash
Expand Down
239 changes: 239 additions & 0 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,11 @@
testExportLocalNoPlatformSplit,
testExportLocalNoPlatformSplitOverwrite,
testExportLocalForcePlatformSplit,
testExportLocalModeCopyKeepsStaleDestinationFiles,
testExportLocalModeDeleteRemovesStaleDestinationFiles,
testExportLocalModeCopyMultiPlatformKeepsAllPlatforms,
testExportLocalModeDeleteMultiPlatformKeepsAllPlatforms,
testExportLocalModeInvalid,
testSolverOptLocalDirsStillWorks,
testOCIIndexMediatype,
testLayerLimitOnMounts,
Expand Down Expand Up @@ -7622,6 +7627,240 @@
require.Equal(t, "hello", string(dt))
}

func testExportLocalModeCopyKeepsStaleDestinationFiles(t *testing.T, sb integration.Sandbox) {
c, err := New(sb.Context(), sb.Address())
require.NoError(t, err)
defer c.Close()

st := llb.Scratch().File(
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
)
def, err := st.Marshal(sb.Context())
require.NoError(t, err)

destDir := t.TempDir()
err = os.WriteFile(filepath.Join(destDir, "stale.txt"), []byte("stale"), 0600)
require.NoError(t, err)
err = os.MkdirAll(filepath.Join(destDir, "stale-dir"), 0755)
require.NoError(t, err)
err = os.WriteFile(filepath.Join(destDir, "stale-dir", "old.txt"), []byte("stale"), 0600)
require.NoError(t, err)

_, err = c.Solve(sb.Context(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
}, nil)
require.NoError(t, err)

dt, err := os.ReadFile(filepath.Join(destDir, "fresh.txt"))
require.NoError(t, err)
require.Equal(t, "fresh", string(dt))

_, err = os.Stat(filepath.Join(destDir, "stale.txt"))
require.NoError(t, err)
_, err = os.Stat(filepath.Join(destDir, "stale-dir", "old.txt"))
require.NoError(t, err)
}

func testExportLocalModeDeleteRemovesStaleDestinationFiles(t *testing.T, sb integration.Sandbox) {
c, err := New(sb.Context(), sb.Address())
require.NoError(t, err)
defer c.Close()

st := llb.Scratch().File(
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
)
def, err := st.Marshal(sb.Context())
require.NoError(t, err)

destDir := t.TempDir()
err = os.WriteFile(filepath.Join(destDir, "stale.txt"), []byte("stale"), 0600)
require.NoError(t, err)
err = os.MkdirAll(filepath.Join(destDir, "stale-dir"), 0755)
require.NoError(t, err)
err = os.WriteFile(filepath.Join(destDir, "stale-dir", "old.txt"), []byte("stale"), 0600)
require.NoError(t, err)

_, err = c.Solve(sb.Context(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
Attrs: map[string]string{
"mode": "delete",
},
},
},
}, nil)
require.NoError(t, err)

Check failure on line 7700 in client/client_test.go

View workflow job for this annotation

GitHub Actions / test / run (./client, containerd, nydus, integration)

Failed: client/TestIntegration/TestExportLocalModeDeleteRemovesStaleDestinationFiles/worker=containerd

=== RUN TestIntegration/TestExportLocalModeDeleteRemovesStaleDestinationFiles/worker=containerd === PAUSE TestIntegration/TestExportLocalModeDeleteRemovesStaleDestinationFiles/worker=containerd === CONT TestIntegration/TestExportLocalModeDeleteRemovesStaleDestinationFiles/worker=containerd client_test.go:7700: Error Trace: /src/client/client_test.go:7700 /src/util/testutil/integration/run.go:105 /src/util/testutil/integration/run.go:253 Error: Received unexpected error: lstat /tmp/TestIntegrationTestExportLocalModeDeleteRemovesStaleDestination3891179094/001/stale-dir/old.txt: no such file or directory github.com/moby/buildkit/client.reconcileLocalDeleteExport /src/client/solve.go:512 github.com/moby/buildkit/client.(*Client).solve /src/client/solve.go:447 github.com/moby/buildkit/client.(*Client).Solve /src/client/solve.go:91 github.com/moby/buildkit/client.testExportLocalModeDeleteRemovesStaleDestinationFiles /src/client/client_test.go:7689 github.com/moby/buildkit/util/testutil/integration.testFunc.Run /src/util/testutil/integration/run.go:105 github.com/moby/buildkit/util/testutil/integration.Run.Run.func2.func4 /src/util/testutil/integration/run.go:253 testing.tRunner /usr/local/go/src/testing/testing.go:2036 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:1771 Test: TestIntegration/TestExportLocalModeDeleteRemovesStaleDestinationFiles/worker=containerd sandbox.go:202: stderr: /usr/bin/containerd --config /tmp/bktest_containerd1599826397/config.toml sandbox.go:205: > StartCmd 2026-03-31 19:46:31.704251363 +0000 UTC m=+217.173169811 /usr/bin/containerd --config /tmp/bktest_containerd1599826397/config.toml sandbox.go:205: time="2026-03-31T19:46:31.739215918Z" level=info msg="starting containerd" revision=1c4457e00facac03ce1d75f7b6777a7a851e5c41 version=v2.2.0 sandbox.go:205: time="2026-03-31T19:46:31.752851554Z" level=warning msg="Configuration migrated from version 2, use `containerd config migrate` to avoid migration" t="2.345µs" sandbox.go:205: time="2026-03-31T19:46:31.752941472Z" level=info msg="loading plugin" id=io.containerd.content.v1.content type=io.containerd.content.v1 sandbox.go:205: time="2026-03-31T19:46:31.752982018Z" level=info msg="loading plugin" id=io.containerd.image-verifier.v1.bindir type=io.containerd.image-verifier.v1 sandbox.go:205: time="2026-03-31T19:46:31.753006023Z" level=info msg="loading plugin" id=io.containerd.internal.v1.opt type=io.containerd.internal.v1 sandbox.go:205: time="2026-03-31T19:46:31.753039074Z" level=info msg="loading plugin" id=io.containerd.warning.v1.deprecations type=io.containerd.warning.v1 sandbox.go:205: time="2026-03-31T19:46:31.753053932Z" level=info msg="loading plugin" id=io.containerd.mount-handler.v1.erofs type=io.containerd.mount-handler.v1 sandbox.go:205: time="2026-03-31T19:46:31.753062789Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.blockfile type=io.containerd.snapshotter.v1 sandbox.go:205: time="2026-03-31T19:46:31.753142788Z" level=info msg="skip loading plugin" error="no scratch file generator: skip plugin" id=io.containerd.snapshotter.v1.blockfile type=io.containerd.snapshotter.v1 sandbox.go:205: time="2026-03-31T19:46:31.753172153Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.devmapper type=io.containerd.snapshotter.v1 sandbox.go:205: time="2026-03-31T19:46:31.753184646Z" level=info msg="skip loading plugin" error="devmapper not configured: skip plugin" id=io.containerd.snapshotter.v1.devmapper type=io.containerd.snapshotter.v1 sandbox.go:205: time="2026-03-31T19:46:31.7531936
dt, err := os.ReadFile(filepath.Join(destDir, "fresh.txt"))
require.NoError(t, err)
require.Equal(t, "fresh", string(dt))

_, err = os.Stat(filepath.Join(destDir, "stale.txt"))
require.ErrorIs(t, err, os.ErrNotExist)
_, err = os.Stat(filepath.Join(destDir, "stale-dir"))
require.ErrorIs(t, err, os.ErrNotExist)
}

func testExportLocalModeCopyMultiPlatformKeepsAllPlatforms(t *testing.T, sb integration.Sandbox) {
testExportLocalModeMultiPlatformKeepsAllPlatforms(t, sb, false)
}

func testExportLocalModeDeleteMultiPlatformKeepsAllPlatforms(t *testing.T, sb integration.Sandbox) {
testExportLocalModeMultiPlatformKeepsAllPlatforms(t, sb, true)
}

func testExportLocalModeMultiPlatformKeepsAllPlatforms(t *testing.T, sb integration.Sandbox, deleteMode bool) {
workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureMultiPlatform)
c, err := New(sb.Context(), sb.Address())
require.NoError(t, err)
defer c.Close()

const filesPerPlatform = 20
platformsToTest := []string{"linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x"}

frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
res := gateway.NewResult()
expPlatforms := &exptypes.Platforms{
Platforms: make([]exptypes.Platform, len(platformsToTest)),
}
for i, platform := range platformsToTest {
st := llb.Scratch()
for j := range filesPerPlatform {
st = st.File(
llb.Mkfile(fmt.Sprintf("file-%03d.txt", j), 0600, fmt.Appendf(nil, "%s-%d", platform, j)),
)
}

def, err := st.Marshal(ctx)
if err != nil {
return nil, err
}

r, err := c.Solve(ctx, gateway.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}

ref, err := r.SingleRef()
if err != nil {
return nil, err
}

_, err = ref.ToState()
if err != nil {
return nil, err
}
res.AddRef(platform, ref)

expPlatforms.Platforms[i] = exptypes.Platform{
ID: platform,
Platform: platforms.MustParse(platform),
}
}
dt, err := json.Marshal(expPlatforms)
if err != nil {
return nil, err
}
res.AddMeta(exptypes.ExporterPlatformsKey, dt)

return res, nil
}

destDir := t.TempDir()

// Pre-populate dest with directories matching the platform-split naming
// convention. This is critical for exposing the race in delete mode:
// each platform's fsutil.Receive does a dest-walk at the start and, with
// Merge=false (mode=delete), flags everything not in its own stream for
// deletion — including other platforms' pre-existing directories. Without
// pre-population the dest starts empty and the concurrent dest-walks all
// see nothing to delete, hiding the cross-platform deletion race.
err = os.WriteFile(filepath.Join(destDir, "stale.txt"), []byte("stale"), 0600)
require.NoError(t, err)
for _, platform := range platformsToTest {
platDir := filepath.Join(destDir, strings.ReplaceAll(platform, "/", "_"))
err = os.MkdirAll(platDir, 0755)
require.NoError(t, err)
for j := range filesPerPlatform {
err = os.WriteFile(filepath.Join(platDir, fmt.Sprintf("old-%03d.txt", j)), []byte("old"), 0600)
require.NoError(t, err)
}
}

attrs := map[string]string{}
if deleteMode {
attrs["mode"] = "delete"
}

_, err = c.Build(sb.Context(), SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
Attrs: attrs,
},
},
}, "", frontend, nil)
require.NoError(t, err)

if deleteMode {
// Stale top-level file must be gone.
_, err = os.Stat(filepath.Join(destDir, "stale.txt"))
require.ErrorIs(t, err, os.ErrNotExist)
}

// Every platform's build output must survive (no cross-platform deletion).
for _, platform := range platformsToTest {
platformDir := filepath.Join(destDir, strings.ReplaceAll(platform, "/", "_"))
for j := range filesPerPlatform {
dt, err := os.ReadFile(filepath.Join(platformDir, fmt.Sprintf("file-%03d.txt", j)))
require.NoError(t, err, "missing build output file-%03d.txt for %s", j, platform)
require.Equal(t, fmt.Sprintf("%s-%d", platform, j), string(dt))
}
if deleteMode {
// Pre-existing files within each platform dir must be cleaned up.
_, err = os.Stat(filepath.Join(platformDir, "old-000.txt"))
require.ErrorIs(t, err, os.ErrNotExist, "stale file not removed for %s", platform)
}
}
}

func testExportLocalModeInvalid(t *testing.T, sb integration.Sandbox) {
c, err := New(sb.Context(), sb.Address())
require.NoError(t, err)
defer c.Close()

st := llb.Scratch().File(
llb.Mkfile("fresh.txt", 0600, []byte("fresh")),
)
def, err := st.Marshal(sb.Context())
require.NoError(t, err)

destDir := t.TempDir()
_, err = c.Solve(sb.Context(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
Attrs: map[string]string{
"mode": "backup",
},
},
},
}, nil)
require.Error(t, err)
require.ErrorContains(t, err, `invalid local exporter mode "backup"`)
}

func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) {
def, err := llb.Image(ref).Marshal(ctx)
if err != nil {
Expand Down
Loading
Loading