client: add reset option for local cache exporter

Add `reset=true` attribute to the local cache exporter that removes
unreferenced blobs from the cache directory after export, preventing
unbounded growth.

Signed-off-by: Jiří Moravčík <jiri.moravcik@gmail.com>

Author: jirimoravcik

README.md

@@ -497,6 +497,7 @@ The directory layout conforms to OCI Image Spec v1.0.
 * `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)
 * `force-compression=true`: forcibly apply `compression` option to all layers
 * `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
+* `reset=<true|false>`: remove any blobs in the cache directory that are not referenced by the current manifests in `index.json` (default: `false`). This is useful for keeping the local cache directory from growing indefinitely.
 
 `--import-cache` options:
 * `type=local`

client/client_test.go

@@ -199,6 +199,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testZstdLocalCacheExport,
 	testCacheExportIgnoreError,
 	testCacheExportCacheDeletedContent,
+	testLocalCacheExportReset,
 	testZstdRegistryCacheImportExport,
 	testZstdLocalCacheImportExport,
 	testUncompressedLocalCacheImportExport,
@@ -516,6 +517,93 @@ func testCacheExportCacheDeletedContent(t *testing.T, sb integration.Sandbox) {
 	}
 }
 
+func testLocalCacheExportReset(t *testing.T, sb integration.Sandbox) {
+	integration.SkipOnPlatform(t, "windows")
+	workers.CheckFeatureCompat(t, sb, workers.FeatureCacheExport, workers.FeatureCacheBackendLocal)
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	tmpdir := integration.Tmpdir(t)
+	cacheDir := filepath.Join(tmpdir.Name, "cache")
+
+	// Build 1: export cache for a simple build
+	st1 := llb.Image("alpine:latest").Run(llb.Shlex(`sh -c "echo build1 > /out"`)).Root()
+	def1, err := llb.Scratch().File(llb.Copy(st1, "/out", "/out")).Marshal(sb.Context())
+	require.NoError(t, err)
+
+	_, err = c.Solve(sb.Context(), def1, SolveOpt{
+		CacheExports: []CacheOptionsEntry{
+			{
+				Type: "local",
+				Attrs: map[string]string{
+					"dest": cacheDir,
+				},
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	// Verify blobs were written
+	blobDir := filepath.Join(cacheDir, "blobs", "sha256")
+	entries1, err := os.ReadDir(blobDir)
+	require.NoError(t, err)
+	require.Greater(t, len(entries1), 0)
+
+	// Build 2: different build, export with reset=true
+	st2 := llb.Image("alpine:latest").Run(llb.Shlex(`sh -c "echo build2-different > /out2"`)).Root()
+	def2, err := llb.Scratch().File(llb.Copy(st2, "/out2", "/out2")).Marshal(sb.Context())
+	require.NoError(t, err)
+
+	_, err = c.Solve(sb.Context(), def2, SolveOpt{
+		CacheExports: []CacheOptionsEntry{
+			{
+				Type: "local",
+				Attrs: map[string]string{
+					"dest":  cacheDir,
+					"reset": "true",
+				},
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	// Read the manifest to determine which blobs should be referenced
+	dt, err := os.ReadFile(filepath.Join(cacheDir, "index.json"))
+	require.NoError(t, err)
+
+	var index ocispecs.Index
+	err = json.Unmarshal(dt, &index)
+	require.NoError(t, err)
+	require.Len(t, index.Manifests, 1)
+
+	dt, err = os.ReadFile(filepath.Join(blobDir, index.Manifests[0].Digest.Hex()))
+	require.NoError(t, err)
+
+	var mfst ocispecs.Manifest
+	err = json.Unmarshal(dt, &mfst)
+	require.NoError(t, err)
+
+	// Build the set of expected blobs: manifest + config + all layers
+	expectedBlobs := map[string]struct{}{
+		index.Manifests[0].Digest.Hex(): {},
+		mfst.Config.Digest.Hex():        {},
+	}
+	for _, l := range mfst.Layers {
+		expectedBlobs[l.Digest.Hex()] = struct{}{}
+	}
+
+	// Verify only referenced blobs remain
+	entries2, err := os.ReadDir(blobDir)
+	require.NoError(t, err)
+
+	actualBlobs := map[string]struct{}{}
+	for _, e := range entries2 {
+		actualBlobs[e.Name()] = struct{}{}
+	}
+	require.Equal(t, expectedBlobs, actualBlobs, "after reset, only blobs referenced by the current manifest should remain")
+}
+
 func testBridgeNetworking(t *testing.T, sb integration.Sandbox) {
 	if os.Getenv("BUILDKIT_RUN_NETWORK_INTEGRATION_TESTS") == "" {
 		t.SkipNow()

client/solve.go

@@ -8,6 +8,7 @@ import (
 	"maps"
 	"os"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -420,9 +421,61 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			}
 		}
 	}
+	// Reset cache stores that have reset=true — delete unreferenced blobs
+	for _, ref := range cacheOpt.storesToReset {
+		if err := resetCacheStore(ctx, ref.store, ref.path); err != nil {
+			bklog.G(ctx).WithError(err).Warn("failed to reset cache store")
+		}
+	}
 	return res, nil
 }
 
+// resetCacheStore deletes all blobs not referenced by any manifest in
+// index.json. Referenced blobs are always preserved.
+func resetCacheStore(ctx context.Context, cs content.Store, storePath string) error {
+	idx := ociindex.NewStoreIndex(storePath)
+	index, err := idx.Read()
+	if err != nil {
+		return errors.Wrap(err, "reset: failed to read index.json")
+	}
+
+	referenced := make(map[digest.Digest]struct{})
+
+	for _, manifestDesc := range index.Manifests {
+		referenced[manifestDesc.Digest] = struct{}{}
+
+		dt, err := content.ReadBlob(ctx, cs, manifestDesc)
+		if err != nil {
+			return errors.Wrapf(err, "reset: failed to read manifest %s", manifestDesc.Digest)
+		}
+
+		var manifest ocispecs.Manifest
+		if err := json.Unmarshal(dt, &manifest); err == nil && manifest.Config.Digest != "" {
+			referenced[manifest.Config.Digest] = struct{}{}
+			for _, l := range manifest.Layers {
+				referenced[l.Digest] = struct{}{}
+			}
+		}
+	}
+
+	var toDelete []digest.Digest
+	if err := cs.Walk(ctx, func(info content.Info) error {
+		if _, ok := referenced[info.Digest]; !ok {
+			toDelete = append(toDelete, info.Digest)
+		}
+		return nil
+	}); err != nil {
+		return errors.Wrap(err, "reset: failed to walk content store")
+	}
+
+	for _, dgst := range toDelete {
+		if err := cs.Delete(ctx, dgst); err != nil {
+			bklog.G(ctx).WithError(err).Warnf("reset: failed to delete blob %s", dgst)
+		}
+	}
+	return nil
+}
+
 func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (filesync.StaticDirSource, error) {
 	resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult {
 		st.Uid = 0
@@ -467,10 +520,16 @@ func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (
 	return result, nil
 }
 
+type cacheStoreRef struct {
+	path  string
+	store content.Store
+}
+
 type cacheOptions struct {
 	options        controlapi.CacheOptions
 	contentStores  map[string]content.Store // key: ID of content store ("local:" + csDir)
 	storesToUpdate map[string]string        // key: path to content store, value: tag
+	storesToReset  []cacheStoreRef          // cache stores with reset=true
 	frontendAttrs  map[string]string
 }
 
@@ -479,6 +538,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
 		cacheExports []*controlapi.CacheOptionsEntry
 		cacheImports []*controlapi.CacheOptionsEntry
 	)
+	var storesToReset []cacheStoreRef
 	contentStores := make(map[string]content.Store)
 	storesToUpdate := make(map[string]string)
 	frontendAttrs := make(map[string]string)
@@ -503,6 +563,16 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
 			}
 			// TODO(AkihiroSuda): support custom index JSON path and tag
 			storesToUpdate[csDir] = tag
+
+			if v, ok := ex.Attrs["reset"]; ok {
+				b, err := strconv.ParseBool(v)
+				if err != nil {
+					return nil, errors.Wrapf(err, "failed to parse reset attribute")
+				}
+				if b {
+					storesToReset = append(storesToReset, cacheStoreRef{path: csDir, store: cs})
+				}
+			}
 		}
 		if ex.Type == "registry" {
 			regRef := ex.Attrs["ref"]
@@ -582,6 +652,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
 		},
 		contentStores:  contentStores,
 		storesToUpdate: storesToUpdate,
+		storesToReset:  storesToReset,
 		frontendAttrs:  frontendAttrs,
 	}
 	return &res, nil

client/solve_resetcache_test.go

@@ -0,0 +1,167 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/containerd/containerd/v2/core/content"
+	contentlocal "github.com/containerd/containerd/v2/plugins/content/local"
+	"github.com/moby/buildkit/client/ociindex"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/require"
+)
+
+func writeBlob(ctx context.Context, t *testing.T, cs content.Store, data []byte) digest.Digest {
+	t.Helper()
+	dgst := digest.FromBytes(data)
+	desc := ocispecs.Descriptor{Digest: dgst, Size: int64(len(data))}
+	err := content.WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(data), desc)
+	require.NoError(t, err)
+	return dgst
+}
+
+func listDigests(ctx context.Context, t *testing.T, cs content.Store) map[digest.Digest]struct{} {
+	t.Helper()
+	result := map[digest.Digest]struct{}{}
+	err := cs.Walk(ctx, func(info content.Info) error {
+		result[info.Digest] = struct{}{}
+		return nil
+	})
+	require.NoError(t, err)
+	return result
+}
+
+// setupCacheStore creates a content store with an index.json and a manifest
+// referencing the given config and layers. Returns the store path, content
+// store, and manifest digest.
+func setupCacheStore(t *testing.T, ctx context.Context, configData []byte, layersData [][]byte, tag string) (string, content.Store, digest.Digest) {
+	t.Helper()
+	dir := t.TempDir()
+	cs, err := contentlocal.NewStore(dir)
+	require.NoError(t, err)
+
+	configDgst := writeBlob(ctx, t, cs, configData)
+
+	layers := make([]ocispecs.Descriptor, len(layersData))
+	for i, ld := range layersData {
+		dgst := writeBlob(ctx, t, cs, ld)
+		layers[i] = ocispecs.Descriptor{Digest: dgst, Size: int64(len(ld))}
+	}
+
+	manifest := ocispecs.Manifest{
+		MediaType: ocispecs.MediaTypeImageManifest,
+		Config: ocispecs.Descriptor{
+			Digest:    configDgst,
+			Size:      int64(len(configData)),
+			MediaType: "application/vnd.buildkit.cacheconfig.v0",
+		},
+		Layers: layers,
+	}
+	manifestData, err := json.Marshal(manifest)
+	require.NoError(t, err)
+	manifestDgst := writeBlob(ctx, t, cs, manifestData)
+
+	idx := ociindex.NewStoreIndex(dir)
+	err = idx.Put(ocispecs.Descriptor{
+		Digest:    manifestDgst,
+		Size:      int64(len(manifestData)),
+		MediaType: ocispecs.MediaTypeImageManifest,
+	}, ociindex.Tag(tag))
+	require.NoError(t, err)
+
+	return dir, cs, manifestDgst
+}
+
+func TestResetCacheStoreImageManifest(t *testing.T) {
+	ctx := context.Background()
+	dir, cs, manifestDgst := setupCacheStore(t, ctx,
+		[]byte(`{"test":"config"}`),
+		[][]byte{[]byte("layer1-data")},
+		"latest",
+	)
+
+	// Write orphan blob
+	orphanDgst := writeBlob(ctx, t, cs, []byte("orphan-old-layer"))
+
+	// Verify 4 blobs exist
+	require.Len(t, listDigests(ctx, t, cs), 4)
+
+	err := resetCacheStore(ctx, cs, dir)
+	require.NoError(t, err)
+
+	remaining := listDigests(ctx, t, cs)
+	require.Len(t, remaining, 3) // manifest + config + layer
+	require.Contains(t, remaining, manifestDgst)
+	require.NotContains(t, remaining, orphanDgst)
+}
+
+func TestResetCacheStoreMultipleTags(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	cs, err := contentlocal.NewStore(dir)
+	require.NoError(t, err)
+
+	// Manifest 1 (tag=v1)
+	config1Dgst := writeBlob(ctx, t, cs, []byte(`{"tag":"v1"}`))
+	layer1Dgst := writeBlob(ctx, t, cs, []byte("layer1-only-in-v1"))
+	m1 := ocispecs.Manifest{
+		MediaType: ocispecs.MediaTypeImageManifest,
+		Config:    ocispecs.Descriptor{Digest: config1Dgst, Size: 12, MediaType: "application/vnd.buildkit.cacheconfig.v0"},
+		Layers:    []ocispecs.Descriptor{{Digest: layer1Dgst, Size: 17}},
+	}
+	m1Data, err := json.Marshal(m1)
+	require.NoError(t, err)
+	m1Dgst := writeBlob(ctx, t, cs, m1Data)
+
+	// Manifest 2 (tag=v2)
+	config2Dgst := writeBlob(ctx, t, cs, []byte(`{"tag":"v2"}`))
+	layer2Dgst := writeBlob(ctx, t, cs, []byte("layer2-only-in-v2"))
+	m2 := ocispecs.Manifest{
+		MediaType: ocispecs.MediaTypeImageManifest,
+		Config:    ocispecs.Descriptor{Digest: config2Dgst, Size: 12, MediaType: "application/vnd.buildkit.cacheconfig.v0"},
+		Layers:    []ocispecs.Descriptor{{Digest: layer2Dgst, Size: 17}},
+	}
+	m2Data, err := json.Marshal(m2)
+	require.NoError(t, err)
+	m2Dgst := writeBlob(ctx, t, cs, m2Data)
+
+	// Orphan blob
+	orphanDgst := writeBlob(ctx, t, cs, []byte("orphan-blob"))
+
+	// Write index.json with both tags
+	idx := ociindex.NewStoreIndex(dir)
+	require.NoError(t, idx.Put(ocispecs.Descriptor{Digest: m1Dgst, Size: int64(len(m1Data)), MediaType: ocispecs.MediaTypeImageManifest}, ociindex.Tag("v1")))
+	require.NoError(t, idx.Put(ocispecs.Descriptor{Digest: m2Dgst, Size: int64(len(m2Data)), MediaType: ocispecs.MediaTypeImageManifest}, ociindex.Tag("v2")))
+
+	require.Len(t, listDigests(ctx, t, cs), 7)
+
+	err = resetCacheStore(ctx, cs, dir)
+	require.NoError(t, err)
+
+	remaining := listDigests(ctx, t, cs)
+	require.Len(t, remaining, 6)
+	require.Contains(t, remaining, m1Dgst)
+	require.Contains(t, remaining, config1Dgst)
+	require.Contains(t, remaining, layer1Dgst)
+	require.Contains(t, remaining, m2Dgst)
+	require.Contains(t, remaining, config2Dgst)
+	require.Contains(t, remaining, layer2Dgst)
+	require.NotContains(t, remaining, orphanDgst)
+}
+
+func TestResetCacheStoreNoOrphans(t *testing.T) {
+	ctx := context.Background()
+	dir, cs, _ := setupCacheStore(t, ctx,
+		[]byte(`{"test":"config"}`),
+		nil,
+		"latest",
+	)
+
+	err := resetCacheStore(ctx, cs, dir)
+	require.NoError(t, err)
+
+	require.Len(t, listDigests(ctx, t, cs), 2) // manifest + config
+}