Skip to content

Commit 4c9d94f

Browse files
committed
add cache key debuginfo lookup
This allows opt-in to cache key debug database on daemon startup. If enabled, all cache keys generated by builds are saved into this database together with the plaintexts of the original data so a reverse lookup can be performed later to compare two checksums and find out their original difference. If checksum contains other checksums internally then these are saved as well. For storage constraints, the plaintext of file content is not saved but the metadata portion can be still looked up. Signed-off-by: Tonis Tiigi <[email protected]>
1 parent 7bf2360 commit 4c9d94f

File tree

16 files changed

+819
-26
lines changed

16 files changed

+819
-26
lines changed

cache/contenthash/checksum.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package contenthash
33
import (
44
"bytes"
55
"context"
6-
"crypto/sha256"
76
"io"
87
"os"
98
"path"
@@ -18,6 +17,7 @@ import (
1817
"github.com/moby/buildkit/cache"
1918
"github.com/moby/buildkit/session"
2019
"github.com/moby/buildkit/snapshot"
20+
"github.com/moby/buildkit/util/cachedigest"
2121
"github.com/moby/locker"
2222
"github.com/moby/patternmatcher"
2323
digest "github.com/opencontainers/go-digest"
@@ -450,15 +450,15 @@ func (cc *cacheContext) Checksum(ctx context.Context, mountable cache.Mountable,
450450
return digest.Digest(includedPaths[0].record.Digest), nil
451451
}
452452

453-
digester := digest.Canonical.Digester()
453+
h := cachedigest.NewHash(cachedigest.TypeFileList)
454454
for i, w := range includedPaths {
455455
if i != 0 {
456-
digester.Hash().Write([]byte{0})
456+
h.Write([]byte{0})
457457
}
458-
digester.Hash().Write([]byte(path.Base(w.path)))
459-
digester.Hash().Write([]byte(w.record.Digest))
458+
h.Write([]byte(path.Base(w.path)))
459+
h.Write([]byte(w.record.Digest))
460460
}
461-
return digester.Digest(), nil
461+
return h.Sum(), nil
462462
}
463463

464464
func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, opts ChecksumOpts) ([]*includedPath, error) {
@@ -881,7 +881,7 @@ func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node[*CacheRe
881881

882882
switch cr.Type {
883883
case CacheRecordTypeDir:
884-
h := sha256.New()
884+
h := cachedigest.NewHash(cachedigest.TypeFileList)
885885
next := append(k, 0)
886886
iter := root.Iterator()
887887
iter.SeekLowerBound(append(slices.Clone(next), 0))
@@ -906,7 +906,7 @@ func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node[*CacheRe
906906
}
907907
subk, _, ok = iter.Next()
908908
}
909-
dgst = digest.NewDigest(digest.SHA256, h)
909+
dgst = h.Sum()
910910

911911
default:
912912
p := convertKeyToPath(bytes.TrimSuffix(k, []byte{0}))

cache/contenthash/filehash.go

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@ package contenthash
22

33
import (
44
"archive/tar"
5-
"crypto/sha256"
5+
"encoding/hex"
66
"hash"
77
"os"
88
"path/filepath"
99
"time"
1010

11+
"github.com/moby/buildkit/util/cachedigest"
1112
"github.com/pkg/errors"
1213
fstypes "github.com/tonistiigi/fsutil/types"
1314
)
@@ -62,13 +63,14 @@ func NewFromStat(stat *fstypes.Stat) (hash.Hash, error) {
6263
}
6364
}
6465
// fmt.Printf("hdr: %#v\n", hdr)
65-
tsh := &tarsumHash{hdr: hdr, Hash: sha256.New()}
66+
h := cachedigest.NewHash(cachedigest.TypeFile)
67+
tsh := &tarsumHash{hdr: hdr, Hash: h}
6668
tsh.Reset() // initialize header
6769
return tsh, nil
6870
}
6971

7072
type tarsumHash struct {
71-
hash.Hash
73+
*cachedigest.Hash
7274
hdr *tar.Header
7375
}
7476

@@ -79,6 +81,19 @@ func (tsh *tarsumHash) Reset() {
7981
WriteV1TarsumHeaders(tsh.hdr, tsh.Hash)
8082
}
8183

84+
func (tsh *tarsumHash) Write(p []byte) (n int, err error) {
85+
n, err = tsh.WriteNoDebug(p)
86+
if n > 0 {
87+
tsh.hdr.Size += int64(n)
88+
}
89+
return n, err
90+
}
91+
92+
func (tsh *tarsumHash) Sum(_ []byte) []byte {
93+
b, _ := hex.DecodeString(tsh.Hash.Sum().Hex())
94+
return b
95+
}
96+
8297
type statInfo struct {
8398
*fstypes.Stat
8499
}

cmd/buildkitd/debug.go

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package main
22

33
import (
4+
"context"
5+
"encoding/json"
46
"expvar"
57
"net/http"
68
"net/http/pprof"
@@ -10,6 +12,9 @@ import (
1012
"time"
1113

1214
"github.com/moby/buildkit/util/bklog"
15+
"github.com/moby/buildkit/util/cachedigest"
16+
digest "github.com/opencontainers/go-digest"
17+
"github.com/pkg/errors"
1318
"github.com/prometheus/client_golang/prometheus/promhttp"
1419
"golang.org/x/net/trace"
1520
)
@@ -24,6 +29,8 @@ func setupDebugHandlers(addr string) error {
2429
m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
2530
m.Handle("/debug/requests", http.HandlerFunc(trace.Traces))
2631
m.Handle("/debug/events", http.HandlerFunc(trace.Events))
32+
m.Handle("/debug/cache/all", http.HandlerFunc(handleCacheAll))
33+
m.Handle("/debug/cache/lookup", http.HandlerFunc(handleCacheLookup))
2734

2835
m.Handle("/debug/gc", http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
2936
runtime.GC()
@@ -59,3 +66,137 @@ func setupDebugHandlers(addr string) error {
5966
}()
6067
return nil
6168
}
69+
70+
func handleCacheAll(w http.ResponseWriter, r *http.Request) {
71+
records, err := loadCacheAll(r.Context())
72+
if err != nil {
73+
http.Error(w, err.Error(), http.StatusInternalServerError)
74+
return
75+
}
76+
switch r.Header.Get("Accept") {
77+
case "application/json":
78+
w.Header().Set("Content-Type", "application/json")
79+
enc := json.NewEncoder(w)
80+
enc.SetIndent("", " ")
81+
enc.Encode(records)
82+
default:
83+
w.Header().Set("Content-Type", "text/plain")
84+
for _, rec := range records {
85+
w.Write([]byte(rec.Digest.String() + " (" + rec.Type.String() + "):\n"))
86+
for _, subRec := range rec.SubRecords {
87+
w.Write([]byte(" " + subRec.Digest.String() + " (" + subRec.Type.String() + "):\n"))
88+
}
89+
for _, frame := range rec.Data {
90+
switch frame.ID {
91+
case cachedigest.FrameIDData:
92+
w.Write([]byte(" " + frame.ID.String() + ": " + string(frame.Data) + "\n"))
93+
case cachedigest.FrameIDSkip:
94+
w.Write([]byte(" skipping " + string(frame.Data) + " bytes\n"))
95+
}
96+
}
97+
w.Write([]byte("\n"))
98+
}
99+
}
100+
}
101+
102+
func handleCacheLookup(w http.ResponseWriter, r *http.Request) {
103+
dgstStr := r.URL.Query().Get("digest")
104+
if dgstStr == "" {
105+
http.Error(w, "digest query parameter is required", http.StatusBadRequest)
106+
return
107+
}
108+
109+
dgst, err := digest.Parse(dgstStr)
110+
if err != nil {
111+
http.Error(w, "invalid digest: "+err.Error(), http.StatusBadRequest)
112+
return
113+
}
114+
115+
record, err := cacheRecordLookup(r.Context(), dgst)
116+
if err != nil {
117+
http.Error(w, err.Error(), http.StatusInternalServerError)
118+
return
119+
}
120+
121+
switch r.Header.Get("Accept") {
122+
case "application/json":
123+
w.Header().Set("Content-Type", "application/json")
124+
if err := json.NewEncoder(w).Encode(record); err != nil {
125+
http.Error(w, "failed to encode record: "+err.Error(), http.StatusInternalServerError)
126+
return
127+
}
128+
default:
129+
w.Header().Set("Content-Type", "text/plain")
130+
w.Write([]byte(record.Digest.String() + " (" + record.Type.String() + "):\n"))
131+
for _, subRec := range record.SubRecords {
132+
w.Write([]byte(" " + subRec.Digest.String() + " (" + subRec.Type.String() + "):\n"))
133+
}
134+
for _, frame := range record.Data {
135+
switch frame.ID {
136+
case cachedigest.FrameIDData:
137+
w.Write([]byte(" " + frame.ID.String() + ": " + string(frame.Data) + "\n"))
138+
case cachedigest.FrameIDSkip:
139+
w.Write([]byte(" skipping " + string(frame.Data) + " bytes\n"))
140+
}
141+
}
142+
}
143+
}
144+
145+
func cacheRecordLookup(ctx context.Context, dgst digest.Digest) (*cachedigest.Record, error) {
146+
db := cachedigest.GetDefaultDB()
147+
typ, frames, err := db.Get(ctx, dgst.String())
148+
if err != nil {
149+
return nil, errors.Wrapf(err, "failed to get digest %s from cache", dgst.String())
150+
}
151+
record := &cachedigest.Record{
152+
Digest: dgst,
153+
Type: typ,
154+
Data: frames,
155+
}
156+
if err := record.LoadSubRecords(func(d digest.Digest) (cachedigest.Type, []cachedigest.Frame, error) {
157+
typ, frames, err := db.Get(ctx, d.String())
158+
if err != nil {
159+
return "", nil, errors.Wrapf(err, "failed to load sub-record for %s", d.String())
160+
}
161+
return typ, frames, nil
162+
}); err != nil {
163+
return nil, errors.Wrapf(err, "failed to load sub-records for %s", dgst.String())
164+
}
165+
return record, nil
166+
}
167+
168+
func loadCacheAll(ctx context.Context) ([]*cachedigest.Record, error) {
169+
var records []*cachedigest.Record
170+
m := map[digest.Digest]*cachedigest.Record{}
171+
db := cachedigest.GetDefaultDB()
172+
err := db.All(ctx, func(key string, typ cachedigest.Type, frames []cachedigest.Frame) error {
173+
dgst, err := digest.Parse(key)
174+
if err != nil {
175+
return errors.Wrapf(err, "failed to parse digest %q", key)
176+
}
177+
r := &cachedigest.Record{
178+
Digest: dgst,
179+
Type: typ,
180+
Data: frames,
181+
}
182+
records = append(records, r)
183+
m[dgst] = r
184+
return nil
185+
})
186+
if err != nil {
187+
return nil, err
188+
}
189+
190+
for _, rec := range records {
191+
if err := rec.LoadSubRecords(func(d digest.Digest) (cachedigest.Type, []cachedigest.Frame, error) {
192+
rec, ok := m[d]
193+
if !ok {
194+
return "", nil, errors.Errorf("digest %s not found in cache", d)
195+
}
196+
return rec.Type, rec.Data, nil
197+
}); err != nil {
198+
return nil, errors.Wrapf(err, "failed to load sub-records for %s", rec.Digest.String())
199+
}
200+
}
201+
return records, nil
202+
}

cmd/buildkitd/main.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import (
4545
"github.com/moby/buildkit/util/appdefaults"
4646
"github.com/moby/buildkit/util/archutil"
4747
"github.com/moby/buildkit/util/bklog"
48+
"github.com/moby/buildkit/util/cachedigest"
4849
"github.com/moby/buildkit/util/db/boltutil"
4950
"github.com/moby/buildkit/util/disk"
5051
"github.com/moby/buildkit/util/grpcerrors"
@@ -225,6 +226,10 @@ func main() {
225226
Name: "cdi-spec-dir",
226227
Usage: "list of directories to scan for CDI spec files",
227228
},
229+
cli.BoolFlag{
230+
Name: "save-cache-debug",
231+
Usage: "enable saving cache debug info",
232+
},
228233
)
229234
app.Flags = append(app.Flags, appFlags...)
230235
app.Flags = append(app.Flags, serviceFlags()...)
@@ -345,6 +350,15 @@ func main() {
345350
return err
346351
}
347352

353+
if c.GlobalBool("save-cache-debug") {
354+
db, err := cachedigest.NewDB(filepath.Join(cfg.Root, "cache-debug.db"))
355+
if err != nil {
356+
return errors.Wrap(err, "failed to create cache debug db")
357+
}
358+
cachedigest.SetDefaultDB(db)
359+
defer db.Close()
360+
}
361+
348362
controller, err := newController(ctx, c, &cfg)
349363
if err != nil {
350364
return err

solver/cachemanager.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
"github.com/moby/buildkit/identity"
1111
"github.com/moby/buildkit/util/bklog"
12+
"github.com/moby/buildkit/util/cachedigest"
1213
digest "github.com/opencontainers/go-digest"
1314
"github.com/sirupsen/logrus"
1415
)
@@ -448,8 +449,9 @@ func (c *cacheManager) getIDFromDeps(k *CacheKey) string {
448449
}
449450

450451
func rootKey(dgst digest.Digest, output Index) digest.Digest {
452+
dgst, _ = cachedigest.FromBytes(fmt.Appendf(nil, "%s@%d", dgst, output), cachedigest.TypeString)
451453
if strings.HasPrefix(dgst.String(), "random:") {
452-
return digest.Digest("random:" + digest.FromBytes(fmt.Appendf(nil, "%s@%d", dgst, output)).Encoded())
454+
return digest.Digest("random:" + dgst.Encoded())
453455
}
454-
return digest.FromBytes(fmt.Appendf(nil, "%s@%d", dgst, output))
456+
return dgst
455457
}

solver/llbsolver/ops/build.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/moby/buildkit/solver"
1414
"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
1515
"github.com/moby/buildkit/solver/pb"
16+
"github.com/moby/buildkit/util/cachedigest"
1617
"github.com/moby/buildkit/worker"
1718
digest "github.com/opencontainers/go-digest"
1819
"github.com/pkg/errors"
@@ -51,8 +52,12 @@ func (b *BuildOp) CacheMap(ctx context.Context, g session.Group, index int) (*so
5152
return nil, false, err
5253
}
5354

55+
dgst, err := cachedigest.FromBytes(dt, cachedigest.TypeJSON)
56+
if err != nil {
57+
return nil, false, err
58+
}
5459
return &solver.CacheMap{
55-
Digest: digest.FromBytes(dt),
60+
Digest: dgst,
5661
Deps: make([]struct {
5762
Selector digest.Digest
5863
ComputeDigestFunc solver.ResultBasedCacheFunc

solver/llbsolver/ops/exec.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/moby/buildkit/solver/llbsolver/mounts"
2424
"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
2525
"github.com/moby/buildkit/solver/pb"
26+
"github.com/moby/buildkit/util/cachedigest"
2627
"github.com/moby/buildkit/util/progress/logs"
2728
utilsystem "github.com/moby/buildkit/util/system"
2829
"github.com/moby/buildkit/worker"
@@ -173,8 +174,12 @@ func (e *ExecOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol
173174
return nil, false, err
174175
}
175176

177+
dgst, err := cachedigest.FromBytes(dt, cachedigest.TypeJSON)
178+
if err != nil {
179+
return nil, false, err
180+
}
176181
cm := &solver.CacheMap{
177-
Digest: digest.FromBytes(dt),
182+
Digest: dgst,
178183
Deps: make([]struct {
179184
Selector digest.Digest
180185
ComputeDigestFunc solver.ResultBasedCacheFunc

0 commit comments

Comments
 (0)