Skip to content

Commit 4586db5

Browse files
xnoxxnox
authored
lock: populate auto-discovered keys into the json lock file (#1985)
- **Commit claude code** - **Export ParseAlpineVersion, instead of wrapping it** - **Ignore missing alpine keys when offline** - **Add new lock test with alpine discovery keys** - **lock: Perform chainguard-style key discovery** - **Ensure stable lock output** Populate auto-discovered keys into the expanded lock file. This helps with subsequent offline builds, plugging into existing (fixed up) caching mechanism which appears to fix all offline building issues for me on the rules_apko side. I think this will correctly support multi-arch builds, with per-arch keys, with auto-discovered keys, actually offline for the first time properly.
1 parent f62a501 commit 4586db5

File tree

5 files changed

+328
-36
lines changed

5 files changed

+328
-36
lines changed

‎internal/cli/lock.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,20 @@ import (
1818
"context"
1919
"encoding/base64"
2020
"fmt"
21+
"net/http"
2122
"os"
2223
"path/filepath"
2324
"slices"
25+
"sort"
2426
"strings"
27+
"time"
2528

2629
"github.com/spf13/cobra"
2730

2831
"github.com/chainguard-dev/clog"
2932

3033
"chainguard.dev/apko/pkg/apk/apk"
34+
"chainguard.dev/apko/pkg/apk/auth"
3135
apkfs "chainguard.dev/apko/pkg/apk/fs"
3236
"chainguard.dev/apko/pkg/build"
3337
"chainguard.dev/apko/pkg/build/types"
@@ -169,6 +173,10 @@ func LockCmd(ctx context.Context, output string, archs []types.Architecture, opt
169173
})
170174
}
171175

176+
// Discover and add auto-discovered keys from repositories
177+
discoveredKeys := discoverKeysForLock(ctx, ic, archs)
178+
lock.Contents.Keyrings = append(lock.Contents.Keyrings, discoveredKeys...)
179+
172180
// TODO: If the archs can't agree on package versions (e.g., arm builds are ahead of x86) then we should fail instead of producing inconsistent locks.
173181
for _, arch := range archs {
174182
log := log.With("arch", arch.ToAPK())
@@ -236,6 +244,12 @@ func LockCmd(ctx context.Context, output string, archs []types.Architecture, opt
236244
lock.Contents.Repositories = append(lock.Contents.Repositories, repoLock)
237245
}
238246
}
247+
248+
// Sort keyrings by name for reproducible lock files
249+
sort.Slice(lock.Contents.Keyrings, func(i, j int) bool {
250+
return lock.Contents.Keyrings[i].Name < lock.Contents.Keyrings[j].Name
251+
})
252+
239253
return lock.SaveToFile(output)
240254
}
241255

@@ -262,3 +276,94 @@ func stripURLScheme(url string) string {
262276
"http://",
263277
)
264278
}
279+
280+
// discoverKeysForLock discovers keys from repositories and returns them as LockKeyring entries
281+
func discoverKeysForLock(ctx context.Context, ic *types.ImageConfiguration, archs []types.Architecture) []pkglock.LockKeyring {
282+
log := clog.FromContext(ctx)
283+
284+
// Collect all unique repositories
285+
repoSet := make(map[string]struct{})
286+
for _, repo := range ic.Contents.BuildRepositories {
287+
repoSet[repo] = struct{}{}
288+
}
289+
for _, repo := range ic.Contents.RuntimeOnlyRepositories {
290+
repoSet[repo] = struct{}{}
291+
}
292+
for _, repo := range ic.Contents.Repositories {
293+
repoSet[repo] = struct{}{}
294+
}
295+
296+
// Map to track discovered keys by URL to avoid duplicates
297+
discoveredKeyMap := make(map[string]pkglock.LockKeyring)
298+
299+
// Fetch Alpine releases once (cached by HTTP client)
300+
client := &http.Client{}
301+
var alpineReleases *apk.Releases
302+
303+
// Discover keys for each repository and architecture
304+
for repo := range repoSet {
305+
// Try Alpine-style key discovery
306+
if ver, ok := apk.ParseAlpineVersion(repo); ok {
307+
// Fetch releases.json if not already fetched
308+
if alpineReleases == nil {
309+
releases, err := apk.FetchAlpineReleases(ctx, client)
310+
if err != nil {
311+
log.Warnf("Failed to fetch Alpine releases: %v", err)
312+
continue
313+
}
314+
alpineReleases = releases
315+
}
316+
317+
branch := alpineReleases.GetReleaseBranch(ver)
318+
if branch == nil {
319+
log.Debugf("Alpine version %s not found in releases", ver)
320+
continue
321+
}
322+
323+
// Get keys for each architecture
324+
for _, arch := range archs {
325+
log.Debugf("Discovering Alpine keys for %s (version %s, arch %s)", repo, ver, arch.ToAPK())
326+
urls := branch.KeysFor(arch.ToAPK(), time.Now())
327+
if len(urls) == 0 {
328+
log.Debugf("No keys found for arch %s and version %s", arch.ToAPK(), ver)
329+
continue
330+
}
331+
332+
// Add discovered key URLs to the map
333+
for _, u := range urls {
334+
discoveredKeyMap[u] = pkglock.LockKeyring{
335+
Name: stripURLScheme(u),
336+
URL: u,
337+
}
338+
}
339+
}
340+
}
341+
342+
// Try Chainguard-style key discovery
343+
log.Debugf("Attempting Chainguard-style key discovery for %s", repo)
344+
keys, err := apk.DiscoverKeys(ctx, client, auth.DefaultAuthenticators, repo)
345+
if err != nil {
346+
log.Debugf("Chainguard-style key discovery failed for %s: %v", repo, err)
347+
} else if len(keys) > 0 {
348+
log.Debugf("Discovered %d Chainguard-style keys for %s", len(keys), repo)
349+
// For each JWKS key, emit a URL: repository + "/" + KeyID
350+
repoBase := strings.TrimSuffix(repo, "/")
351+
for _, key := range keys {
352+
keyURL := repoBase + "/" + key.ID
353+
discoveredKeyMap[keyURL] = pkglock.LockKeyring{
354+
Name: stripURLScheme(keyURL),
355+
URL: keyURL,
356+
}
357+
}
358+
}
359+
}
360+
361+
// Convert map to slice
362+
discoveredKeys := make([]pkglock.LockKeyring, 0, len(discoveredKeyMap))
363+
for _, key := range discoveredKeyMap {
364+
discoveredKeys = append(discoveredKeys, key)
365+
}
366+
367+
log.Infof("Discovered %d auto-discovered keys", len(discoveredKeys))
368+
return discoveredKeys
369+
}

‎internal/cli/lock_test.go‎

Lines changed: 32 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -33,25 +33,38 @@ func TestLock(t *testing.T) {
3333
ctx := context.Background()
3434
tmp := t.TempDir()
3535

36-
golden := filepath.Join("testdata", "apko.lock.json")
37-
38-
config := "apko.yaml"
39-
archs := types.ParseArchitectures([]string{"amd64", "arm64"})
40-
opts := []build.Option{build.WithConfig(config, []string{"testdata"})}
41-
outputPath := filepath.Join(tmp, "apko.lock.json")
42-
43-
err := cli.LockCmd(ctx, outputPath, archs, opts)
44-
require.NoError(t, err)
45-
46-
want, err := os.ReadFile(golden)
47-
require.NoError(t, err)
48-
got, err := os.ReadFile(outputPath)
49-
require.NoError(t, err)
50-
51-
if !bytes.Equal(want, got) {
52-
if diff := cmp.Diff(want, got); diff != "" {
53-
t.Errorf("Mismatched lock files: (-%q +%q):\n%s", golden, outputPath, diff)
54-
}
36+
tests := []struct {
37+
basename string
38+
}{
39+
{
40+
basename: "apko",
41+
}, {
42+
basename: "apko-discover",
43+
},
44+
}
45+
for _, tt := range tests {
46+
t.Run(tt.basename, func(t *testing.T) {
47+
golden := filepath.Join("testdata", tt.basename+".lock.json")
48+
49+
config := tt.basename + ".yaml"
50+
archs := types.ParseArchitectures([]string{"amd64", "arm64"})
51+
opts := []build.Option{build.WithConfig(config, []string{"testdata"})}
52+
outputPath := filepath.Join(tmp, tt.basename+".lock.json")
53+
54+
err := cli.LockCmd(ctx, outputPath, archs, opts)
55+
require.NoError(t, err)
56+
57+
want, err := os.ReadFile(golden)
58+
require.NoError(t, err)
59+
got, err := os.ReadFile(outputPath)
60+
require.NoError(t, err)
61+
62+
if !bytes.Equal(want, got) {
63+
if diff := cmp.Diff(want, got); diff != "" {
64+
t.Errorf("Mismatched lock files: (-%q +%q):\n%s", golden, outputPath, diff)
65+
}
66+
}
67+
})
5568
}
5669
}
5770

‎internal/cli/testdata/apko-discover.lock.json‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
{
2+
"version": "v1",
3+
"config": {
4+
"name": "apko-discover.yaml",
5+
"checksum": "sha256-D1mtHLoTBln0iH+VqWZMIPQOScXMWprVolB2sihPWAk="
6+
},
7+
"contents": {
8+
"keyring": [
9+
{
10+
"name": "./testdata/melange.rsa.pub",
11+
"url": "./testdata/melange.rsa.pub"
12+
},
13+
{
14+
"name": "alpinelinux.org/keys/alpine-devel%40lists.alpinelinux.org-6165ee59.rsa.pub",
15+
"url": "https://alpinelinux.org/keys/alpine-devel%40lists.alpinelinux.org-6165ee59.rsa.pub"
16+
},
17+
{
18+
"name": "alpinelinux.org/keys/alpine-devel%40lists.alpinelinux.org-616ae350.rsa.pub",
19+
"url": "https://alpinelinux.org/keys/alpine-devel%40lists.alpinelinux.org-616ae350.rsa.pub"
20+
},
21+
{
22+
"name": "apk.cgr.dev/chainguard/chainguard-6ea100e7571978828f8455393090cc468e1f22dfc771ad2d26df14e52b73c37f.rsa.pub",
23+
"url": "https://apk.cgr.dev/chainguard/chainguard-6ea100e7571978828f8455393090cc468e1f22dfc771ad2d26df14e52b73c37f.rsa.pub"
24+
},
25+
{
26+
"name": "apk.cgr.dev/chainguard/chainguard-7fb528a64a862d44bbea6069093f1fec29fa864ba7e9754828eeceed3f487239.rsa.pub",
27+
"url": "https://apk.cgr.dev/chainguard/chainguard-7fb528a64a862d44bbea6069093f1fec29fa864ba7e9754828eeceed3f487239.rsa.pub"
28+
},
29+
{
30+
"name": "apk.cgr.dev/chainguard/chainguard-a87b67b52e9fa08414a5535a99adc7b66643536ccd52f4da66057b67adf21363.rsa.pub",
31+
"url": "https://apk.cgr.dev/chainguard/chainguard-a87b67b52e9fa08414a5535a99adc7b66643536ccd52f4da66057b67adf21363.rsa.pub"
32+
}
33+
],
34+
"build_repositories": [],
35+
"runtime_repositories": [],
36+
"repositories": [
37+
{
38+
"name": "dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64",
39+
"url": "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64/APKINDEX.tar.gz",
40+
"architecture": "x86_64"
41+
},
42+
{
43+
"name": "apk.cgr.dev/chainguard/x86_64",
44+
"url": "https://apk.cgr.dev/chainguard/x86_64/APKINDEX.tar.gz",
45+
"architecture": "x86_64"
46+
},
47+
{
48+
"name": "./testdata/packages/x86_64",
49+
"url": "./testdata/packages/x86_64/APKINDEX.tar.gz",
50+
"architecture": "x86_64"
51+
},
52+
{
53+
"name": "dl-cdn.alpinelinux.org/alpine/v3.22/main/aarch64",
54+
"url": "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/aarch64/APKINDEX.tar.gz",
55+
"architecture": "aarch64"
56+
},
57+
{
58+
"name": "apk.cgr.dev/chainguard/aarch64",
59+
"url": "https://apk.cgr.dev/chainguard/aarch64/APKINDEX.tar.gz",
60+
"architecture": "aarch64"
61+
},
62+
{
63+
"name": "./testdata/packages/aarch64",
64+
"url": "./testdata/packages/aarch64/APKINDEX.tar.gz",
65+
"architecture": "aarch64"
66+
}
67+
],
68+
"packages": [
69+
{
70+
"name": "pretend-baselayout",
71+
"url": "./testdata/packages/x86_64/pretend-baselayout-1.0.0-r0.apk",
72+
"version": "1.0.0-r0",
73+
"architecture": "x86_64",
74+
"signature": {
75+
"range": "bytes=0-648",
76+
"checksum": "sha1-V+Htugmm+Ru2ogsWm7VgD4A1DsQ="
77+
},
78+
"control": {
79+
"range": "bytes=649-1562",
80+
"checksum": "sha1-DRtLIHolxOMB++9L4ZjkeUFaKYc="
81+
},
82+
"data": {
83+
"range": "bytes=1563-2767",
84+
"checksum": "sha256-dZB1iTdQ2sfndKG6Ohf5VwWFjqz6kjSzZbqYU17BSRM="
85+
},
86+
"checksum": "Q1DRtLIHolxOMB++9L4ZjkeUFaKYc="
87+
},
88+
{
89+
"name": "replayout",
90+
"url": "./testdata/packages/x86_64/replayout-1.0.0-r0.apk",
91+
"version": "1.0.0-r0",
92+
"architecture": "x86_64",
93+
"signature": {
94+
"range": "bytes=0-647",
95+
"checksum": "sha1-ZrPCeQ4XeDjZSQw+IhJ4g4BcUlo="
96+
},
97+
"control": {
98+
"range": "bytes=648-1589",
99+
"checksum": "sha1-IvTcfj6zzLipr9akZ+YRTIyQCr8="
100+
},
101+
"data": {
102+
"range": "bytes=1590-2786",
103+
"checksum": "sha256-IIzbGjwv4H9h6N1bEbF8p4cqkV0Ex54sXEsvf6txnEo="
104+
},
105+
"checksum": "Q1IvTcfj6zzLipr9akZ+YRTIyQCr8="
106+
},
107+
{
108+
"name": "pretend-baselayout",
109+
"url": "./testdata/packages/aarch64/pretend-baselayout-1.0.0-r0.apk",
110+
"version": "1.0.0-r0",
111+
"architecture": "aarch64",
112+
"signature": {
113+
"range": "bytes=0-644",
114+
"checksum": "sha1-n9SJ91H1UwE+mkVVCifh6ziTwbc="
115+
},
116+
"control": {
117+
"range": "bytes=645-1555",
118+
"checksum": "sha1-URAMn9SfiCvjs6C812GovkgRgVo="
119+
},
120+
"data": {
121+
"range": "bytes=1556-2762",
122+
"checksum": "sha256-igCqZKcxRp6yHq2IlyozkM68Z6v9PMr/PDAIuSh2W3A="
123+
},
124+
"checksum": "Q1URAMn9SfiCvjs6C812GovkgRgVo="
125+
},
126+
{
127+
"name": "replayout",
128+
"url": "./testdata/packages/aarch64/replayout-1.0.0-r0.apk",
129+
"version": "1.0.0-r0",
130+
"architecture": "aarch64",
131+
"signature": {
132+
"range": "bytes=0-646",
133+
"checksum": "sha1-1ifrimC4bUlo4O6aCGXcjOKAcTo="
134+
},
135+
"control": {
136+
"range": "bytes=647-1586",
137+
"checksum": "sha1-SWYSZF3dGLrN8kebGjOBfDH6vG4="
138+
},
139+
"data": {
140+
"range": "bytes=1587-2787",
141+
"checksum": "sha256-U85iWddHApXsVosa4S0srhkr3SSlzuoBrEpZg5f1irQ="
142+
},
143+
"checksum": "Q1SWYSZF3dGLrN8kebGjOBfDH6vG4="
144+
}
145+
]
146+
}
147+
}

‎internal/cli/testdata/apko-discover.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
contents:
2+
keyring:
3+
- ./testdata/melange.rsa.pub
4+
repositories:
5+
- https://dl-cdn.alpinelinux.org/alpine/v3.22/main
6+
- https://apk.cgr.dev/chainguard
7+
- ./testdata/packages
8+
packages:
9+
- replayout
10+
11+
entrypoint:
12+
command: /bin/sh -l
13+
14+
archs:
15+
- x86_64
16+
- aarch64

0 commit comments

Comments
 (0)