Skip to content

Commit af7b858

Browse files
tianontianon
committed
Refactor meta.jq to prepare for signing and oci-import implementation
- explicit "containerd image storage in dockerd" placeholder - explicit "annotations" helper/generator (so they can be added in the `oci-import` builder later) - setting the `org.opencontainers.image.created` annoation inside the image index to the actual build date (otherwise `SOURCE_DATE_EPOCH` makes it hard to tell an image is actually freshly built) - more whitespace in generated commands for better readability - extract OCI tar in the build step (so the "output" of the build is an OCI layout directory vs tarball) instead of the pull step (so we can more ergonomically add pre-push signing) - use buildx's `--annotation` flag for making annotations easier to read
1 parent 7d4bdda commit af7b858

File tree

3 files changed

+180
-67
lines changed

3 files changed

+180
-67
lines changed

‎.test/example-commands.sh

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,49 @@
33

44
# </pull>
55
# <build>
6-
SOURCE_DATE_EPOCH=1700741054 docker buildx build --progress=plain --provenance=mode=max --sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" --output '"type=oci","dest=temp.tar","annotation.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation-manifest-descriptor.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation-manifest-descriptor.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation.org.opencontainers.image.version=24.0.7-cli","annotation-manifest-descriptor.org.opencontainers.image.version=24.0.7-cli","annotation.org.opencontainers.image.url=https://hub.docker.com/_/docker","annotation-manifest-descriptor.org.opencontainers.image.url=https://hub.docker.com/_/docker"' --tag 'docker:24.0.7-cli' --tag 'docker:24.0-cli' --tag 'docker:24-cli' --tag 'docker:cli' --tag 'docker:24.0.7-cli-alpine3.18' --tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' --platform 'linux/amd64' --build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' --build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli'
7-
# </build>
8-
# <push>
6+
SOURCE_DATE_EPOCH=1700741054 \
7+
docker buildx build --progress=plain \
8+
--provenance=mode=max \
9+
--sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \
10+
--output '"type=oci","dest=temp.tar"' \
11+
--annotation 'org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \
12+
--annotation 'org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \
13+
--annotation 'org.opencontainers.image.created=2023-11-23T12:04:14Z' \
14+
--annotation 'org.opencontainers.image.version=24.0.7-cli' \
15+
--annotation 'org.opencontainers.image.url=https://hub.docker.com/_/docker' \
16+
--annotation 'manifest-descriptor:org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \
17+
--annotation 'manifest-descriptor:org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \
18+
--annotation 'manifest-descriptor:org.opencontainers.image.created=1970-01-01T00:00:00Z' \
19+
--annotation 'manifest-descriptor:org.opencontainers.image.version=24.0.7-cli' \
20+
--annotation 'manifest-descriptor:org.opencontainers.image.url=https://hub.docker.com/_/docker' \
21+
--tag 'docker:24.0.7-cli' \
22+
--tag 'docker:24.0-cli' \
23+
--tag 'docker:24-cli' \
24+
--tag 'docker:cli' \
25+
--tag 'docker:24.0.7-cli-alpine3.18' \
26+
--tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' \
27+
--platform 'linux/amd64' \
28+
--build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' \
29+
--build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" \
30+
--file 'Dockerfile' \
31+
'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli'
932
mkdir temp
1033
tar -xvf temp.tar -C temp
11-
jq '.manifests |= (del(.[].annotations) | unique)' temp/index.json > temp/index.json.new
34+
rm temp.tar
35+
jq '
36+
.manifests |= (
37+
del(.[].annotations)
38+
| unique
39+
| if length != 1 then
40+
error("unexpected number of manifests: " + length)
41+
else . end
42+
)
43+
' temp/index.json > temp/index.json.new
1244
mv temp/index.json.new temp/index.json
45+
# </build>
46+
# <push>
1347
crane push temp 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43'
14-
rm -rf temp temp.tar
48+
rm -rf temp
1549
# </push>
1650

1751
# docker:24.0.7-windowsservercore-ltsc2022 [windows-amd64]
@@ -20,7 +54,21 @@ docker pull 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fc
2054
docker tag 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fce6edc5df459565a4c96bbb1d0148065b215ab5ddcab1e42eb4' 'mcr.microsoft.com/windows/servercore:ltsc2022'
2155
# </pull>
2256
# <build>
23-
SOURCE_DATE_EPOCH=1700741054 DOCKER_BUILDKIT=0 docker build --tag 'docker:24.0.7-windowsservercore-ltsc2022' --tag 'docker:24.0-windowsservercore-ltsc2022' --tag 'docker:24-windowsservercore-ltsc2022' --tag 'docker:windowsservercore-ltsc2022' --tag 'docker:24.0.7-windowsservercore' --tag 'docker:24.0-windowsservercore' --tag 'docker:24-windowsservercore' --tag 'docker:windowsservercore' --tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' --platform 'windows/amd64' --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022'
57+
SOURCE_DATE_EPOCH=1700741054 \
58+
DOCKER_BUILDKIT=0 \
59+
docker build \
60+
--tag 'docker:24.0.7-windowsservercore-ltsc2022' \
61+
--tag 'docker:24.0-windowsservercore-ltsc2022' \
62+
--tag 'docker:24-windowsservercore-ltsc2022' \
63+
--tag 'docker:windowsservercore-ltsc2022' \
64+
--tag 'docker:24.0.7-windowsservercore' \
65+
--tag 'docker:24.0-windowsservercore' \
66+
--tag 'docker:24-windowsservercore' \
67+
--tag 'docker:windowsservercore' \
68+
--tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' \
69+
--platform 'windows/amd64' \
70+
--file 'Dockerfile' \
71+
'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022'
2472
# </build>
2573
# <push>
2674
docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e'

‎.test/test.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ time "$dir/../sources.sh" "$@" > "$dir/sources.json"
1919
time "$dir/../builds.sh" --cache "$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json"
2020

2121
# generate an "example commands" file so that changes to generated commands are easier to review
22-
jq -r -L "$dir/.." '
22+
SOURCE_DATE_EPOCH=0 jq -r -L "$dir/.." '
2323
include "meta";
2424
[
2525
first(.[] | select(normalized_builder == "buildkit")),

‎meta.jq

Lines changed: 125 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,26 @@ def normalized_builder:
1717
end
1818
else . end
1919
;
20+
def docker_uses_containerd_storage:
21+
# TODO somehow detect docker-with-containerd-storage
22+
false
23+
;
2024
# input: "build" object (with "buildId" top level key)
2125
# output: boolean
2226
def should_use_docker_buildx_driver:
2327
normalized_builder == "buildkit"
2428
and (
25-
.build.arch as $arch
26-
# bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c
27-
| ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"]
28-
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
29-
| index($arch)
30-
| not
29+
docker_uses_containerd_storage
30+
or (
31+
.build.arch as $arch
32+
# bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c
33+
| ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"]
34+
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
35+
| index($arch)
36+
| not
37+
# TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit"; need *all* the arches we care about/support for consistent support)
38+
)
3139
)
32-
# TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit")
3340
;
3441
# input: "build" object (with "buildId" top level key)
3542
# output: string "pull command" ("docker pull ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail", might be empty
@@ -71,66 +78,110 @@ def git_build_url:
7178
) + "#" + .GitCommit + ":" + .Directory
7279
;
7380
# input: "build" object (with "buildId" top level key)
81+
# output: map of annotations to set
82+
def build_annotations($buildUrl):
83+
{
84+
# https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys
85+
"org.opencontainers.image.source": $buildUrl,
86+
"org.opencontainers.image.revision": .source.entry.GitCommit,
87+
"org.opencontainers.image.created": (.source.entry.SOURCE_DATE_EPOCH | strftime("%FT%TZ")), # see notes below about image index vs image manifest
88+
89+
# TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption)
90+
"org.opencontainers.image.version": ( # value of the first image tag
91+
first(.source.allTags[] | select(contains(":")))
92+
| sub("^.*:"; "")
93+
# TODO maybe we should do the first, longest, non-latest tag instead of just the first tag?
94+
),
95+
"org.opencontainers.image.url": ( # URL to Docker Hub
96+
first(.source.allTags[] | select(contains(":")))
97+
| sub(":.*$"; "")
98+
| if contains("/") then
99+
"r/" + .
100+
else
101+
"_/" + .
102+
end
103+
| "https://hub.docker.com/" + .
104+
),
105+
106+
# TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic)
107+
}
108+
| with_entries(select(.value)) # strip off anything missing a value (possibly "source", "url", "version", etc)
109+
;
110+
def build_annotations:
111+
build_annotations(git_build_url)
112+
;
113+
# input: multi-line string with indentation and comments
114+
# output: multi-line string with less indentation and no comments
115+
def unindent_and_decomment_jq($indents):
116+
# trim out comment lines and unnecessary indentation
117+
gsub("(?m)^(\t+#[^\n]*\n?|\t{\($indents)}(?<extra>.*)$)"; "\(.extra // "")")
118+
# trim out empty lines
119+
| gsub("\n\n+"; "\n")
120+
;
121+
# input: "build" object (with "buildId" top level key)
74122
# output: string "build command" ("docker buildx build ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail"
75123
def build_command:
76124
normalized_builder as $builder
77-
| git_build_url as $buildUrl
78125
| if $builder == "buildkit" then
79-
[
126+
git_build_url as $buildUrl
127+
| (
128+
(should_use_docker_buildx_driver | not)
129+
or docker_uses_containerd_storage
130+
) as $supportsAnnotationsAndAttestsations
131+
| [
80132
(
81133
[
82134
@sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)",
83135
# TODO EXPERIMENTAL_BUILDKIT_SOURCE_POLICY=<(jq ...)
84136
"docker buildx build --progress=plain",
85-
if should_use_docker_buildx_driver then "--load" else # TODO if we get containerd integration and thus use "--load" unconditionally again, we should update this to still set annotations! (and still gate SBOMs on appropriate scanner-supported architectures)
137+
if $supportsAnnotationsAndAttestsations then
86138
"--provenance=mode=max",
87139
# see "bashbrew remote arches docker/scout-sbom-indexer:1" (we need the SBOM scanner to be runnable on the host architecture)
88140
# bashbrew remote arches --json docker/scout-sbom-indexer:1 | jq '.arches | keys_unsorted' -c
89141
if .build.arch as $arch | ["amd64","arm32v5","arm32v7","arm64v8","i386","ppc64le","riscv64","s390x"] | index($arch) then
90142
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
91143
"--sbom=generator=\"$BASHBREW_BUILDKIT_SBOM_GENERATOR\""
144+
# TODO this should also be totally optional -- for example, Tianon doesn't want SBOMs on his personal images
92145
else empty end,
93-
(
94-
"--output " + (
95-
[
96-
"type=oci", # TODO find a better way to build/tag with a full list of tags but only actually *push* to one of them so we don't have to round-trip through containerd
97-
"dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily)
98-
(
99-
{
100-
# https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys
101-
"org.opencontainers.image.source": $buildUrl,
102-
"org.opencontainers.image.revision": .source.entry.GitCommit,
103-
104-
# TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption)
105-
"org.opencontainers.image.version": ( # value of the first image tag
106-
first(.source.allTags[] | select(contains(":")))
107-
| sub("^.*:"; "")
108-
# TODO maybe we should do the first, longest, non-latest tag instead of just the first tag?
109-
),
110-
"org.opencontainers.image.url": ( # URL to Docker Hub
111-
first(.source.allTags[] | select(contains(":")))
112-
| sub(":.*$"; "")
113-
| if contains("/") then
114-
"r/" + .
115-
else
116-
"_/" + .
117-
end
118-
| "https://hub.docker.com/" + .
119-
),
120-
# TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic)
121-
}
122-
| to_entries[] | select(.value != null) |
123-
"annotation." + .key + "=" + .value,
124-
"annotation-manifest-descriptor." + .key + "=" + .value
125-
),
126-
empty
127-
]
128-
| @csv
129-
| @sh
130-
)
131-
),
132146
empty
133-
end,
147+
else empty end,
148+
"--output " + (
149+
[
150+
if should_use_docker_buildx_driver then
151+
"type=docker"
152+
else
153+
"type=oci",
154+
"dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily)
155+
empty
156+
end,
157+
empty
158+
]
159+
| @csv
160+
| @sh
161+
),
162+
(
163+
if $supportsAnnotationsAndAttestsations then
164+
build_annotations($buildUrl)
165+
| to_entries
166+
# separate loops so that "image manifest" annotations are grouped separate from the index/descriptor annotations (easier to read)
167+
| (
168+
.[]
169+
| @sh "--annotation \(.key + "=" + .value)"
170+
),
171+
(
172+
.[]
173+
| @sh "--annotation \(
174+
"manifest-descriptor:" + .key + "="
175+
+ if .key == "org.opencontainers.image.created" then
176+
# the "current" time breaks reproducibility (for the purposes of build verification), so we put "now" in the image index but "SOURCE_DATE_EPOCH" in the image manifest (which is the thing we'd ideally like to have reproducible, eventually)
177+
(env.SOURCE_DATE_EPOCH // now) | tonumber | strftime("%FT%TZ")
178+
# (this assumes the actual build is going to happen shortly after generating the command)
179+
else .value end
180+
)",
181+
empty
182+
)
183+
else empty end
184+
),
134185
(
135186
.source.arches[].tags[],
136187
.source.arches[].archTags[],
@@ -148,8 +199,26 @@ def build_command:
148199
@sh "--file \(.source.entry.File)",
149200
($buildUrl | @sh),
150201
empty
151-
] | join(" ")
202+
] | join(" \\\n\t")
152203
),
204+
if should_use_docker_buildx_driver then empty else
205+
# munge the tarball into a suitable "oci layout" directory (ready for "crane push")
206+
"mkdir temp",
207+
"tar -xvf temp.tar -C temp",
208+
"rm temp.tar",
209+
# munge the index to what crane wants ("Error: layout contains 5 entries, consider --index")
210+
@sh "jq \("
211+
.manifests |= (
212+
del(.[].annotations)
213+
| unique
214+
| if length != 1 then
215+
error(\"unexpected number of manifests: \" + length)
216+
else . end
217+
)
218+
" | unindent_and_decomment_jq(4)) temp/index.json > temp/index.json.new",
219+
"mv temp/index.json.new temp/index.json",
220+
empty
221+
end,
153222
# possible improvements in buildkit/buildx that could help us:
154223
# - allowing OCI output directly to a directory instead of a tar (thus getting symmetry with the oci-layout:// inputs it can take)
155224
# - allowing tag as one thing and push as something else, potentially mutually exclusive
@@ -158,7 +227,8 @@ def build_command:
158227
empty
159228
] | join("\n")
160229
elif $builder == "classic" then
161-
[
230+
git_build_url as $buildUrl
231+
| [
162232
(
163233
[
164234
@sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)",
@@ -175,7 +245,7 @@ def build_command:
175245
($buildUrl | @sh),
176246
empty
177247
]
178-
| join(" ")
248+
| join(" \\\n\t")
179249
),
180250
empty
181251
] | join("\n")
@@ -199,14 +269,9 @@ def push_command:
199269
@sh "docker push \(.build.img)"
200270
elif $builder == "buildkit" then
201271
[
202-
# extract to a directory and "crane push" (easier to get correct than "ctr image import" + "ctr image push", especially with authentication)
203-
"mkdir temp",
204-
"tar -xvf temp.tar -C temp",
205-
# munge the index to what crane wants ("Error: layout contains 5 entries, consider --index")
206-
@sh "jq \(".manifests |= (del(.[].annotations) | unique)") temp/index.json > temp/index.json.new",
207-
"mv temp/index.json.new temp/index.json",
272+
# "crane push" is easier to get correct than "ctr image import" + "ctr image push", especially with authentication
208273
@sh "crane push temp \(.build.img)",
209-
"rm -rf temp temp.tar",
274+
"rm -rf temp",
210275
empty
211276
] | join("\n")
212277
elif $builder == "oci-import" then

0 commit comments

Comments
 (0)