Skip to content

Commit a80a2c8

Browse files
yosifkityosifkit
authored
Merge pull request #17 from infosiftr/meta-refactor
Refactor `meta.jq` to prepare for signing and `oci-import` implementation
2 parents 7d4bdda + af7b858 commit a80a2c8

File tree

3 files changed

+180
-67
lines changed

3 files changed

+180
-67
lines changed

‎.test/example-commands.sh

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,49 @@
33

44
# </pull>
55
# <build>
6-
SOURCE_DATE_EPOCH=1700741054 docker buildx build --progress=plain --provenance=mode=max --sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" --output '"type=oci","dest=temp.tar","annotation.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation-manifest-descriptor.org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli","annotation.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation-manifest-descriptor.org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74","annotation.org.opencontainers.image.version=24.0.7-cli","annotation-manifest-descriptor.org.opencontainers.image.version=24.0.7-cli","annotation.org.opencontainers.image.url=https://hub.docker.com/_/docker","annotation-manifest-descriptor.org.opencontainers.image.url=https://hub.docker.com/_/docker"' --tag 'docker:24.0.7-cli' --tag 'docker:24.0-cli' --tag 'docker:24-cli' --tag 'docker:cli' --tag 'docker:24.0.7-cli-alpine3.18' --tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' --platform 'linux/amd64' --build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' --build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli'
7-
# </build>
8-
# <push>
6+
SOURCE_DATE_EPOCH=1700741054 \
7+
docker buildx build --progress=plain \
8+
--provenance=mode=max \
9+
--sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \
10+
--output '"type=oci","dest=temp.tar"' \
11+
--annotation 'org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \
12+
--annotation 'org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \
13+
--annotation 'org.opencontainers.image.created=2023-11-23T12:04:14Z' \
14+
--annotation 'org.opencontainers.image.version=24.0.7-cli' \
15+
--annotation 'org.opencontainers.image.url=https://hub.docker.com/_/docker' \
16+
--annotation 'manifest-descriptor:org.opencontainers.image.source=https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli' \
17+
--annotation 'manifest-descriptor:org.opencontainers.image.revision=6d541d27b5dd12639e5a33a675ebca04d3837d74' \
18+
--annotation 'manifest-descriptor:org.opencontainers.image.created=1970-01-01T00:00:00Z' \
19+
--annotation 'manifest-descriptor:org.opencontainers.image.version=24.0.7-cli' \
20+
--annotation 'manifest-descriptor:org.opencontainers.image.url=https://hub.docker.com/_/docker' \
21+
--tag 'docker:24.0.7-cli' \
22+
--tag 'docker:24.0-cli' \
23+
--tag 'docker:24-cli' \
24+
--tag 'docker:cli' \
25+
--tag 'docker:24.0.7-cli-alpine3.18' \
26+
--tag 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43' \
27+
--platform 'linux/amd64' \
28+
--build-context 'alpine:3.18=docker-image://alpine:3.18@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389' \
29+
--build-arg BUILDKIT_SYNTAX="$BASHBREW_BUILDKIT_SYNTAX" \
30+
--file 'Dockerfile' \
31+
'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/cli'
932
mkdir temp
1033
tar -xvf temp.tar -C temp
11-
jq '.manifests |= (del(.[].annotations) | unique)' temp/index.json > temp/index.json.new
34+
rm temp.tar
35+
jq '
36+
.manifests |= (
37+
del(.[].annotations)
38+
| unique
39+
| if length != 1 then
40+
error("unexpected number of manifests: " + length)
41+
else . end
42+
)
43+
' temp/index.json > temp/index.json.new
1244
mv temp/index.json.new temp/index.json
45+
# </build>
46+
# <push>
1347
crane push temp 'oisupport/staging-amd64:4b199ac326c74b3058a147e14f553af9e8e1659abc29bd3e82c9c9807b66ee43'
14-
rm -rf temp temp.tar
48+
rm -rf temp
1549
# </push>
1650

1751
# docker:24.0.7-windowsservercore-ltsc2022 [windows-amd64]
@@ -20,7 +54,21 @@ docker pull 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fc
2054
docker tag 'mcr.microsoft.com/windows/servercore:ltsc2022@sha256:d4ab2dd7d3d0fce6edc5df459565a4c96bbb1d0148065b215ab5ddcab1e42eb4' 'mcr.microsoft.com/windows/servercore:ltsc2022'
2155
# </pull>
2256
# <build>
23-
SOURCE_DATE_EPOCH=1700741054 DOCKER_BUILDKIT=0 docker build --tag 'docker:24.0.7-windowsservercore-ltsc2022' --tag 'docker:24.0-windowsservercore-ltsc2022' --tag 'docker:24-windowsservercore-ltsc2022' --tag 'docker:windowsservercore-ltsc2022' --tag 'docker:24.0.7-windowsservercore' --tag 'docker:24.0-windowsservercore' --tag 'docker:24-windowsservercore' --tag 'docker:windowsservercore' --tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' --platform 'windows/amd64' --file 'Dockerfile' 'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022'
57+
SOURCE_DATE_EPOCH=1700741054 \
58+
DOCKER_BUILDKIT=0 \
59+
docker build \
60+
--tag 'docker:24.0.7-windowsservercore-ltsc2022' \
61+
--tag 'docker:24.0-windowsservercore-ltsc2022' \
62+
--tag 'docker:24-windowsservercore-ltsc2022' \
63+
--tag 'docker:windowsservercore-ltsc2022' \
64+
--tag 'docker:24.0.7-windowsservercore' \
65+
--tag 'docker:24.0-windowsservercore' \
66+
--tag 'docker:24-windowsservercore' \
67+
--tag 'docker:windowsservercore' \
68+
--tag 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e' \
69+
--platform 'windows/amd64' \
70+
--file 'Dockerfile' \
71+
'https://github.com/docker-library/docker.git#6d541d27b5dd12639e5a33a675ebca04d3837d74:24/windows/windowsservercore-ltsc2022'
2472
# </build>
2573
# <push>
2674
docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e'

‎.test/test.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ time "$dir/../sources.sh" "$@" > "$dir/sources.json"
1919
time "$dir/../builds.sh" --cache "$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json"
2020

2121
# generate an "example commands" file so that changes to generated commands are easier to review
22-
jq -r -L "$dir/.." '
22+
SOURCE_DATE_EPOCH=0 jq -r -L "$dir/.." '
2323
include "meta";
2424
[
2525
first(.[] | select(normalized_builder == "buildkit")),

‎meta.jq

Lines changed: 125 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,26 @@ def normalized_builder:
1717
end
1818
else . end
1919
;
20+
def docker_uses_containerd_storage:
21+
# TODO somehow detect docker-with-containerd-storage
22+
false
23+
;
2024
# input: "build" object (with "buildId" top level key)
2125
# output: boolean
2226
def should_use_docker_buildx_driver:
2327
normalized_builder == "buildkit"
2428
and (
25-
.build.arch as $arch
26-
# bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c
27-
| ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"]
28-
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
29-
| index($arch)
30-
| not
29+
docker_uses_containerd_storage
30+
or (
31+
.build.arch as $arch
32+
# bashbrew remote arches --json tianon/buildkit:0.12 | jq '.arches | keys_unsorted' -c
33+
| ["amd64","arm32v5","arm32v6","arm32v7","arm64v8","i386","mips64le","ppc64le","riscv64","s390x"]
34+
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
35+
| index($arch)
36+
| not
37+
# TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit"; need *all* the arches we care about/support for consistent support)
38+
)
3139
)
32-
# TODO "failed to read dockerfile: failed to load cache key: subdir not supported yet" asdflkjalksdjfklasdjfklajsdklfjasdklgfnlkasdfgbhnkljasdhgouiahsdoifjnask,.dfgnklasdbngoikasdhfoiasjdklfjasdlkfjalksdjfkladshjflikashdbgiohasdfgiohnaskldfjhnlkasdhfnklasdhglkahsdlfkjasdlkfjadsklfjsdl (hence "tianon/buildkit" instead of "moby/buildkit")
3340
;
3441
# input: "build" object (with "buildId" top level key)
3542
# output: string "pull command" ("docker pull ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail", might be empty
@@ -71,66 +78,110 @@ def git_build_url:
7178
) + "#" + .GitCommit + ":" + .Directory
7279
;
7380
# input: "build" object (with "buildId" top level key)
81+
# output: map of annotations to set
82+
def build_annotations($buildUrl):
83+
{
84+
# https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys
85+
"org.opencontainers.image.source": $buildUrl,
86+
"org.opencontainers.image.revision": .source.entry.GitCommit,
87+
"org.opencontainers.image.created": (.source.entry.SOURCE_DATE_EPOCH | strftime("%FT%TZ")), # see notes below about image index vs image manifest
88+
89+
# TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption)
90+
"org.opencontainers.image.version": ( # value of the first image tag
91+
first(.source.allTags[] | select(contains(":")))
92+
| sub("^.*:"; "")
93+
# TODO maybe we should do the first, longest, non-latest tag instead of just the first tag?
94+
),
95+
"org.opencontainers.image.url": ( # URL to Docker Hub
96+
first(.source.allTags[] | select(contains(":")))
97+
| sub(":.*$"; "")
98+
| if contains("/") then
99+
"r/" + .
100+
else
101+
"_/" + .
102+
end
103+
| "https://hub.docker.com/" + .
104+
),
105+
106+
# TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic)
107+
}
108+
| with_entries(select(.value)) # strip off anything missing a value (possibly "source", "url", "version", etc)
109+
;
110+
def build_annotations:
111+
build_annotations(git_build_url)
112+
;
113+
# input: multi-line string with indentation and comments
114+
# output: multi-line string with less indentation and no comments
115+
def unindent_and_decomment_jq($indents):
116+
# trim out comment lines and unnecessary indentation
117+
gsub("(?m)^(\t+#[^\n]*\n?|\t{\($indents)}(?<extra>.*)$)"; "\(.extra // "")")
118+
# trim out empty lines
119+
| gsub("\n\n+"; "\n")
120+
;
121+
# input: "build" object (with "buildId" top level key)
74122
# output: string "build command" ("docker buildx build ..."), may be multiple lines, expects to run in Bash with "set -Eeuo pipefail"
75123
def build_command:
76124
normalized_builder as $builder
77-
| git_build_url as $buildUrl
78125
| if $builder == "buildkit" then
79-
[
126+
git_build_url as $buildUrl
127+
| (
128+
(should_use_docker_buildx_driver | not)
129+
or docker_uses_containerd_storage
130+
) as $supportsAnnotationsAndAttestsations
131+
| [
80132
(
81133
[
82134
@sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)",
83135
# TODO EXPERIMENTAL_BUILDKIT_SOURCE_POLICY=<(jq ...)
84136
"docker buildx build --progress=plain",
85-
if should_use_docker_buildx_driver then "--load" else # TODO if we get containerd integration and thus use "--load" unconditionally again, we should update this to still set annotations! (and still gate SBOMs on appropriate scanner-supported architectures)
137+
if $supportsAnnotationsAndAttestsations then
86138
"--provenance=mode=max",
87139
# see "bashbrew remote arches docker/scout-sbom-indexer:1" (we need the SBOM scanner to be runnable on the host architecture)
88140
# bashbrew remote arches --json docker/scout-sbom-indexer:1 | jq '.arches | keys_unsorted' -c
89141
if .build.arch as $arch | ["amd64","arm32v5","arm32v7","arm64v8","i386","ppc64le","riscv64","s390x"] | index($arch) then
90142
# TODO this needs to be based on the *host* architecture, not the *target* architecture (amd64 vs i386)
91143
"--sbom=generator=\"$BASHBREW_BUILDKIT_SBOM_GENERATOR\""
144+
# TODO this should also be totally optional -- for example, Tianon doesn't want SBOMs on his personal images
92145
else empty end,
93-
(
94-
"--output " + (
95-
[
96-
"type=oci", # TODO find a better way to build/tag with a full list of tags but only actually *push* to one of them so we don't have to round-trip through containerd
97-
"dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily)
98-
(
99-
{
100-
# https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/annotations.md#pre-defined-annotation-keys
101-
"org.opencontainers.image.source": $buildUrl,
102-
"org.opencontainers.image.revision": .source.entry.GitCommit,
103-
104-
# TODO come up with less assuming values here? (Docker Hub assumption, tag ordering assumption)
105-
"org.opencontainers.image.version": ( # value of the first image tag
106-
first(.source.allTags[] | select(contains(":")))
107-
| sub("^.*:"; "")
108-
# TODO maybe we should do the first, longest, non-latest tag instead of just the first tag?
109-
),
110-
"org.opencontainers.image.url": ( # URL to Docker Hub
111-
first(.source.allTags[] | select(contains(":")))
112-
| sub(":.*$"; "")
113-
| if contains("/") then
114-
"r/" + .
115-
else
116-
"_/" + .
117-
end
118-
| "https://hub.docker.com/" + .
119-
),
120-
# TODO org.opencontainers.image.vendor ? (feels leaky to put "Docker Official Images" here when this is all otherwise mostly generic)
121-
}
122-
| to_entries[] | select(.value != null) |
123-
"annotation." + .key + "=" + .value,
124-
"annotation-manifest-descriptor." + .key + "=" + .value
125-
),
126-
empty
127-
]
128-
| @csv
129-
| @sh
130-
)
131-
),
132146
empty
133-
end,
147+
else empty end,
148+
"--output " + (
149+
[
150+
if should_use_docker_buildx_driver then
151+
"type=docker"
152+
else
153+
"type=oci",
154+
"dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily)
155+
empty
156+
end,
157+
empty
158+
]
159+
| @csv
160+
| @sh
161+
),
162+
(
163+
if $supportsAnnotationsAndAttestsations then
164+
build_annotations($buildUrl)
165+
| to_entries
166+
# separate loops so that "image manifest" annotations are grouped separate from the index/descriptor annotations (easier to read)
167+
| (
168+
.[]
169+
| @sh "--annotation \(.key + "=" + .value)"
170+
),
171+
(
172+
.[]
173+
| @sh "--annotation \(
174+
"manifest-descriptor:" + .key + "="
175+
+ if .key == "org.opencontainers.image.created" then
176+
# the "current" time breaks reproducibility (for the purposes of build verification), so we put "now" in the image index but "SOURCE_DATE_EPOCH" in the image manifest (which is the thing we'd ideally like to have reproducible, eventually)
177+
(env.SOURCE_DATE_EPOCH // now) | tonumber | strftime("%FT%TZ")
178+
# (this assumes the actual build is going to happen shortly after generating the command)
179+
else .value end
180+
)",
181+
empty
182+
)
183+
else empty end
184+
),
134185
(
135186
.source.arches[].tags[],
136187
.source.arches[].archTags[],
@@ -148,8 +199,26 @@ def build_command:
148199
@sh "--file \(.source.entry.File)",
149200
($buildUrl | @sh),
150201
empty
151-
] | join(" ")
202+
] | join(" \\\n\t")
152203
),
204+
if should_use_docker_buildx_driver then empty else
205+
# munge the tarball into a suitable "oci layout" directory (ready for "crane push")
206+
"mkdir temp",
207+
"tar -xvf temp.tar -C temp",
208+
"rm temp.tar",
209+
# munge the index to what crane wants ("Error: layout contains 5 entries, consider --index")
210+
@sh "jq \("
211+
.manifests |= (
212+
del(.[].annotations)
213+
| unique
214+
| if length != 1 then
215+
error(\"unexpected number of manifests: \" + length)
216+
else . end
217+
)
218+
" | unindent_and_decomment_jq(4)) temp/index.json > temp/index.json.new",
219+
"mv temp/index.json.new temp/index.json",
220+
empty
221+
end,
153222
# possible improvements in buildkit/buildx that could help us:
154223
# - allowing OCI output directly to a directory instead of a tar (thus getting symmetry with the oci-layout:// inputs it can take)
155224
# - allowing tag as one thing and push as something else, potentially mutually exclusive
@@ -158,7 +227,8 @@ def build_command:
158227
empty
159228
] | join("\n")
160229
elif $builder == "classic" then
161-
[
230+
git_build_url as $buildUrl
231+
| [
162232
(
163233
[
164234
@sh "SOURCE_DATE_EPOCH=\(.source.entry.SOURCE_DATE_EPOCH)",
@@ -175,7 +245,7 @@ def build_command:
175245
($buildUrl | @sh),
176246
empty
177247
]
178-
| join(" ")
248+
| join(" \\\n\t")
179249
),
180250
empty
181251
] | join("\n")
@@ -199,14 +269,9 @@ def push_command:
199269
@sh "docker push \(.build.img)"
200270
elif $builder == "buildkit" then
201271
[
202-
# extract to a directory and "crane push" (easier to get correct than "ctr image import" + "ctr image push", especially with authentication)
203-
"mkdir temp",
204-
"tar -xvf temp.tar -C temp",
205-
# munge the index to what crane wants ("Error: layout contains 5 entries, consider --index")
206-
@sh "jq \(".manifests |= (del(.[].annotations) | unique)") temp/index.json > temp/index.json.new",
207-
"mv temp/index.json.new temp/index.json",
272+
# "crane push" is easier to get correct than "ctr image import" + "ctr image push", especially with authentication
208273
@sh "crane push temp \(.build.img)",
209-
"rm -rf temp temp.tar",
274+
"rm -rf temp",
210275
empty
211276
] | join("\n")
212277
elif $builder == "oci-import" then

0 commit comments

Comments
 (0)