Skip to content

Commit 646b11c

Browse files
tianontianon
committed
Add initial Builder: oci-import support
This includes a workaround for the bug it helped me find (and fix), but I don't think it needs to wait for the fix because the workaround is really trivial / straightforward. The biggest benefit here (besides being one step closer to being able to switch to the new method for _everything_) is that this new way we build images will be significantly less susceptible to the downsides of the previous `oci-import` support (most notably, that the tarballs round-tripping through Docker recompresses them -- the containerd image storage in Docker itself should also help make that much better).
1 parent c3c2985 commit 646b11c

File tree

7 files changed

+249
-7
lines changed

7 files changed

+249
-7
lines changed

‎.test/builds.json

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2716,5 +2716,61 @@
27162716
}
27172717
}
27182718
}
2719+
},
2720+
"93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf": {
2721+
"buildId": "93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf",
2722+
"build": {
2723+
"img": "oisupport/staging-amd64:93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf",
2724+
"resolved": null,
2725+
"sourceId": "a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b",
2726+
"arch": "amd64",
2727+
"parents": {},
2728+
"resolvedParents": {}
2729+
},
2730+
"source": {
2731+
"sourceId": "a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b",
2732+
"reproducibleGitChecksum": "0ea17ff707666270e1fcb3edd76545906c68714d658ccde88c22d1606b2d7e4f",
2733+
"allTags": [
2734+
"ubuntu:22.04",
2735+
"ubuntu:jammy-20240111",
2736+
"ubuntu:jammy",
2737+
"ubuntu:latest"
2738+
],
2739+
"entry": {
2740+
"GitRepo": "https://git.launchpad.net/cloud-images/+oci/ubuntu-base",
2741+
"GitFetch": "refs/tags/dist-jammy-amd64-20240111-e6e3490a",
2742+
"GitCommit": "e6e3490ad3f524ccaa072edafe525f8ca8ac5490",
2743+
"Directory": "oci",
2744+
"File": "index.json",
2745+
"Builder": "oci-import",
2746+
"SOURCE_DATE_EPOCH": 1704931200
2747+
},
2748+
"arches": {
2749+
"amd64": {
2750+
"tags": [
2751+
"ubuntu:22.04",
2752+
"ubuntu:jammy-20240111",
2753+
"ubuntu:jammy",
2754+
"ubuntu:latest"
2755+
],
2756+
"archTags": [],
2757+
"froms": [
2758+
"scratch"
2759+
],
2760+
"lastStageFrom": "scratch",
2761+
"platformString": "linux/amd64",
2762+
"platform": {
2763+
"architecture": "amd64",
2764+
"os": "linux"
2765+
},
2766+
"parents": {
2767+
"scratch": {
2768+
"sourceId": null,
2769+
"pin": null
2770+
}
2771+
}
2772+
}
2773+
}
2774+
}
27192775
}
27202776
}

‎.test/example-commands.sh

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,54 @@ SOURCE_DATE_EPOCH=1700741054 \
7777
# <push>
7878
docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1fee7582174de82ae861613ae3072e'
7979
# </push>
80+
81+
# ubuntu:22.04 [amd64]
82+
# <pull>
83+
84+
# </pull>
85+
# <build>
86+
export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
87+
gitCache="$BASHBREW_CACHE/git"
88+
git init --bare "$gitCache"
89+
_git() { git -C "$gitCache" "$@"; }
90+
_git config gc.auto 0
91+
_commit() { _git rev-parse 'e6e3490ad3f524ccaa072edafe525f8ca8ac5490^{commit}'; }
92+
if ! _commit &> /dev/null; then _git fetch 'https://git.launchpad.net/cloud-images/+oci/ubuntu-base' 'e6e3490ad3f524ccaa072edafe525f8ca8ac5490:' || _git fetch 'refs/tags/dist-jammy-amd64-20240111-e6e3490a:'; fi
93+
_commit
94+
mkdir temp
95+
_git archive --format=tar 'e6e3490ad3f524ccaa072edafe525f8ca8ac5490:oci/' | tar -xvC temp
96+
jq -s '
97+
if length != 1 then
98+
error("unexpected '\''oci-layout'\'' document count: " + length)
99+
else .[0] end
100+
| if .imageLayoutVersion != "1.0.0" then
101+
error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
102+
else . end
103+
' temp/oci-layout > /dev/null
104+
jq -s '
105+
if length != 1 then
106+
error("unexpected '\''index.json'\'' document count: " + length)
107+
else .[0] end
108+
| if .schemaVersion != 2 then
109+
error("unsupported schemaVersion: " + .schemaVersion)
110+
else . end
111+
| if .manifests | length != 1 then
112+
error("expected only one manifests entry, not " + (.manifests | length))
113+
else . end
114+
| .manifests[0] |= (
115+
if .mediaType != "application/vnd.oci.image.manifest.v1+json" then
116+
error("unsupported descriptor mediaType: " + .mediaType)
117+
else . end
118+
| if .size < 0 then
119+
error("invalid descriptor size: " + .size)
120+
else . end
121+
| del(.annotations, .urls)
122+
| .annotations = {"org.opencontainers.image.source":"https://git.launchpad.net/cloud-images/+oci/ubuntu-base","org.opencontainers.image.revision":"e6e3490ad3f524ccaa072edafe525f8ca8ac5490","org.opencontainers.image.created":"2024-01-11T00:00:00Z","org.opencontainers.image.version":"22.04","org.opencontainers.image.url":"https://hub.docker.com/_/ubuntu","org.opencontainers.image.base.name":"scratch"}
123+
)
124+
' temp/index.json > temp/index.json.new
125+
mv temp/index.json.new temp/index.json
126+
# </build>
127+
# <push>
128+
crane push --index temp 'oisupport/staging-amd64:93476ae64659d71f4ee7fac781d6d1890df8926682e2fa6bd647a246b33ad9bf'
129+
rm -rf temp
130+
# </push>

‎.test/library/ubuntu

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# https://github.com/docker-library/official-images/blob/fe9c059402181390eac083cbdd7229b5d123236e/library/ubuntu but intentionally slimmed down (just "latest" on one architecture, no email addresses)
2+
3+
Maintainers: Tomáš Virtus (@woky), Cristóvão Cordeiro (@cjdcordeiro)
4+
GitRepo: https://git.launchpad.net/cloud-images/+oci/ubuntu-base
5+
GitCommit: fa42be9027eccb928a1f0f43d95ffd9a45d36737
6+
Builder: oci-import
7+
File: index.json
8+
9+
# 20240111 (jammy)
10+
Tags: 22.04, jammy-20240111, jammy, latest
11+
Architectures: amd64
12+
Directory: oci
13+
# https://git.launchpad.net/cloud-images/+oci/ubuntu-base/tree/?h=dist-jammy-amd64-20240111-e6e3490a
14+
amd64-GitFetch: refs/tags/dist-jammy-amd64-20240111-e6e3490a
15+
amd64-GitCommit: e6e3490ad3f524ccaa072edafe525f8ca8ac5490

‎.test/sources.json

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -757,5 +757,50 @@
757757
}
758758
}
759759
}
760+
},
761+
"a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b": {
762+
"sourceId": "a7e4b56dd1e5dde4c9d988092cb8639987559f13c5c92210664a4ffb880f222b",
763+
"reproducibleGitChecksum": "0ea17ff707666270e1fcb3edd76545906c68714d658ccde88c22d1606b2d7e4f",
764+
"allTags": [
765+
"ubuntu:22.04",
766+
"ubuntu:jammy-20240111",
767+
"ubuntu:jammy",
768+
"ubuntu:latest"
769+
],
770+
"entry": {
771+
"GitRepo": "https://git.launchpad.net/cloud-images/+oci/ubuntu-base",
772+
"GitFetch": "refs/tags/dist-jammy-amd64-20240111-e6e3490a",
773+
"GitCommit": "e6e3490ad3f524ccaa072edafe525f8ca8ac5490",
774+
"Directory": "oci",
775+
"File": "index.json",
776+
"Builder": "oci-import",
777+
"SOURCE_DATE_EPOCH": 1704931200
778+
},
779+
"arches": {
780+
"amd64": {
781+
"tags": [
782+
"ubuntu:22.04",
783+
"ubuntu:jammy-20240111",
784+
"ubuntu:jammy",
785+
"ubuntu:latest"
786+
],
787+
"archTags": [],
788+
"froms": [
789+
"scratch"
790+
],
791+
"lastStageFrom": "scratch",
792+
"platformString": "linux/amd64",
793+
"platform": {
794+
"architecture": "amd64",
795+
"os": "linux"
796+
},
797+
"parents": {
798+
"scratch": {
799+
"sourceId": null,
800+
"pin": null
801+
}
802+
}
803+
}
804+
}
760805
}
761806
}

‎.test/test.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ dir="$(dirname "$BASH_SOURCE")"
88
dir="$(readlink -ve "$dir")"
99
export BASHBREW_LIBRARY="$dir/library"
1010

11-
set -- docker:cli docker:dind docker:windowsservercore notary # a little bit of Windows, a little bit of Linux, a little bit of multi-stage
11+
set -- docker:cli docker:dind docker:windowsservercore notary ubuntu:latest # a little bit of Windows, a little bit of Linux, a little bit of multi-stage, a little bit of oci-import
1212
# (see "library/" and ".external-pins/" for where these come from / are hard-coded for consistent testing purposes)
1313
# NOTE: we are explicitly *not* pinning "golang:1.19-alpine3.16" so that this also tests unpinned parent behavior (that image is deprecated so should stay unchanging)
1414

‎meta.jq

Lines changed: 75 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -265,10 +265,76 @@ def build_command:
265265
] | join("\n")
266266
elif $builder == "oci-import" then
267267
[
268-
"git init temp", # TODO figure out a good, safe place to temporary "git init"??
269-
@sh "git -C temp fetch \(.source.entry.GitRepo) \(.source.entry.GitCommit): || git -C temp fetch \(.source.entry.GitRepo) \(.source.entry.GitFetch):",
270-
@sh "git -C temp checkout -q \(.source.entry.GitCommit)",
271-
# TODO something clever, especially to deal with "index.json" vs not-"index.json" (possibly using "jq" to either synthesize/normalize to what we actually need it to be for "crane push temp/dir \(.build.img)")
268+
# initialize "~/.cache/bashbrew/git"
269+
#"gitCache=\"$(bashbrew cat --format '{{ gitCache }}' <(echo 'Maintainers: empty hack (@example)'))\"",
270+
# https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L52-L80
271+
"export BASHBREW_CACHE=\"${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}\"",
272+
"gitCache=\"$BASHBREW_CACHE/git\"",
273+
"git init --bare \"$gitCache\"",
274+
"_git() { git -C \"$gitCache\" \"$@\"; }",
275+
"_git config gc.auto 0",
276+
# "bashbrew fetch" but in Bash (because we have bashbrew, but not the library file -- we could synthesize a library file instead, but six of one half a dozen of another)
277+
@sh "_commit() { _git rev-parse \(.source.entry.GitCommit + "^{commit}"); }",
278+
@sh "if ! _commit &> /dev/null; then _git fetch \(.source.entry.GitRepo) \(.source.entry.GitCommit + ":") || _git fetch \(.source.entry.GitFetch + ":"); fi",
279+
"_commit",
280+
281+
# TODO figure out a good, safe place to store our temporary build/push directory (maybe this is fine? we do it for buildx build too)
282+
"mkdir temp",
283+
# https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L140-L147 (TODO "bashbrew context" ?)
284+
@sh "_git archive --format=tar \(.source.entry.GitCommit + ":" + (.source.entry.Directory | if . == "." then "" else . + "/" end)) | tar -xvC temp",
285+
286+
# validate oci-layout file (https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L104-L112)
287+
@sh "jq -s \("
288+
if length != 1 then
289+
error(\"unexpected 'oci-layout' document count: \" + length)
290+
else .[0] end
291+
| if .imageLayoutVersion != \"1.0.0\" then
292+
error(\"unsupported imageLayoutVersion: \" + .imageLayoutVersion)
293+
else . end
294+
" | unindent_and_decomment_jq(3)) temp/oci-layout > /dev/null",
295+
296+
# https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L116
297+
if .source.entry.File != "index.json" then
298+
@sh "jq -s \("{ schemaVersion: 2, manifests: . }") \("./" + .source.entry.File) > temp/index.json"
299+
else empty end,
300+
301+
@sh "jq -s \("
302+
if length != 1 then
303+
error(\"unexpected 'index.json' document count: \" + length)
304+
else .[0] end
305+
306+
# https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L117-L127
307+
| if .schemaVersion != 2 then
308+
error(\"unsupported schemaVersion: \" + .schemaVersion)
309+
else . end
310+
# TODO check .mediaType ? (technically optional, but does not have to be *and* shouldn't be); https://github.com/moby/buildkit/issues/4595
311+
| if .manifests | length != 1 then
312+
error(\"expected only one manifests entry, not \" + (.manifests | length))
313+
else . end
314+
315+
| .manifests[0] |= (
316+
# https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L135-L144
317+
if .mediaType != \"application/vnd.oci.image.manifest.v1+json\" then
318+
error(\"unsupported descriptor mediaType: \" + .mediaType)
319+
else . end
320+
# TODO validate .digest somehow (`crane validate`? see below) - would also be good to validate all descriptors recursively (not sure if `crane push` does that)
321+
| if .size < 0 then
322+
error(\"invalid descriptor size: \" + .size)
323+
else . end
324+
325+
# purge maintainer-provided URLs / annotations (https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L146-L147)
326+
| del(.annotations, .urls)
327+
328+
# inject our annotations
329+
| .annotations = \(build_annotations(.source.entry.GitRepo) | @json)
330+
)
331+
" | unindent_and_decomment_jq(3)) temp/index.json > temp/index.json.new",
332+
"mv temp/index.json.new temp/index.json",
333+
334+
# TODO consider / check what "crane validate" does and if it would be appropriate here
335+
336+
# TODO generate SBOM? ... somehow
337+
272338
empty
273339
] | join("\n")
274340
else
@@ -289,7 +355,11 @@ def push_command:
289355
empty
290356
] | join("\n")
291357
elif $builder == "oci-import" then
292-
"TODO"
358+
[
359+
@sh "crane push --index temp \(.build.img)",
360+
"rm -rf temp",
361+
empty
362+
] | join("\n")
293363
else
294364
error("unknown/unimplemented Builder: \($builder)")
295365
end

‎sources.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,12 @@ bashbrew cat --build-order --format '
6565
"tags": {{ $.Tags namespace false . | json }},
6666
"archTags": {{ if $archNs -}} {{ $.Tags $archNs false . | json }} {{- else -}} [] {{- end }},
6767
"froms": {{ $.ArchDockerFroms $a . | json }},
68-
"lastStageFrom": {{ $.ArchLastStageFrom $a . | json }},
68+
"lastStageFrom": {{ if eq $builder "oci-import" -}}
69+
{{- /* TODO remove this special case: https://github.com/docker-library/bashbrew/pull/92 */ -}}
70+
"scratch"
71+
{{- else -}}
72+
{{ $.ArchLastStageFrom $a . | json }}
73+
{{- end }},
6974
"platformString": {{ (ociPlatform $a).String | json }},
7075
"platform": {{ ociPlatform $a | json }},
7176
"parents": { }

0 commit comments

Comments
 (0)