hack: Rework+cleanup container build

Main motivation: I was looking at making more changes here

- Use an idiom I'd like to standardize more of copy context to `FROM scratch` image
  which is then mounted and consumed in other phases by mounting. This helps
  avoid polluting later containers with intermediate copied files.
- Change `build.sh` to handle being run from any directory
- Drop the `dev-rootfs` stuff as it's weird and awkward; instead we should
  encourage multi-step builds deriving from this image
- Don't make `bootc.tar.zst` only to immediately untar it; just use `COPY`
  from the build container
- Use heredocs to condense multiple `RUN` invocations to avoid pointless
  small layers

Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

.dockerignore

@@ -1,6 +1,13 @@
-.cosa
+# The big one - this can get HUGE and we don't want
+# to copy it around.
 target
-!target/dev-rootfs
+# This one can have large .qcow2 files written by coreos-assembler
+.cosa
 # These directories don't contribute to our container build
 docs/
+# TMT interprets these, not the container build
 plans/
+# We don't use this by default in containers
+.git
+# Avoid changes to this blowing out all layer caches
+hack/Containerfile

hack/Containerfile

@@ -5,33 +5,42 @@
 # You can also generate an image with cloud-init and other dependencies
 # with `--build-arg=tmt` which is intended for use particularly via
 # https://tmt.readthedocs.io/en/stable/
+
 ARG base=quay.io/centos-bootc/centos-bootc:stream9
-FROM $base as build
-# Keep this stuff before the `COPY . /build` below to ensure that the packages
-# are cached, i.e. we don't invalidate the package install stage by editing the source.
+
+FROM scratch as context
+# We only need this stuff in the initial context
+COPY hack /hack
 COPY contrib /contrib
-COPY hack/build.sh /build.sh
-RUN /build.sh && rm -v /build.sh
+
+FROM $base as build
+# This installs our package dependencies, and we want to cache it independently of the rest.
+# Basically we don't want changing a .rs file to blow out the cache of packages.
+RUN --mount=type=bind,from=context,target=/run/context /run/context/hack/build.sh
+# Now copy the rest of the source
 COPY . /build
 WORKDIR /build
-RUN mkdir -p /build/target/dev-rootfs  # This can hold arbitrary extra content
 # See https://www.reddit.com/r/rust/comments/126xeyx/exploring_the_problem_of_faster_cargo_docker/
 # We aren't using the full recommendations there, just the simple bits.
-RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome make test-bin-archive && mkdir -p /out && cp target/bootc.tar.zst /out
+RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome \
+    make && make install-all DESTDIR=/out
 
 FROM $base
 # We support e.g. adding cloud-init
 ARG variant=
-COPY hack/provision-derived.sh /tmp
-RUN /tmp/provision-derived.sh "$variant" && rm -f /tmp/*.sh
+# First, create a layer that is our new binaries.
+COPY --from=build /out/ /
+# And this layer has additional stuff for testing, such as nushell etc.
+RUN --mount=type=bind,from=context,target=/run/context <<EORUN
+set -xeuo pipefail
+/run/context/hack/provision-derived.sh "$variant"
+# Add some testing kargs into our dev builds
+cp -a /run/context/hack/test-kargs /usr/lib/bootc/kargs.d/
 # Also copy in some default install configs we use for testing
-COPY hack/install-test-configs/* /usr/lib/bootc/install/
-# And some test kargs
-COPY hack/test-kargs /usr/lib/bootc/kargs.d/
-# Inject our built code
-COPY --from=build /out/bootc.tar.zst /tmp
-RUN tar -C / --zstd -xvf /tmp/bootc.tar.zst && rm -vrf /tmp/*
-# Also copy over arbitrary bits from the target root
-COPY --from=build /build/target/dev-rootfs/ /
-# Test our own linting
-RUN bootc container lint --fatal-warnings
+cp -a /run/context/hack/install-test-configs/* /usr/lib/bootc/install/
+# Finally only in this containerfile, inject a file which signifies
+# this comes from this development image.
+touch /usr/lib/.bootc-dev-stamp
+# Finally, test our own linting
+bootc container lint --fatal-warnings
+EORUN

hack/build.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 set -xeu
 . /usr/lib/os-release
+dn=$(cd $(dirname $0) && pwd)
 case $ID in
   centos|rhel) dnf config-manager --set-enabled crb;;
   fedora) dnf -y install dnf-utils 'dnf5-command(builddep)';;
 esac
-dnf -y builddep ./contrib/packaging/bootc.spec
+dnf -y builddep ${dn}/../contrib/packaging/bootc.spec
 # Extra dependencies
 dnf -y install git-core