build: Introduce "min" version of CI builder (#30694)

Adds a "min" flavor of the CI builder image which is ~1/10th the size of
the full image. Because our Bazel build is fully hermetic, it already
pulls in the necessary toolchains and thus the image we use can be quite
small. The "min" image can be fetched in ~30 seconds, whereas the full
image takes ~2 minutes and 30 seconds. This should save about 4 minutes
end-to-end in CI because both the initial mkpipeline step and the Build
steps will each save 2 minutes.

Similarly most CI jobs just run other Docker images built by the initial
step, so they also can use this "min" image, which should save another 2
minutes.

### Motivation

Reduce CI time by ~4 minutes

### Checklist

- [ ] This PR has adequate test coverage / QA involvement has been duly
considered. ([trigger-ci for additional test/nightly
runs](https://trigger-ci.dev.materialize.com/))
- [ ] This PR has an associated up-to-date [design
doc](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/README.md),
is a design doc
([template](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/00000000_template.md)),
or is sufficiently small to not require a design.
  <!-- Reference the design in the description. -->
- [ ] If this PR evolves [an existing `$T ⇔ Proto$T`
mapping](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/command-and-response-binary-encoding.md)
(possibly in a backwards-incompatible way), then it is tagged with a
`T-proto` label.
- [ ] If this PR will require changes to cloud orchestration or tests,
there is a companion cloud PR to account for those changes that is
tagged with the release-blocker label
([example](https://github.com/MaterializeInc/cloud/pull/5021)).
<!-- Ask in #team-cloud on Slack if you need help preparing the cloud
PR. -->
- [ ] If this PR includes major [user-facing behavior
changes](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/guide-changes.md#what-changes-require-a-release-note),
I have pinged the relevant PM to schedule a changelog post.

Author: ParkMyCar

bin/ci-builder

@@ -24,7 +24,7 @@ cd "$(dirname "$0")/.."
 
 if [[ $# -lt 2 ]]
 then
-    echo "usage: $0 <command> <stable|nightly> [<args>...]
+    echo "usage: $0 <command> <stable|nightly|min> [<args>...]
 
 Manages the ci-builder Docker image, which contains the dependencies required
 to build, test, and deploy the code in this repository.
@@ -40,25 +40,33 @@ For details, consult ci/builder/README.md."
 fi
 
 cmd=$1 && shift
-channel=$1 && shift
+flavor=$1 && shift
 
 rust_date=
-case "$channel" in
-    stable) rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml) ;;
+case "$flavor" in
+    min)
+        docker_target=ci-builder-min
+        rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml)
+        ;;
+    stable)
+        docker_target=ci-builder-full
+        rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml)
+        ;;
     nightly)
+        docker_target=ci-builder-full
         rust_version=nightly
         rust_date=/$NIGHTLY_RUST_DATE
         ;;
     *)
-        printf "unknown rust channel %q\n" "$channel"
+        printf "unknown CI builder flavor %q\n" "$flavor"
         exit 1
         ;;
 esac
 
 arch_gcc=${MZ_DEV_CI_BUILDER_ARCH:-$(arch_gcc)}
 arch_go=$(arch_go "$arch_gcc")
 
-cid_file=ci/builder/.${channel%%-*}.cidfile
+cid_file=ci/builder/.${flavor%%-*}.cidfile
 
 rust_components=rustc,cargo,rust-std-$arch_gcc-unknown-linux-gnu,llvm-tools-preview
 if [[ $rust_version = nightly ]]; then
@@ -86,6 +94,7 @@ build() {
         --build-arg "BAZEL_VERSION=$bazel_version" \
         --tag materialize/ci-builder:"$tag" \
         --tag materialize/ci-builder:"$cache_tag" \
+        --target $docker_target \
         "$@" ci/builder
 }
 
@@ -111,6 +120,7 @@ files+="
 rust-version:$rust_version
 rust-date:$rust_date
 arch:$arch_gcc
+flavor:$flavor
 "
 tag=$(echo "$files" | python3 -c '
 import base64
@@ -121,7 +131,7 @@ input = sys.stdin.buffer.read()
 hash = base64.b32encode(hashlib.sha1(input).digest())
 print(hash.decode())
 ')
-cache_tag=cache-$rust_version-$arch_go
+cache_tag=cache-$flavor-$rust_version-$arch_go
 
 
 case "$cmd" in

ci/builder/Dockerfile

@@ -7,10 +7,112 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-# Build a cross-compiling toolchain that targets the oldest version of Linux
-# that we support.
+# Stage 1: Build a minimum CI Builder image that we can use for the initial
+# steps like `mkpipeline` and `Build`, as well as any tests that are self
+# contained and use other Docker images.
+FROM ubuntu:noble-20241015 AS ci-builder-min
 
+WORKDIR /workdir
+
+ARG ARCH_GCC
+ARG ARCH_GO
+
+# Environment variables that should be set for the entire build container.
+
+# Ensure any Rust binaries that crash print a backtrace.
+ENV RUST_BACKTRACE=1
+# Ensure that all python output is unbuffered, otherwise it is not
+# logged properly in Buildkite.
+ENV PYTHONUNBUFFERED=1
+# Set a environment variable that tools can check to see if they're in the
+# builder or not.
+ENV MZ_DEV_CI_BUILDER=1
+
+# Faster uncompression
+ARG XZ_OPT=-T0
+
+# Absolute minimum set of dependencies needed for a CI job.
+#
+# Please take care with what gets added here. The goal of this initial layer is to be as small as
+# possible since it's used for the `mkpipeline` and `Build` CI jobs, which block __all other__
+# jobs.
+RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    docker.io \
+    gdb \
+    git \
+    gnupg2 \
+    libxml2 \
+    python3 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies. These are necessary to run some of our base tooling.
+COPY requirements.txt /workdir/
+RUN curl -LsSf https://astral.sh/uv/0.4.25/install.sh | UV_INSTALL_DIR=/usr/local UV_UNMANAGED_INSTALL=1 sh \
+    && uv pip install --system --break-system-packages -r /workdir/requirements.txt && rm /workdir/requirements*.txt
+
+# Install extra tools not available in apt repositories.
+
+COPY rust.asc .
+RUN gpg --import rust.asc \
+    && rm rust.asc \
+    && echo "trusted-key 85AB96E6FA1BE5FE" >> ~/.gnupg/gpg.conf
+
+ARG BAZEL_VERSION
+ARG RUST_DATE
+ARG RUST_VERSION
+
+RUN \
+    # 1. autouseradd
+    #
+    # Ensures that the UID used when running the container has a proper entry in
+    # `/etc/passwd`, and writable home directory.
+    curl -fsSL https://github.com/benesch/autouseradd/releases/download/1.3.0/autouseradd-1.3.0-$ARCH_GO.tar.gz \
+    | tar xz -C / --strip-components 1 \
+    # 2. Bazel
+    #
+    # We primarily build Materialize via Bazel in CI, and Bazel pulls in its own dependencies.
+    && arch_bazel=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm64/" -e "s/amd64/x86_64/") bazel_version=$(echo "$BAZEL_VERSION") \
+    && curl -fsSL -o /usr/local/bin/bazel https://github.com/bazelbuild/bazel/releases/download/$bazel_version/bazel-$bazel_version-linux-$arch_bazel \
+    && if [ "$arch_bazel" = arm64 ]; then echo 'fac4b954e0501c2be8b9653a550b443eb85284e568d08b102977e2bf587b09d7 /usr/local/bin/bazel' | sha256sum --check; fi \
+    && if [ "$arch_bazel" = x86_64 ]; then echo '48ea0ff9d397a48add6369c261c5a4431fe6d5d5348cfb81411782fb80c388d3 /usr/local/bin/bazel' | sha256sum --check; fi \
+    && chmod +x /usr/local/bin/bazel \
+    # 3. Docker
+    #
+    # If you upgrade Docker (Compose) version here, also update it in misc/python/cli/mzcompose.py.
+    && mkdir -p /usr/local/lib/docker/cli-plugins \
+    && curl -fsSL https://github.com/docker/compose/releases/download/v2.15.1/docker-compose-linux-$ARCH_GCC > /usr/local/lib/docker/cli-plugins/docker-compose \
+    && chmod +x /usr/local/lib/docker/cli-plugins/docker-compose \
+    && curl -fsSL https://github.com/christian-korneck/docker-pushrm/releases/download/v1.9.0/docker-pushrm_linux_$ARCH_GO > /usr/local/lib/docker/cli-plugins/docker-pushrm \
+    && chmod +x /usr/local/lib/docker/cli-plugins/docker-pushrm \
+    # 4. Cargo
+    #
+    # Some parts of our stack use 'cargo' to read metadata, so we install just that. Importantly we
+    # do not install 'rustc' or any of the other tools, this keeps the Docker image small.
+    && mkdir rust \
+    && curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-$ARCH_GCC-unknown-linux-gnu.tar.gz > rust.tar.gz \
+    && curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-$ARCH_GCC-unknown-linux-gnu.tar.gz.asc > rust.asc \
+    && gpg --verify rust.asc rust.tar.gz \
+    && tar -xzf rust.tar.gz -C rust --strip-components=1 \
+    && rust/install.sh --components=cargo \
+    && rm -rf rust.asc rust.tar.gz rust
+
+# Make the image as small as possible.
+RUN find /workdir /root -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+
+# Remove Ubuntu user causing UID collisions.
+# https://bugs.launchpad.net/cloud-images/+bug/2005129
+RUN userdel -r ubuntu
+
+ENTRYPOINT ["autouseradd", "--user", "materialize"]
+
+# Stage 2. Build a cross-compiling toolchain that targets the oldest version of
+# Linux that we support.
+#
+# TODO(parkmycar): This shouldn't be necessary anymore with Bazel.
 FROM ubuntu:noble-20241015 as crosstool
+
 ARG ARCH_GCC
 ARG ARCH_GO
 
@@ -68,10 +170,10 @@ RUN DEFCONFIG=crosstool-$ARCH_GCC.defconfig ct-ng defconfig \
     && rm crosstool-$ARCH_GCC.defconfig \
     && ct-ng build
 
-# Import the cross-compiling toolchain into a fresh image, omitting the
-# dependencies that we needed to actually build the toolchain.
+# Stage 3: Build a full CI Builder image that imports the cross-compiling
+# toolchain and can be used for any CI job.
+FROM ci-builder-min as ci-builder-full
 
-FROM ubuntu:noble-20241015
 ARG ARCH_GCC
 ARG ARCH_GO
 
@@ -135,9 +237,7 @@ RUN gpg --dearmor < nodesource.asc > /etc/apt/keyrings/nodesource.gpg \
     && apt-get update \
     && apt-get install -y --no-install-recommends nodejs
 
-RUN curl -fsSL https://github.com/benesch/autouseradd/releases/download/1.3.0/autouseradd-1.3.0-$ARCH_GO.tar.gz \
-    | tar xz -C / --strip-components 1 \
-    && curl -fsSL https://github.com/koalaman/shellcheck/releases/download/v0.8.0/shellcheck-v0.8.0.linux.$ARCH_GCC.tar.xz > shellcheck.tar.xz \
+RUN curl -fsSL https://github.com/koalaman/shellcheck/releases/download/v0.8.0/shellcheck-v0.8.0.linux.$ARCH_GCC.tar.xz > shellcheck.tar.xz \
     && tar -xJf shellcheck.tar.xz -C /usr/local/bin --strip-components 1 shellcheck-v0.8.0/shellcheck \
     && rm shellcheck.tar.xz \
     && curl -fsSL https://github.com/bufbuild/buf/releases/download/v1.18.0/buf-Linux-$ARCH_GCC.tar.gz > buf.tar.gz \
@@ -148,12 +248,6 @@ RUN curl -fsSL https://github.com/benesch/autouseradd/releases/download/1.3.0/au
     && tar -xf kail.tar.gz -C /usr/local/bin kail \
     && rm kail.tar.gz \
     && chmod +x /usr/local/bin/kail \
-    && mkdir -p /usr/local/lib/docker/cli-plugins \
-    # If you upgrade Docker (Compose) version here, also update it in misc/python/cli/mzcompose.py \
-    && curl -fsSL https://github.com/docker/compose/releases/download/v2.15.1/docker-compose-linux-$ARCH_GCC > /usr/local/lib/docker/cli-plugins/docker-compose \
-    && chmod +x /usr/local/lib/docker/cli-plugins/docker-compose \
-    && curl -fsSL https://github.com/christian-korneck/docker-pushrm/releases/download/v1.9.0/docker-pushrm_linux_$ARCH_GO > /usr/local/lib/docker/cli-plugins/docker-pushrm \
-    && chmod +x /usr/local/lib/docker/cli-plugins/docker-pushrm \
     && curl -fsSL https://github.com/parca-dev/parca-debuginfo/releases/download/v0.11.0/parca-debuginfo_0.11.0_Linux_$(echo "$ARCH_GCC" | sed "s/aarch64/arm64/").tar.gz \
     | tar xz -C /usr/local/bin parca-debuginfo
 
@@ -212,8 +306,6 @@ RUN mkdir rust \
 RUN ln -s /usr/bin/lld /opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin/$ARCH_GCC-unknown-linux-gnu-ld.lld \
     && ln -s /usr/bin/lld /opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin/$ARCH_GCC-unknown-linux-gnu-lld
 
-RUN curl -LsSf https://astral.sh/uv/0.4.25/install.sh | UV_INSTALL_DIR=/usr/local UV_UNMANAGED_INSTALL=1 sh
-
 # Shims for sanitizers
 COPY sanshim/$ARCH_GCC /sanshim
 
@@ -223,13 +315,6 @@ COPY sanshim/$ARCH_GCC /sanshim
 COPY pyright-version.sh /workdir/
 RUN npx pyright@$(sh /workdir/pyright-version.sh) --help
 
-# Install Python dependencies. These are so quick to install and change
-# frequently enough that it makes sense to install them last.
-
-COPY requirements.txt /workdir/
-
-RUN uv pip install --system --break-system-packages -r /workdir/requirements.txt && rm /workdir/requirements*.txt
-
 # Install APT repo generator.
 
 RUN curl -fsSL https://github.com/deb-s3/deb-s3/releases/download/0.11.3/deb-s3-0.11.3.gem > deb-s3.gem \
@@ -258,21 +343,7 @@ RUN if [ $ARCH_GCC = x86_64 ]; then \
     && rm hugo.tar.gz; \
     fi
 
-# Install Bazel.
-#
-# TODO(parkmycar): Run Bazel in a Docker image that does not have access to clang/gcc or any other tools.
-
-ARG BAZEL_VERSION
-
-# Download the bazel binary from the official GitHub releases since the apt repositories do not
-# contain arm64 releases.
-RUN arch_bazel=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm64/" -e "s/amd64/x86_64/") bazel_version=$(echo "$BAZEL_VERSION") \
-    && curl -fsSL -o /usr/local/bin/bazel https://github.com/bazelbuild/bazel/releases/download/$bazel_version/bazel-$bazel_version-linux-$arch_bazel \
-    && if [ "$arch_bazel" = arm64 ]; then echo 'fac4b954e0501c2be8b9653a550b443eb85284e568d08b102977e2bf587b09d7 /usr/local/bin/bazel' | sha256sum --check; fi \
-    && if [ "$arch_bazel" = x86_64 ]; then echo '48ea0ff9d397a48add6369c261c5a4431fe6d5d5348cfb81411782fb80c388d3 /usr/local/bin/bazel' | sha256sum --check; fi \
-    && chmod +x /usr/local/bin/bazel
-
-# Install KinD, kubectl, helm, helm-docs & terraform
+# Install KinD, kubectl, helm & helm-docs
 
 RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.14.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \
     && chmod +x /usr/local/bin/kind \
@@ -340,28 +411,11 @@ ENV CARGO_TARGET_DIR=/mnt/build
 ENV CARGO_INCREMENTAL=0
 ENV HELM_PLUGINS=/usr/local/share/helm/plugins
 
-# Set a environment variable that tools can check to see if they're in the
-# builder or not.
-
-ENV MZ_DEV_CI_BUILDER=1
-
 # Set up for a persistent volume to hold Cargo metadata, so that crate metadata
 # does not need to be refetched on every compile.
-
 ENV CARGO_HOME=/cargo
 RUN mkdir /cargo && chmod 777 /cargo
 VOLUME /cargo
 
-# Ensure any Rust binaries that crash print a backtrace.
-ENV RUST_BACKTRACE=1
-
 # Make the image as small as possible.
 RUN find /workdir /root -mindepth 1 -maxdepth 1 -exec rm -rf {} +
-
-# remove Ubuntu user causing UID collisions
-# https://bugs.launchpad.net/cloud-images/+bug/2005129
-RUN userdel -r ubuntu
-
-# Ensure that all python output is unbuffered, otherwise it is not
-# logged properly in Buildkite
-ENV PYTHONUNBUFFERED=1

ci/mkpipeline.sh

@@ -23,15 +23,15 @@ pipeline=${1:-test}
 bootstrap_steps=
 
 for arch in x86_64 aarch64; do
-    for toolchain in stable nightly; do
-        if ! MZ_DEV_CI_BUILDER_ARCH=$arch bin/ci-builder exists $toolchain; then
+    for flavor in stable nightly min; do
+        if ! MZ_DEV_CI_BUILDER_ARCH=$arch bin/ci-builder exists $flavor; then
             queue=builder-linux-x86_64
             if [[ $arch = aarch64 ]]; then
                 queue=builder-linux-aarch64-mem
             fi
             bootstrap_steps+="
-  - label: bootstrap $toolchain $arch
-    command: bin/ci-builder push $toolchain
+  - label: bootstrap $flavor $arch
+    command: bin/ci-builder push $flavor
     agents:
       queue: $queue
 "
@@ -47,7 +47,7 @@ steps:
     env:
       CI_BAZEL_BUILD: 1
       CI_BAZEL_REMOTE_CACHE: "https://bazel-remote.dev.materialize.com"
-    command: bin/ci-builder run stable bin/pyactivate -m ci.mkpipeline $pipeline $@
+    command: bin/ci-builder run min bin/pyactivate -m ci.mkpipeline $pipeline $@
     priority: 200
     agents:
       queue: hetzner-aarch64-4cpu-8gb

ci/nightly/pipeline.template.yml

@@ -20,7 +20,7 @@ steps:
     steps:
       - id: build-x86_64
         label: ":bazel: Build x86_64"
-        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         artifact_paths: bazel-explain.log
@@ -31,7 +31,7 @@ steps:
 
       - id: build-aarch64
         label: ":bazel: Build aarch64"
-        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         artifact_paths: bazel-explain.log

ci/release-qualification/pipeline.template.yml

@@ -20,7 +20,7 @@ steps:
     steps:
       - id: build-aarch64
         label: ":bazel: Build aarch64"
-        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         artifact_paths: bazel-explain.log
@@ -39,7 +39,7 @@ steps:
         label: ":bazel: Build x86_64"
         env:
           CI_BAZEL_BUILD: "1"
-        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         artifact_paths: bazel-explain.log

ci/test/pipeline.template.yml

@@ -15,7 +15,6 @@
 dag: true
 
 env:
-  CI_BUILDER_SCCACHE: 1
   CI_BAZEL_BUILD: 1
   CI_BAZEL_REMOTE_CACHE: $BAZEL_REMOTE_CACHE
   # Note: In .cargo/config we set the default build jobs to -1 so on developer machines we keep
@@ -90,7 +89,7 @@ steps:
       - id: build-x86_64
         label: ":bazel: Build x86_64"
         env:
-        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         artifact_paths: bazel-explain.log
@@ -102,7 +101,7 @@ steps:
       - id: build-aarch64
         label: ":bazel: Build aarch64"
         env:
-        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         artifact_paths: bazel-explain.log