Streamline rootfs building

.buildkite/pipeline_docker_popular.py

@@ -15,10 +15,10 @@
 pipeline.build_group_per_arch(
     "rootfs-build",
     [
-        "sudo yum install -y systemd-container",
+        "sudo yum install -y squashfs-tools docker",
+        "sudo tools/devtool sh 'tools/test-popular-containers/build_rootfs.sh'",
         "cd tools/test-popular-containers",
-        "sudo ./build_rootfs.sh",
-        f'tar czf "{ROOTFS_TAR}" *.ext4 *.id_rsa',
+        f'tar czf "{ROOTFS_TAR}" *.squashfs *.id_rsa',
         f'buildkite-agent artifact upload "{ROOTFS_TAR}"',
     ],
     depends_on_build=False,

resources/chroot.sh

@@ -9,8 +9,6 @@ set -eu -o pipefail
 set -x
 PS4='+\t '
 
-cp -ruv $rootfs/* /
-
 packages="udev systemd-sysv openssh-server iproute2 curl socat python3-minimal iperf3 iputils-ping fio kmod tmux hwloc-nox vim-tiny trace-cmd linuxptp strace python3-boto3 pciutils"
 
 # msr-tools is only supported on x86-64.

resources/rebuild.sh

@@ -32,13 +32,6 @@ function install_dependencies {
     rm $archive
 }
 
-function prepare_docker {
-    nohup /usr/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 &
-
-    # Wait for Docker socket to be created
-    timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
-}
-
 function compile_and_install {
     local SRC=$1
     local BIN="${SRC%.*}"
@@ -56,45 +49,10 @@ function compile_and_install {
 }
 
 # Build a rootfs
-function build_rootfs {
-    local ROOTFS_NAME=$1
-    local flavour=${2}
-    local FROM_CTR=public.ecr.aws/ubuntu/ubuntu:$flavour
-    local rootfs="tmp_rootfs"
-    mkdir -pv "$rootfs"
-
-    # Launch Docker
+function build_ci_rootfs {
+    local IMAGE_NAME=$1
     prepare_docker
-
-    cp -rvf overlay/* $rootfs
-
-    # curl -O https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-amd64-root.tar.xz
-    #
-    # TBD use systemd-nspawn instead of Docker
-    #   sudo tar xaf ubuntu-22.04-minimal-cloudimg-amd64-root.tar.xz -C $rootfs
-    #   sudo systemd-nspawn --resolv-conf=bind-uplink -D $rootfs
-    docker run --env rootfs=$rootfs --privileged --rm -i -v "$PWD:/work" -w /work "$FROM_CTR" bash -s <<'EOF'
-
-./chroot.sh
-
-# Copy everything we need to the bind-mounted rootfs image file
-dirs="bin etc home lib lib64 root sbin usr"
-for d in $dirs; do tar c "/$d" | tar x -C $rootfs; done
-
-# Make mountpoints
-mkdir -pv $rootfs/{dev,proc,sys,run,tmp,var/lib/systemd}
-# So apt works
-mkdir -pv $rootfs/var/lib/dpkg/
-EOF
-
-    # TBD what abt /etc/hosts?
-    echo | tee $rootfs/etc/resolv.conf
-
-    rootfs_img="$OUTPUT_DIR/$ROOTFS_NAME.squashfs"
-    mv $rootfs/root/manifest $OUTPUT_DIR/$ROOTFS_NAME.manifest
-    mksquashfs $rootfs $rootfs_img -all-root -noappend -comp zstd
-    rm -rf $rootfs
-    rm -f nohup.out
+    build_rootfs "$IMAGE_NAME" "$OUTPUT_DIR" "$PWD/overlay" "chroot.sh"
 }
 
 
@@ -226,7 +184,7 @@ function prepare_and_build_rootfs {
         compile_and_install $BIN_DIR/$SRC
     done
 
-    build_rootfs ubuntu-24.04 noble
+    build_ci_rootfs ubuntu:24.04
     build_initramfs
 
     for SRC in ${SRCS[@]}; do

tools/functions

@@ -125,3 +125,69 @@ function validate_version {
         die "Invalid version number: $version. Version should not contain \`wip\` or \`dirty\`."
     fi
 }
+
+########################
+# Rootfs build helpers #
+########################
+
+# Wait for dockerd to come up in devctr
+prepare_docker() {
+    nohup /usr/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 &
+
+    # Wait for Docker socket to be created
+    timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
+    rm -f nohup.out
+}
+
+# Build a squashfs rootfs from a Docker image.
+# Runs <setup_script> inside the container to install packages and configure.
+# The setup script receives $rootfs as the overlay directory (pre-populated
+# with overlay files).
+# Usage: build_rootfs <image> <output_dir> <overlay_dir> <setup_script>
+build_rootfs() {
+    local IMAGE_NAME=$1
+    local OUTPUT_DIR=$2
+    local OVERLAY_DIR=$3
+    local SETUP_SCRIPT=$4
+    local ROOTFS_NAME="${IMAGE_NAME//:/-}"
+    local FROM_CTR="public.ecr.aws/docker/library/$IMAGE_NAME"
+    local rootfs="rootfs_${ROOTFS_NAME}_$$"
+
+    say "Building rootfs for $IMAGE_NAME"
+
+    mkdir -pv $rootfs
+
+    cp -rvf "$OVERLAY_DIR"/* "$rootfs"
+
+    # Run setup script inside the container image, then copy the
+    # resulting filesystem back to the bind-mounted rootfs directory.
+    docker run --env rootfs="$rootfs" --privileged --rm -i \
+        -v "$PWD:/work" -w "/work" "$FROM_CTR" sh -c '
+            # Make overlay files available inside the container
+            cp -ruv $rootfs/* /
+
+            # Run the setup script
+            if [ -e /bin/bash ]; then
+              bash /work/'"$SETUP_SCRIPT"'
+            else
+              sh /work/'"$SETUP_SCRIPT"'
+            fi
+
+            # Copy filesystem back to bind-mounted rootfs
+            dirs="bin etc home lib lib64 root sbin usr var"
+            for d in $dirs; do [ -d "/$d" ] && tar c "/$d" | tar x -C $rootfs; done
+            mkdir -pv $rootfs/dev $rootfs/proc $rootfs/sys $rootfs/run $rootfs/tmp 
+            '
+
+    # TBD what about /etc/hosts?
+    echo > $rootfs/etc/resolv.conf
+
+    mkdir -pv "$OUTPUT_DIR"
+
+    # If manifest file is present in rootfs, move it to output dir
+    [ -f "$rootfs/root/manifest" ] && mv "$rootfs/root/manifest" "$OUTPUT_DIR/$ROOTFS_NAME.manifest"
+
+    mksquashfs "$rootfs" "$OUTPUT_DIR/$ROOTFS_NAME.squashfs" \
+        -all-root -noappend -comp zstd
+    rm -rf "$rootfs"
+}

tools/test-popular-containers/build_rootfs.sh

@@ -8,78 +8,36 @@ set -x
 
 cd $(dirname $0)
 TOPDIR=$(git rev-parse --show-cdup)
+source "$TOPDIR/tools/functions"
 
-function make_rootfs {
-    local LABEL=$1
-    local rootfs=$LABEL
-    local IMG=$LABEL.ext4
-    mkdir $LABEL
-    ctr image pull public.ecr.aws/docker/library/$LABEL
-    ctr image mount --rw public.ecr.aws/docker/library/$LABEL $LABEL
-    MNT_SIZE=$(du -sb $LABEL |cut -f1)
-    SIZE=$(( $MNT_SIZE + 512 * 2**20 ))
+# Get the executing uid and gid for `chown` and `chgrp`
+USER_UID=$(stat -c '%u' "$TOPDIR")
+USER_GID=$(stat -c '%g' "$TOPDIR")
 
-    # Generate key for ssh access from host
-    if [ ! -s id_rsa ]; then
-        ssh-keygen -f id_rsa -N ""
-    fi
-    cp id_rsa $rootfs.id_rsa
-    chmod a+r $rootfs.id_rsa
 
-    truncate -s "$SIZE" "$IMG"
-    mkfs.ext4 -F "$IMG" -d $LABEL
-    ctr image unmount $LABEL
-    rmdir $LABEL
+OVERLAY_DIR="$TOPDIR/resources/overlay"
+SETUP_SCRIPT="setup-minimal.sh"
+OUTPUT_DIR=$PWD
 
-    mkdir mnt
-    mount $IMG mnt
-    install -d -m 0600 "mnt/root/.ssh/"
-    cp -v id_rsa.pub "mnt/root/.ssh/authorized_keys"
-    cp -rvf $TOPDIR/resources/overlay/* mnt
-    SYSINIT=mnt/etc/systemd/system/sysinit.target.wants
-    mkdir -pv $SYSINIT
-    ln -svf /etc/systemd/system/fcnet.service $SYSINIT/fcnet.service
-    mkdir mnt/etc/local.d
-    cp -v fcnet.start mnt/etc/local.d
-    umount -l mnt
-    rmdir mnt
+IMAGES=(amazonlinux:2023 alpine:latest ubuntu:22.04 ubuntu:24.04 ubuntu:25.04 ubuntu:latest)
 
-    # --timezone=off parameter is needed to prevent systemd-nspawn from
-    # bind-mounting /etc/timezone, which causes a file conflict in Ubuntu 24.04
-    systemd-nspawn --timezone=off --pipe -i $IMG /bin/sh <<EOF
-set -x
-. /etc/os-release
-case \$ID in
-ubuntu)
-    export DEBIAN_FRONTEND=noninteractive
-    apt update
-    apt install -y openssh-server iproute2 udev
-    ;;
-alpine)
-    apk add openssh openrc
-    rc-update add sshd
-    rc-update add local default
-    echo "ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100" >>/etc/inittab
-    ;;
-amzn)
-    dnf update
-    dnf install -y openssh-server iproute passwd systemd-udev
-    # re-do this
-    ln -svf /etc/systemd/system/fcnet.service /etc/systemd/system/sysinit.target.wants/fcnet.service
-    rm -fv /etc/systemd/system/getty.target.wants/getty@tty1.service
-    ;;
-esac
-passwd -d root
-EOF
-}
+# Generate SSH key for access from host
+if [ ! -s id_rsa ]; then
+  ssh-keygen -f id_rsa -N ""
+fi
+
+# install rootfs dependencies
+apt update
+apt install -y busybox-static cpio curl docker.io tree
+prepare_docker
+
+for img in "${IMAGES[@]}"; do
+  build_rootfs "$img" "$OUTPUT_DIR" "$OVERLAY_DIR" "$SETUP_SCRIPT"
+
+  rootfs_name="${img//:/-}"
+  cp id_rsa "$rootfs_name.id_rsa"
+  chmod a+r "$rootfs_name.id_rsa"
 
-# FIXME: The AL image build procedure is known for keeping changing the ext4 file
-# even after the systemd-nspawn command exits, for unknown reason, causing
-# the subsequent tar command to fail.  Placing it first as a mitigation gives it
-# more time to converge to a stable state.
-make_rootfs amazonlinux:2023
-make_rootfs alpine:latest
-make_rootfs ubuntu:22.04
-make_rootfs ubuntu:24.04
-make_rootfs ubuntu:25.04
-make_rootfs ubuntu:latest
+  chown "$USER_UID" "$rootfs_name.squashfs" "$rootfs_name.id_rsa"
+  chgrp "$USER_GID" "$rootfs_name.squashfs" "$rootfs_name.id_rsa"
+done

tools/test-popular-containers/setup-minimal.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Minimal rootfs setup: just enough to boot with SSH and networking.
+# Runs inside a Docker container via build_rootfs().
+
+set -eux
+
+. /etc/os-release
+# On Ubuntu, installing openssh-server automatically sets up required SSH keys for the server.
+# AL2023 and Alpine do not do this, so we should setup keys manually via `ssh-keygen`.
+# Alpine additionally requires /var/empty to be present for sshd to start properly.
+case $ID in
+ubuntu)
+  export DEBIAN_FRONTEND=noninteractive
+  apt update
+  apt install -y openssh-server iproute2 udev
+  ;;
+amzn)
+  dnf install -y openssh-server iproute systemd-udev passwd tar
+  ssh-keygen -A
+  ;;
+alpine)
+  apk add openssh openrc tar
+  mkdir -p /var/empty
+  ssh-keygen -A
+  rc-update add sshd
+  rc-update add local default
+  echo "ttyS0::respawn:/sbin/getty -L ttyS0 115200 vt100" >>/etc/inittab
+  ;;
+esac
+
+passwd -d root
+
+# Install SSH public key if available
+if [ -f /work/id_rsa.pub ]; then
+  install -d -m 0700 /root/.ssh
+  cp /work/id_rsa.pub /root/.ssh/authorized_keys
+  chmod 0600 /root/.ssh/authorized_keys
+fi
+
+if [ -d /etc/systemd/system ]; then
+  # Enable fcnet for systemd-based images
+  mkdir -pv /etc/systemd/system/sysinit.target.wants
+  ln -svf /etc/systemd/system/fcnet.service /etc/systemd/system/sysinit.target.wants/fcnet.service
+  cp -v fcnet.start /etc/local.d
+
+    # The serial getty service hooks up the login prompt to the kernel console
+    # at ttyS0 (where Firecracker connects its serial console). We'll set it up
+    # for autologin to avoid the login prompt.
+    for console in ttyS0; do
+        mkdir "/etc/systemd/system/serial-getty@$console.service.d/"
+        cat <<'EOF' > "/etc/systemd/system/serial-getty@$console.service.d/override.conf"
+    [Service]
+    # systemd requires this empty ExecStart line to override
+    ExecStart=
+    ExecStart=-/sbin/agetty --autologin root -o '-p -- \\u' --keep-baud 115200,38400,9600 %I dumb
+EOF
+    done
+fi

tools/test-popular-containers/test-docker-rootfs.py

@@ -5,7 +5,7 @@
 # pylint:disable=invalid-name
 
 """
-Test all the ext4 rootfs in the current directory
+Test all the squashfs rootfs in the current directory
 """
 
 import os
@@ -29,7 +29,7 @@
 vmfcty = MicroVMFactory(DEFAULT_BINARY_DIR)
 # (may take a while to compile Firecracker...)
 
-for rootfs in Path(".").glob("*.ext4"):
+for rootfs in Path(".").glob("*.squashfs"):
     print(f">>>> Testing {rootfs}")
     uvm = vmfcty.build(kernel, rootfs)
     uvm.spawn()