Add a devtool command for build CI artifacts

.gitignore

@@ -12,3 +12,6 @@ test_results/*
 *.profraw
 .DS_Store
 *.bin
+/resources/linux
+/resources/x86_64
+/resources/aarch64

docs/rootfs-and-kernel-setup.md

@@ -56,21 +56,28 @@ For a list of currently supported kernel versions, check out the
 ### Use the provided recipe
 
 The kernel images used in our CI to test Firecracker's features are obtained by
-using the recipe inside devtool:
+running the script `resources/rebuild.sh`.
+
+Users can build those locally by running:
 
 ```bash
-config="resources/guest_configs/microvm-kernel-x86_64-4.14.config"
-./tools/devtool build_kernel -c $config -n 8
+./tools/devtool build_ci_artifacts kernels
 ```
 
-or
+This will build all versions that we currently use in our CI. `kernels`
+subcommand allows passing a specific kernel version to build. For example:
 
 ```bash
-config="resources/guest_configs/microvm-kernel-arm64-4.14.config"
-./tools/devtool build_kernel -c $config -n 8
+./tools/devtool build_ci_artifacts kernels 6.1
 ```
 
-on an aarch64 machine.
+will build only the 6.1 kernel.
+
+Currently supported kernel versions are: `5.10`, `5.10-no-acpi` (same as 5.10
+but without ACPI support) and `6.1`.
+
+After the command finishes, the kernels along with the corresponding KConfig
+used will be stored under `resources/$(uname -m)`.
 
 ## Creating a rootfs Image
 
@@ -170,11 +177,11 @@ The disk images used in our CI to test Firecracker's features are obtained by
 using the recipe (in a Ubuntu 22.04 host):
 
 ```bash
-./resources/rebuild.sh
+./tools/devtool build_ci_artifacts rootfs
 ```
 
 The images resulting using this method are minimized Ubuntu 22.04. Feel free to
 adjust the script(s) to suit your use case.
 
-You should now have a kernel image (`vmlinux`) and a rootfs image
-(`rootfs.ext4`), that you can boot with Firecracker.
+You should now have a rootfs image (`ubuntu-22.04.ext4`), that you can boot with
+Firecracker.

resources/rebuild.sh

@@ -5,17 +5,19 @@
 # fail if we encounter an error, uninitialized variable or a pipe breaks
 set -eu -o pipefail
 
-set -x
 PS4='+\t '
 
 cd $(dirname $0)
 ARCH=$(uname -m)
 OUTPUT_DIR=$PWD/$ARCH
 
+GIT_ROOT_DIR=$(git rev-parse --show-toplevel)
+source "$GIT_ROOT_DIR/tools/functions"
+
 # Make sure we have all the needed tools
 function install_dependencies {
-    sudo apt update
-    sudo apt install -y bc flex bison gcc make libelf-dev libssl-dev squashfs-tools busybox-static tree cpio curl
+    apt update
+    apt install -y bc flex bison gcc make libelf-dev libssl-dev squashfs-tools busybox-static tree cpio curl patch docker.io
 }
 
 function dir2ext4img {
@@ -28,13 +30,21 @@ function dir2ext4img {
     local TMP_MNT=$(mktemp -d)
     truncate -s "$SIZE" "$IMG"
     mkfs.ext4 -F "$IMG"
-    sudo mount "$IMG" "$TMP_MNT"
-    sudo tar c -C $DIR . |sudo tar x -C "$TMP_MNT"
+    mount "$IMG" "$TMP_MNT"
+    tar c -C $DIR . |tar x -C "$TMP_MNT"
     # cleanup
-    sudo umount "$TMP_MNT"
+    # Use the -l flag for lazy unmounting since sometimes umount fails
+    # with "device busy" and simply calling `sync` doesn't help
+    umount -l "$TMP_MNT"
     rmdir $TMP_MNT
 }
 
+function prepare_docker {
+    nohup /usr/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 &
+
+    # Wait for Docker socket to be created
+    timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
+}
 
 function compile_and_install {
     local C_FILE=$1
@@ -50,7 +60,10 @@ function build_rootfs {
     local flavour=${2}
     local FROM_CTR=public.ecr.aws/ubuntu/ubuntu:$flavour
     local rootfs="tmp_rootfs"
-    mkdir -pv "$rootfs" "$OUTPUT_DIR"
+    mkdir -pv "$rootfs"
+
+    # Launch Docker
+    prepare_docker
 
     cp -rvf overlay/* $rootfs
 
@@ -74,25 +87,29 @@ mkdir -pv $rootfs/var/lib/dpkg/
 EOF
 
     # TBD what abt /etc/hosts?
-    echo |sudo tee $rootfs/etc/resolv.conf
+    echo | tee $rootfs/etc/resolv.conf
 
     # Generate key for ssh access from host
     if [ ! -s id_rsa ]; then
         ssh-keygen -f id_rsa -N ""
     fi
-    sudo install -d -m 0600 "$rootfs/root/.ssh/"
-    sudo cp id_rsa.pub "$rootfs/root/.ssh/authorized_keys"
+    install -d -m 0600 "$rootfs/root/.ssh/"
+    cp id_rsa.pub "$rootfs/root/.ssh/authorized_keys"
     id_rsa=$OUTPUT_DIR/$ROOTFS_NAME.id_rsa
-    sudo cp id_rsa $id_rsa
+    cp id_rsa $id_rsa
 
     # -comp zstd but guest kernel does not support
     rootfs_img="$OUTPUT_DIR/$ROOTFS_NAME.squashfs"
-    sudo mv $rootfs/root/manifest $OUTPUT_DIR/$ROOTFS_NAME.manifest
-    sudo mksquashfs $rootfs $rootfs_img -all-root -noappend
+    mv $rootfs/root/manifest $OUTPUT_DIR/$ROOTFS_NAME.manifest
+    mksquashfs $rootfs $rootfs_img -all-root -noappend
     rootfs_ext4=$OUTPUT_DIR/$ROOTFS_NAME.ext4
     dir2ext4img $rootfs $rootfs_ext4
-    sudo rm -rf $rootfs
-    sudo chown -Rc $USER. $OUTPUT_DIR
+    rm -rf $rootfs
+    for bin in fast_page_fault_helper fillmem init readmem; do
+        rm $PWD/overlay/usr/local/bin/$bin
+    done
+    rm -f id_rsa{,.pub}
+    rm -f nohup.out
 }
 
 
@@ -199,31 +216,117 @@ function build_al_kernel {
     popd &>/dev/null
 }
 
-#### main ####
+function prepare_and_build_rootfs {
+    BIN=overlay/usr/local/bin
+    compile_and_install $BIN/init.c $BIN/init
+    compile_and_install $BIN/fillmem.c $BIN/fillmem
+    compile_and_install $BIN/fast_page_fault_helper.c $BIN/fast_page_fault_helper
+    compile_and_install $BIN/readmem.c $BIN/readmem
+    if [ $ARCH == "aarch64" ]; then
+        compile_and_install $BIN/devmemread.c $BIN/devmemread
+    fi
+
+    build_rootfs ubuntu-22.04 jammy
+    build_initramfs
+}
 
-install_dependencies
+function build_al_kernels {
+    if [[ $# = 0 ]]; then
+        local KERNEL_VERSION="all"
+    elif [[ $# -ne 1 ]]; then
+        die "Too many arguments in '$(basename $0) kernels' command. Please use \`$0 help\` for help."
+    else 
+        KERNEL_VERSION=$1
+        if [[ "$KERNEL_VERSION" != @(5.10|5.10-no-acpi|6.1) ]]; then
+            die "Unsupported kernel version: '$KERNEL_VERSION'. Please use \`$0 help\` for help."
+        fi
+    fi
 
-BIN=overlay/usr/local/bin
-compile_and_install $BIN/init.c    $BIN/init
-compile_and_install $BIN/fillmem.c $BIN/fillmem
-compile_and_install $BIN/fast_page_fault_helper.c $BIN/fast_page_fault_helper
-compile_and_install $BIN/readmem.c $BIN/readmem
-if [ $ARCH == "aarch64" ]; then
-    compile_and_install $BIN/devmemread.c $BIN/devmemread
-fi
+    clone_amazon_linux_repo
 
-build_rootfs ubuntu-22.04 jammy
-build_initramfs
+    # Apply kernel patches on top of AL configuration
+    apply_kernel_patches_for_ci
 
-clone_amazon_linux_repo
+    if [[ "$KERNEL_VERSION" == @(all|5.10) ]]; then
+        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10.config
+    fi
+    if [[ $ARCH == "x86_64" && "$KERNEL_VERSION" == @(all|5.10-no-acpi) ]]; then
+        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10-no-acpi.config
+    fi
+    if [[ "$KERNEL_VERSION" == @(all|6.1) ]]; then
+        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-6.1.config 5.10
+    fi
 
-# Apply kernel patches on top of AL configuration
-apply_kernel_patches_for_ci
+    # Undo kernel patches on top of AL configuration
+    git restore $PWD/guest_configs
+    rm -rf $PWD/guest_configs/*.orig 
+}
 
-build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10.config
-if [ $ARCH == "x86_64" ]; then
-    build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10-no-acpi.config
-fi
-build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-6.1.config
+function print_help {
+    cat <<EOF
+Firecracker CI artifacts build script
+
+Usage: $(basename $0) [<command>] [<command args>]
+
+Available commands:
+
+    all (default)
+        Build CI rootfs and default guest kernels using configurations from
+        resources/guest_configs.
+        This will patch the guest configurations with all the patches under
+        resources/guest_configs/patches.
+        This is the default command, if no command is chosen.
+
+    rootfs
+        Builds only the CI rootfs.
+
+    kernels [version]
+        Builds our the currently supported CI kernels.
+
+        version: Optionally choose a kernel version to build. Supported
+                 versions are: 5.10, 5.10-no-acpi or 6.1.
+
+    help
+        Displays the help message and exits.
+EOF
+}
+
+function main {
+    if [[ $# = 0 ]]; then
+        local MODE="all"
+    else
+        case $1 in
+            all|rootfs|kernels)
+                local MODE=$1
+                shift
+                ;;
+            help)
+                print_help
+                exit 0
+                ;;
+            *)
+                die "Unknown command: '$1'. Please use \`$0 help\` for help."
+        esac
+    fi
+
+    set -x
+
+    install_dependencies
+
+    # Create the directory in which we will store the kernels and rootfs
+    mkdir -pv $OUTPUT_DIR
+
+    if [[ "$MODE" =~ (all|rootfs) ]]; then
+        say "Building rootfs"
+        prepare_and_build_rootfs
+    fi
+
+    if [[ "$MODE" =~ (all|kernels) ]]; then
+        say "Building CI kernels"
+        build_al_kernels "$@"
+    fi
+
+    tree -h $OUTPUT_DIR
+}
 
-tree -h $OUTPUT_DIR
+main "$@"

tools/devtool

@@ -134,6 +134,8 @@ OPT_UNATTENDED=false
 # Get the target prefix to avoid repeated calls to uname -m
 TARGET_PREFIX="$(uname -m)-unknown-linux-"
 
+# Container path to directory where we store built CI artifacts.
+CTR_CI_ARTIFACTS_PATH="${CTR_FC_ROOT_DIR}/resources/$(uname -m)"
 
 # Check if Docker is available and exit if it's not.
 # Upon returning from this call, the caller can be certain Docker is available.
@@ -231,7 +233,7 @@ cmd_fix_perms() {
     run_devctr \
         --workdir "$CTR_FC_ROOT_DIR" \
         -- \
-        chown -R "$(id -u):$(id -g)" "$CTR_FC_BUILD_DIR"  "$CTR_TEST_RESULTS_DIR"
+        chown -R "$(id -u):$(id -g)" "$CTR_FC_BUILD_DIR" "$CTR_TEST_RESULTS_DIR" "$CTR_CI_ARTIFACTS_PATH"
 }
 
 # Builds the development container from its Dockerfile.
@@ -412,6 +414,10 @@ cmd_help() {
     echo "            --performance            Tweak various setting of the host running the tests (such as C- and P-states)"
     echo "                                     to achieve consistent performance. Used for running performance tests in CI."
     echo ""
+    echo "    build_ci_artifacts [all|rootfs|kernels]"
+    echo "        Builds the rootfs and guest kernel artifacts we use for our CI."
+    echo "        Run './tools/devtool build_ci_artifacts help' for more details about the available commands."
+    echo ""    
 
     cat <<EOF
     test_debug [-- [<pytest args>]]
@@ -1161,6 +1167,20 @@ cmd_install() {
     done
 }
 
+cmd_build_ci_artifacts() {
+    # Check prerequisites
+    ensure_devctr
+
+    # We need to run nested Docker here, so run this container as privileged.
+    run_devctr \
+        --privileged \
+        --workdir "$CTR_FC_ROOT_DIR" \
+        -- \
+        ./resources/rebuild.sh "$@"
+
+    cmd_fix_perms
+}
+
 
 main() {