ci: generate ext4 image after downloading artifacts

This avoids the need to store and download the image from S3.

Signed-off-by: Pablo Barbáchano <[email protected]>

Author: pb8o

resources/rebuild.sh

@@ -20,25 +20,6 @@ function install_dependencies {
     apt install -y bc flex bison gcc make libelf-dev libssl-dev squashfs-tools busybox-static tree cpio curl patch docker.io
 }
 
-function dir2ext4img {
-    # ext4
-    # https://unix.stackexchange.com/questions/503211/how-can-an-image-file-be-created-for-a-directory
-    local DIR=$1
-    local IMG=$2
-    # Default size for the resulting rootfs image is 300M
-    local SIZE=${3:-300M}
-    local TMP_MNT=$(mktemp -d)
-    truncate -s "$SIZE" "$IMG"
-    mkfs.ext4 -F "$IMG"
-    mount "$IMG" "$TMP_MNT"
-    tar c -C $DIR . |tar x -C "$TMP_MNT"
-    # cleanup
-    # Use the -l flag for lazy unmounting since sometimes umount fails
-    # with "device busy" and simply calling `sync` doesn't help
-    umount -l "$TMP_MNT"
-    rmdir $TMP_MNT
-}
-
 function prepare_docker {
     nohup /usr/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 &
 
@@ -102,8 +83,6 @@ EOF
     rootfs_img="$OUTPUT_DIR/$ROOTFS_NAME.squashfs"
     mv $rootfs/root/manifest $OUTPUT_DIR/$ROOTFS_NAME.manifest
     mksquashfs $rootfs $rootfs_img -all-root -noappend
-    rootfs_ext4=$OUTPUT_DIR/$ROOTFS_NAME.ext4
-    dir2ext4img $rootfs $rootfs_ext4
     rm -rf $rootfs
     for bin in fast_page_fault_helper fillmem init readmem; do
         rm $PWD/overlay/usr/local/bin/$bin
@@ -235,7 +214,7 @@ function build_al_kernels {
         local KERNEL_VERSION="all"
     elif [[ $# -ne 1 ]]; then
         die "Too many arguments in '$(basename $0) kernels' command. Please use \`$0 help\` for help."
-    else 
+    else
         KERNEL_VERSION=$1
         if [[ "$KERNEL_VERSION" != @(5.10|5.10-no-acpi|6.1) ]]; then
             die "Unsupported kernel version: '$KERNEL_VERSION'. Please use \`$0 help\` for help."
@@ -259,33 +238,33 @@ function build_al_kernels {
 
     # Undo kernel patches on top of AL configuration
     git restore $PWD/guest_configs
-    rm -rf $PWD/guest_configs/*.orig 
+    rm -rf $PWD/guest_configs/*.orig
 }
 
 function print_help {
     cat <<EOF
 Firecracker CI artifacts build script
 
 Usage: $(basename $0) [<command>] [<command args>]
-    
+
 Available commands:
-    
+
     all (default)
         Build CI rootfs and default guest kernels using configurations from
         resources/guest_configs.
         This will patch the guest configurations with all the patches under
         resources/guest_configs/patches.
         This is the default command, if no command is chosen.
-    
+
     rootfs
         Builds only the CI rootfs.
-    
+
     kernels [version]
         Builds our the currently supported CI kernels.
-    
+
         version: Optionally choose a kernel version to build. Supported
                  versions are: 5.10, 5.10-no-acpi or 6.1.
-    
+
     help
         Displays the help message and exits.
 EOF
@@ -310,7 +289,7 @@ function main {
     fi
 
     set -x
-        
+
     install_dependencies
 
     # Create the directory in which we will store the kernels and rootfs

tools/devtool

@@ -417,7 +417,7 @@ cmd_help() {
     echo "    build_ci_artifacts [all|rootfs|kernels]"
     echo "        Builds the rootfs and guest kernel artifacts we use for our CI."
     echo "        Run './tools/devtool build_ci_artifacts help' for more details about the available commands."
-    echo ""    
+    echo ""
 
     cat <<EOF
     test_debug [-- [<pytest args>]]
@@ -567,9 +567,7 @@ ensure_ci_artifacts() {
     if [ ! -d "$ARTIFACTS" ]; then
         mkdir -pv $ARTIFACTS
         aws s3 sync --no-sign-request "$S3_URL" "$ARTIFACTS"
-        # fix permissions
-        find "$ARTIFACTS" -type f -name "*.id_rsa" |xargs chmod -c 400
-        find "$ARTIFACTS/firecracker" -type f |xargs chmod -c 755
+        cmd_sh "./tools/setup-ci-artifacts.sh"
     fi
 }
 

tools/setup-ci-artifacts.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# fail if we encounter an error, uninitialized variable or a pipe breaks
+set -eu -o pipefail
+
+TOOLS_DIR=$(dirname $0)
+source "$TOOLS_DIR/functions"
+
+say "Setup CI artifacts"
+cd build/img/$(uname -m)
+
+say "Fix executable permissions"
+find "firecracker" -type f |xargs chmod -c 755
+
+say "Fix RSA key permissions"
+find . -type f -name "*.id_rsa" |xargs chmod -c 400
+
+for SQUASHFS in *.squashfs; do
+    EXT4=$(basename $SQUASHFS .squashfs).ext4
+
+    # Create rw ext4 image from ro squashfs
+    [ -f $EXT4 ] && continue
+    say "Converting $SQUASHFS to $EXT4"
+    truncate -s 400M $EXT4
+    unsquashfs $SQUASHFS
+    mkfs.ext4 -F $EXT4 -d squashfs-root
+    rm -rf squashfs-root
+done