Simplify A/B-Test orchestration

.buildkite/common.py

@@ -180,16 +180,7 @@ def random_str(k: int):
 
 def ab_revision_build(revision):
     """Generate steps for building an A/B-test revision"""
-    # Copied from framework/ab_test. Double dollar signs needed for Buildkite (otherwise it will try to interpolate itself)
-    return [
-        f"commitish={revision}",
-        f"if ! git cat-file -t $$commitish; then commitish=origin/{revision}; fi",
-        "branch_name=tmp-$$commitish",
-        "git branch $$branch_name $$commitish",
-        f"git clone -b $$branch_name . build/{revision}",
-        f"cd build/{revision} && ./tools/devtool -y build --release && cd -",
-        "git branch -D $$branch_name",
-    ]
+    return [f"./tools/devtool -y build --rev {revision} --release"]
 
 
 def shared_build():

.buildkite/pipeline_perf.py

@@ -98,7 +98,9 @@
     pytest_opts = ""
     if REVISION_A:
         devtool_opts += " --ab"
-        pytest_opts = f"{ab_opts} run build/{REVISION_A}/build/cargo_target/$(uname -m)-unknown-linux-musl/release build/{REVISION_B}/build/cargo_target/$(uname -m)-unknown-linux-musl/release --test {test_path}"
+        pytest_opts = (
+            f"{ab_opts} run build/{REVISION_A}/ build/{REVISION_B} --test {test_path}"
+        )
     else:
         # Passing `-m ''` below instructs pytest to collect tests regardless of
         # their markers (e.g. it will collect both tests marked as nonci, and

tests/framework/ab_test.py

@@ -101,14 +101,15 @@ def git_ab_test(
     return result_a, result_b, comparison
 
 
+DEFAULT_A_DIRECTORY = FC_WORKSPACE_DIR / "build" / "main"
 DEFAULT_B_DIRECTORY = FC_WORKSPACE_DIR / "build" / "cargo_target" / DEFAULT_TARGET_DIR
 
 
 def binary_ab_test(
     test_runner: Callable[[Path, bool], T],
     comparator: Callable[[T, T], U] = default_comparator,
     *,
-    a_directory: Path,
+    a_directory: Path = DEFAULT_A_DIRECTORY,
     b_directory: Path = DEFAULT_B_DIRECTORY,
 ):
     """

tools/ab_test.py

@@ -155,7 +155,7 @@ def load_data_series(report_path: Path, tag=None, *, reemit: bool = False):
 
 def collect_data(binary_dir: Path, tests: list[str]):
     """Executes the specified test using the provided firecracker binaries"""
-    # Example binary_dir: ../build/main/build/cargo_target/x86_64-unknown-linux-musl/release
+    binary_dir = binary_dir.resolve()
 
     print(f"Collecting samples with {binary_dir}")
     subprocess.run(

tools/devtool

@@ -446,12 +446,14 @@ cmd_build() {
     profile="debug"
     libc="musl"
 
+
     # Parse any command line args.
     while [ $# -gt 0 ]; do
         case "$1" in
             "-h"|"--help")  { cmd_help; exit 1;     } ;;
             "--debug")      { profile="debug";      } ;;
             "--release")    { profile="release";    } ;;
+            "--rev")       { shift; revision=$1; } ;;
             "--ssh-keys")
                 shift
                 [[ -z "$1" ]] && \
@@ -489,16 +491,40 @@ cmd_build() {
         extra_args="--volume $host_pub_key_path:$PUB_KEY_PATH:z \
                     --volume $host_priv_key_path:$PRIV_KEY_PATH:z"
 
+    workdir="$CTR_FC_ROOT_DIR"
+    if [ ! -z "$revision" ]; then
+      commitish="$revision"
+      if ! git cat-file -t "$commitish"; then commitish=origin/"$revision"; fi
+      branch_name=tmp-$commitish
+
+      tmp_dir=$(mktemp -d)
+
+      git branch $branch_name $commitish
+      git clone -b $branch_name . $tmp_dir
+      pushd $tmp_dir
+      workdir=$tmp_dir
+      extra_args="$extra_args --volume $tmp_dir:$tmp_dir:z"
+    fi
+
     # Run the cargo build process inside the container.
     # We don't need any special privileges for the build phase, so we run the
     # container as the current user/group.
     run_devctr \
         --user "$(id -u):$(id -g)" \
-        --workdir "$CTR_FC_ROOT_DIR" \
+        --workdir "$workdir" \
         ${extra_args} \
         -- \
         ./tools/release.sh --libc $libc --profile $profile
     ret=$?
+
+    if [ ! -z "$revision" ]; then
+      popd
+      git branch -D $branch_name
+      mkdir -p build/"$revision"
+      cp $tmp_dir/build/cargo_target/$(uname -m)-unknown-linux-$libc/$profile/* build/"$revision"
+      rm -rf $tmp_dir
+    fi
+
     return $ret
 }