ci: add option to use a different Firecracker binary

This enables us to run a pipeline against an existing binary.

The binary has to exist in the Buildkite artifacts buckets.

(cherry-picked from eb83708 with some
rebasing changes)

Signed-off-by: Pablo Barbáchano <[email protected]>

Author: pb8o

.buildkite/common.py

@@ -110,6 +110,23 @@ def run_all_tests(changed_files):
     )
 
 
+def devtool_test(devtool_opts=None, pytest_opts=None, binary_dir=None):
+    """Generate a `devtool test` command"""
+    cmds = []
+    parts = ["./tools/devtool -y test"]
+    if devtool_opts:
+        parts.append(devtool_opts)
+    parts.append("--")
+    if binary_dir is not None:
+        cmds.append(f'buildkite-agent artifact download "{binary_dir}/$(uname -m)/*" .')
+        cmds.append(f"chmod -v a+x {binary_dir}/**/*")
+        parts.append(f"--binary-dir=../{binary_dir}/$(uname -m)")
+    if pytest_opts:
+        parts.append(pytest_opts)
+    cmds.append(" ".join(parts))
+    return cmds
+
+
 class DictAction(argparse.Action):
     """An argparse action that can receive a nested dictionary
 
@@ -159,3 +176,10 @@ def __call__(self, parser, namespace, value, option_string=None):
     default={},
     type=str,
 )
+COMMON_PARSER.add_argument(
+    "--binary-dir",
+    help="Use the Firecracker binaries from this path",
+    required=False,
+    default=None,
+    type=str,
+)

.buildkite/pipeline_perf.py

@@ -5,7 +5,7 @@
 """Generate Buildkite performance pipelines dynamically"""
 import os
 
-from common import COMMON_PARSER, group, overlay_dict, pipeline_to_json
+from common import COMMON_PARSER, devtool_test, group, overlay_dict, pipeline_to_json
 
 # In `devtool_opts`, we restrict both the set of CPUs on which the docker container's threads can run,
 # and its memory node. For the cpuset, we pick a continuous set of CPUs from a single NUMA node
@@ -66,13 +66,17 @@ def build_group(test):
     devtool_opts = test.pop("devtool_opts")
     test_path = test.pop("test_path")
     ab_opts = test.pop("ab_opts", "")
+    devtool_opts += " --performance"
+    pytest_opts = ""
     if REVISION_A:
-        command = f"./tools/devtool -y test --performance --ab {devtool_opts} -- {REVISION_A} {REVISION_B} --test {test_path} {ab_opts}"
+        devtool_opts += " --ab"
+        pytest_opts = f"{REVISION_A} {REVISION_B} --test {test_path} {ab_opts}"
     else:
-        command = f"./tools/devtool -y test --performance {devtool_opts} -- -m nonci {test_path}"
+        pytest_opts += f" -m nonci {test_path}"
+    binary_dir = test.pop("binary_dir")
     return group(
         label=test.pop("label"),
-        command=command,
+        command=devtool_test(devtool_opts, pytest_opts, binary_dir),
         artifacts=["./test_results/*"],
         instances=test.pop("instances"),
         platforms=test.pop("platforms"),
@@ -99,6 +103,7 @@ def build_group(test):
     test_data.setdefault("instances", args.instances)
     # use ag=1 instances to make sure no two performance tests are scheduled on the same instance
     test_data.setdefault("agents", {"ag": 1})
+    test_data["binary_dir"] = args.binary_dir
     test_data = overlay_dict(test_data, args.step_param)
     test_data["retry"] = {
         "automatic": [

.buildkite/pipeline_pr.py

@@ -6,6 +6,7 @@
 
 from common import (
     COMMON_PARSER,
+    devtool_test,
     get_changed_files,
     group,
     overlay_dict,
@@ -61,7 +62,10 @@
 
 functional_grp = group(
     "⚙ Functional and security 🔒",
-    "./tools/devtool -y test -- -n 8 --dist worksteal integration_tests/{{functional,security}}",
+    devtool_test(
+        pytest_opts="-n 8 --dist worksteal integration_tests/{{functional,security}}",
+        binary_dir=args.binary_dir,
+    ),
     **defaults,
 )
 
@@ -77,7 +81,11 @@
 
 performance_grp = group(
     "⏱ Performance",
-    "./tools/devtool -y test --performance -c 1-10 -m 0 -- ../tests/integration_tests/performance/",
+    devtool_test(
+        devtool_opts="--performance -c 1-10 -m 0",
+        pytest_opts="../tests/integration_tests/performance/",
+        binary_dir=args.binary_dir,
+    ),
     **defaults_for_performance,
 )
 

tests/conftest.py

@@ -248,6 +248,8 @@ def microvm_factory(request, record_property, results_dir):
     if binary_dir := request.config.getoption("--binary-dir"):
         fc_binary_path = Path(binary_dir) / "firecracker"
         jailer_binary_path = Path(binary_dir) / "jailer"
+        if not fc_binary_path.exists():
+            raise RuntimeError("Firecracker binary does not exist")
     else:
         fc_binary_path, jailer_binary_path = build_tools.get_firecracker_binaries()
     record_property("firecracker_bin", str(fc_binary_path))