[A/B] Update script to use stored metrics.json

tests/framework/ab_test.py

@@ -21,19 +21,14 @@
 of both invocations is the same, the test passes (with us being alerted to this situtation via a special pipeline that
 does not block PRs). If not, it fails, preventing PRs from introducing new vulnerable dependencies.
 """
-import statistics
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Callable, List, Optional, TypeVar
-
-import scipy
+from typing import Callable, Optional, TypeVar
 
 from framework import utils
-from framework.defs import FC_WORKSPACE_DIR
 from framework.properties import global_props
 from framework.utils import CommandReturn
 from framework.with_filelock import with_filelock
-from host_tools.cargo_build import DEFAULT_TARGET_DIR
 
 # Locally, this will always compare against main, even if we try to merge into, say, a feature branch.
 # We might want to do a more sophisticated way to determine a "parent" branch here.
@@ -98,27 +93,6 @@ def git_ab_test(
         return result_a, result_b, comparison
 
 
-DEFAULT_A_DIRECTORY = FC_WORKSPACE_DIR / "build" / "main"
-DEFAULT_B_DIRECTORY = FC_WORKSPACE_DIR / "build" / "cargo_target" / DEFAULT_TARGET_DIR
-
-
-def binary_ab_test(
-    test_runner: Callable[[Path, bool], T],
-    comparator: Callable[[T, T], U] = default_comparator,
-    *,
-    a_directory: Path = DEFAULT_A_DIRECTORY,
-    b_directory: Path = DEFAULT_B_DIRECTORY,
-):
-    """
-    Similar to `git_ab_test`, but instead of locally checking out different revisions, it operates on
-    directories containing firecracker/jailer binaries
-    """
-    result_a = test_runner(a_directory, True)
-    result_b = test_runner(b_directory, False)
-
-    return result_a, result_b, comparator(result_a, result_b)
-
-
 def git_ab_test_host_command_if_pr(
     command: str,
     *,
@@ -170,32 +144,6 @@ def set_did_not_grow_comparator(
     )
 
 
-def check_regression(
-    a_samples: List[float], b_samples: List[float], *, n_resamples: int = 9999
-):
-    """Checks for a regression by performing a permutation test. A permutation test is a non-parametric test that takes
-    three parameters: Two populations (sets of samples) and a function computing a "statistic" based on two populations.
-    First, the test computes the statistic for the initial populations. It then randomly
-    permutes the two populations (e.g. merges them and then randomly splits them again). For each such permuted
-    population, the statistic is computed. Then, all the statistics are sorted, and the percentile of the statistic for the
-    initial populations is computed. We then look at the fraction of statistics that are larger/smaller than that of the
-    initial populations. The minimum of these two fractions will then become the p-value.
-
-    The idea is that if the two populations are indeed drawn from the same distribution (e.g. if performance did not
-    change), then permuting will not affect the statistic (indeed, it should be approximately normal-distributed, and
-    the statistic for the initial populations will be somewhere "in the middle").
-
-    Useful for performance tests.
-    """
-    return scipy.stats.permutation_test(
-        (a_samples, b_samples),
-        # Compute the difference of means, such that a positive different indicates potential for regression.
-        lambda x, y: statistics.mean(y) - statistics.mean(x),
-        vectorized=False,
-        n_resamples=n_resamples,
-    )
-
-
 @with_filelock
 def git_clone(clone_path, commitish):
     """Clone the repository at `commit`.

tests/host_tools/metrics.py

@@ -133,89 +133,3 @@ def emit_raw_emf(emf_msg: dict):
             (json.dumps(emf_msg) + "\n").encode("utf-8"),
             (emf_endpoint.hostname, emf_endpoint.port),
         )
-
-
-UNIT_REDUCTIONS = {
-    "Microseconds": "Milliseconds",
-    "Milliseconds": "Seconds",
-    "Bytes": "Kilobytes",
-    "Kilobytes": "Megabytes",
-    "Megabytes": "Gigabytes",
-    "Gigabytes": "Terabytes",
-    "Bits": "Kilobits",
-    "Kilobits": "Megabits",
-    "Megabits": "Gigabits",
-    "Gigabits": "Terabit",
-    "Bytes/Second": "Kilobytes/Second",
-    "Kilobytes/Second": "Megabytes/Second",
-    "Megabytes/Second": "Gigabytes/Second",
-    "Gigabytes/Second": "Terabytes/Second",
-    "Bits/Second": "Kilobits/Second",
-    "Kilobits/Second": "Megabits/Second",
-    "Megabits/Second": "Gigabits/Second",
-    "Gigabits/Second": "Terabits/Second",
-}
-INV_UNIT_REDUCTIONS = {v: k for k, v in UNIT_REDUCTIONS.items()}
-
-
-UNIT_SHORTHANDS = {
-    "Seconds": "s",
-    "Microseconds": "μs",
-    "Milliseconds": "ms",
-    "Bytes": "B",
-    "Kilobytes": "KB",
-    "Megabytes": "MB",
-    "Gigabytes": "GB",
-    "Terabytes": "TB",
-    "Bits": "Bit",
-    "Kilobits": "KBit",
-    "Megabits": "MBit",
-    "Gigabits": "GBit",
-    "Terabits": "TBit",
-    "Percent": "%",
-    "Count": "",
-    "Bytes/Second": "B/s",
-    "Kilobytes/Second": "KB/s",
-    "Megabytes/Second": "MB/s",
-    "Gigabytes/Second": "GB/s",
-    "Terabytes/Second": "TB/s",
-    "Bits/Second": "Bit/s",
-    "Kilobits/Second": "KBit/s",
-    "Megabits/Second": "MBit/s",
-    "Gigabits/Second": "GBit/s",
-    "Terabits/Second": "TBit/s",
-    "Count/Second": "Hz",
-    "None": "",
-}
-
-
-def reduce_value(value, unit):
-    """
-    Utility function for expressing a value in the largest possible unit in which it would still be >= 1
-
-    For example, `reduce_value(1_000_000, Bytes)` would return (1, Megabytes)
-    """
-    # Could do this recursively, but I am worried about infinite recursion
-    # due to precision problems (e.g. infinite loop of dividing/multiplying by 1000, alternating
-    # between values < 1 and >= 1000).
-    while abs(value) < 1 and unit in INV_UNIT_REDUCTIONS:
-        value *= 1000
-        unit = INV_UNIT_REDUCTIONS[unit]
-    while abs(value) >= 1000 and unit in UNIT_REDUCTIONS:
-        value /= 1000
-        unit = UNIT_REDUCTIONS[unit]
-
-    return value, unit
-
-
-def format_with_reduced_unit(value, unit):
-    """
-    Utility function for pretty printing a given value by choosing a unit as large as possible,
-    and then outputting its shorthand.
-
-    For example, `format_with_reduced_unit(1_000_000, Bytes)` would return "1MB".
-    """
-    reduced_value, reduced_unit = reduce_value(value, unit)
-    formatted_unit = UNIT_SHORTHANDS.get(reduced_unit, reduced_unit)
-
-    return f"{reduced_value:.2f}{formatted_unit}"