add compare to benchmark action

Author: ianayl

devops/actions/run-tests/benchmark_v2/action.yml

@@ -100,6 +100,7 @@ runs:
 
       # TODO accomodate for different GPUs and backends
       SAVE_NAME="Baseline_PVC_L0"
+      SAVE_TIMESTAMP="$(date +'%Y%m%d_%H%M%S')"
       if [ -n "$BUILD_HASH" ]; then
           SAVE_NAME="Commit_PVC_$BUILD_HASH"
       fi
@@ -112,14 +113,17 @@ runs:
         --results-dir "./llvm-ci-perf-results/$RUNNER_NAME" \
         --output-dir "./llvm-ci-perf-results/$RUNNER_NAME" \
         --preset Minimal
+        --timestamp-override "$SAVE_TIMESTAMP"
       echo "-----"
+      python3 ./devops/scripts/benchmarks/compare.py to_hist \
+        --name Baseline_PVC_L0 \
+        --compare_file "./llvm-ci-perf-results/$RUNNER_NAME/results/$SAVE_NAME_$SAVE_TIMESTAMP.json"
+        --results_dir "./llvm-ci-perf-results/$RUNNER_NAME/results/"
+
   - name: Push compute-benchmarks results
     if: inputs.upload_results == 'true' && always()
     shell: bash
     run: |
-      # TODO redo configuration
-      # $(python ./devops/scripts/benchmarking/load_config.py ./devops constants)
-
       cd "./llvm-ci-perf-results"
       git config user.name "SYCL Benchmarking Bot"
       git config user.email "[email protected]"

devops/scripts/benchmarks/compare.py

@@ -292,6 +292,7 @@ def print_regression(entry: dict):
         if regressions:
             print("#\n# Regressions:\n#\n")
             for test in regressions: print_regression(test)
+            exit(1)  # Exit 1 to trigger github test failure
     else:
         print("Unsupported operation: exiting.")
         exit(1)

devops/scripts/benchmarks/history.py

@@ -11,6 +11,7 @@
 from options import Compare, options
 from datetime import datetime, timezone
 from utils.utils import run
+from utils.validate import Validate
 
 
 class BenchmarkHistory:
@@ -30,15 +31,18 @@ def load_result(self, file_path: Path) -> BenchmarkRun:
     def load(self, n: int):
         results_dir = Path(self.dir) / "results"
         if not results_dir.exists() or not results_dir.is_dir():
-            return []
+            print(f"Warning: {results_dir} is not a valid directory: no historic results loaded.")
+            return
 
         # Get all JSON files in the results directory
         benchmark_files = list(results_dir.glob("*.json"))
 
         # Extract timestamp and sort files by it
         def extract_timestamp(file_path: Path) -> str:
             try:
-                return file_path.stem.split("_")[-1]
+                # Assumes results are stored as <name>_YYYYMMDD_HHMMSS.json 
+                ts = file_path.stem[-len("YYYYMMDD_HHMMSS"):]
+                return ts if Validate.timestamp(ts) else ""
             except IndexError:
                 return ""
 
@@ -98,7 +102,11 @@ def save(self, save_name, results: list[Result], to_file=True):
         os.makedirs(results_dir, exist_ok=True)
 
         # Use formatted timestamp for the filename
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        timestamp = (
+            datetime.now().strftime("%Y%m%d_%H%M%S")
+            if options.timestamp_override is None else 
+            options.timestamp_override
+        )
         file_path = Path(os.path.join(results_dir, f"{save_name}_{timestamp}.json"))
         with file_path.open("w") as file:
             json.dump(serialized, file, indent=4)

devops/scripts/benchmarks/main.py

@@ -17,6 +17,7 @@
 from history import BenchmarkHistory
 from utils.utils import prepare_workdir
 from utils.compute_runtime import *
+from utils.validate import Validate
 from presets import enabled_suites, presets
 
 import argparse
@@ -481,6 +482,11 @@ def validate_and_parse_env_args(env_args):
         help="Specify a custom results directory",
         default=options.custom_results_dir,
     )
+    parser.add_argument(
+        "--timestamp-override",
+        type=str,
+        help="Used in CI to enforce use of same timestamp across scripts",
+    )
 
     args = parser.parse_args()
     additional_env_vars = validate_and_parse_env_args(args.env)
@@ -518,6 +524,10 @@ def validate_and_parse_env_args(env_args):
         if not os.path.isdir(args.output_dir):
             parser.error("Specified --output-dir is not a valid path")
         options.output_directory = os.path.abspath(args.output_dir)
+    if args.timestamp_override is not None:
+        if not Validate.timestamp(args.timestamp_override):
+            parser.error("--timestamp_override is not a valid timestamp")
+        options.timestamp_override = args.timestamp_override
 
     benchmark_filter = re.compile(args.filter) if args.filter else None
 

devops/scripts/benchmarks/options.py

@@ -46,6 +46,7 @@ class Options:
     custom_results_dir = None
 
     regression_threshold: float = 0.05
+    timestamp_override: str = None
 
 
 options = Options()