Improvements for history run

Author: RobertoPrevato

.github/workflows/perfhistory.yml

@@ -19,10 +19,15 @@ on:
       commits:
         description: "List of commits or tags to benchmark (space-separated)"
         required: true
-        default: "v2.0.1 v2.2.0 v2.3.0"
+        default: "v2.0.1 v2.2.0 v2.3.0 current"
+      memory:
+        description: "Include memory benchmark (Y|N). Time consuming."
+        required: true
+        default: "N"
 
 env:
-  DEFAULT_COMMITS: "v2.0.1 v2.2.0 v2.3.0"
+  DEFAULT_MEMORY: "N"
+  DEFAULT_COMMITS: "v2.0.1 v2.2.0 v2.3.0 current"
 
 jobs:
   perf-tests:
@@ -69,11 +74,20 @@ jobs:
       - name: Run benchmark
         shell: bash
         env:
+          MEMORY: ${{ github.event.inputs.memory || env.DEFAULT_MEMORY }}
           COMMITS: ${{ github.event.inputs.commits || env.DEFAULT_COMMITS }}
         run: |
+
           echo "Running benchmarks for commits: $COMMITS"
           export PYTHONPATH="."
-          python perf/historyrun.py --commits $COMMITS --times 3
+
+          if [ $MEMORY == "Y" ]; then
+            echo "➔ Including memory benchmarks 🟢"
+            python perf/historyrun.py --commits $COMMITS --times 3 --memory
+          else
+            echo "➔ Excluding memory benchmarks 🔴"
+            python perf/historyrun.py --commits $COMMITS --times 3 --no-memory
+          fi
 
       - name: Upload results
         uses: actions/upload-artifact@v4

perf/historyrun.py

@@ -86,7 +86,7 @@ def make_compile():
     subprocess.check_output(["make", "compile"], universal_newlines=True)
 
 
-def run_tests(iterations: int, output_dir: str, times: int):
+def run_tests(iterations: int, output_dir: str, times: int, memory: bool):
     subprocess.check_output(
         [
             "python",
@@ -97,6 +97,7 @@ def run_tests(iterations: int, output_dir: str, times: int):
             output_dir,
             "--times",
             str(times),
+            "--memory" if memory else "--no-memory",
         ],
         universal_newlines=True,
     )
@@ -142,6 +143,12 @@ def main():
         nargs="+",  # Accept one or more commit SHAs
         help="List of Git commit SHAs to benchmark",
     )
+    parser.add_argument(
+        "--memory",
+        default=True,
+        action=argparse.BooleanOptionalAction,
+        help="Includes or skips memory benchmarks (included by default)",
+    )
     args = parser.parse_args()
 
     # Example usage of the commits argument
@@ -155,15 +162,17 @@ def main():
         output_dir = Path(temp_dir) / "results"
         copy_perf_code(temp_dir)
 
-        with gitcontext():
+        with gitcontext() as current_branch:
             for commit in args.commits:
+                if commit.lower() in {"current", "last", "head"}:
+                    commit = current_branch
                 subprocess.check_output(
                     ["git", "checkout", "-f", commit], universal_newlines=True
                 )
                 logger.info("Checked out commit: %s", commit)
                 make_compile()
                 restore_perf_code(temp_dir)
-                run_tests(args.iterations, str(output_dir), args.times)
+                run_tests(args.iterations, str(output_dir), args.times, args.memory)
 
         # Copy the results from output_dir to ./benchmark_results
         copy_results(str(output_dir), "./benchmark_results")

perf/main.py

@@ -132,7 +132,7 @@ def get_system_info():
     }
 
 
-async def run_all_benchmarks(iterations: int, key: str, no_memory: bool):
+async def run_all_benchmarks(iterations: int, key: str, memory: bool):
     results = {
         "timestamp": datetime.now().isoformat(),
         "git_info": get_git_info(),
@@ -156,7 +156,7 @@ async def run_all_benchmarks(iterations: int, key: str, no_memory: bool):
         else:
             results["benchmarks"][name] = func(iterations)
 
-    if no_memory:
+    if memory is False:
         return results
 
     # Memory benchmarks
@@ -205,14 +205,15 @@ def save_results(results, output_dir="./benchmark_results"):
         help="Optional filter to run specific benchmarks",
     )
     parser.add_argument(
-        "--no-memory",
-        action="store_true",
-        help="If set, skips memory benchmarks",
+        "--memory",
+        default=True,
+        action=argparse.BooleanOptionalAction,
+        help="Includes or skips memory benchmarks (included by default)",
     )
     args = parser.parse_args()
 
     for _ in range(args.times):
         results = asyncio.run(
-            run_all_benchmarks(args.iterations, args.filter, args.no_memory)
+            run_all_benchmarks(args.iterations, args.filter, args.memory)
         )
         save_results(results, args.output_dir)