update benchmarking toolchain

Author: crimist

bench/.gitignore

@@ -1 +1,3 @@
 trakxbench
+
+__pycache__/

bench/README.md

@@ -42,7 +42,7 @@ It also sets `TRAKX_CACHE` to a benchmark-only directory and deletes it between
 You run one script on the server and one on the client. They coordinate automatically
 so each goroutine count is started, benchmarked, stopped, and cleaned up.
 
-Requirements: Python 3 on both machines.
+Requirements: Python 3 and Go on both machines (the orchestrators build binaries every run).
 Run these commands from the repo root so relative paths resolve.
 
 ### 1) On the server (runs Trakx)
@@ -58,18 +58,20 @@ Notes:
 - It clears the cache directory between runs (database reset).
 - It sets `TRAKX_CACHE` so old backups are never reused.
 - Do not point `--cache-dir` at any important data; it is deleted every run.
+- The server script builds `trakx` every run (override path with `--trakx-bin`).
 
 ### 2) On the client (runs trakxbench)
 
 ```bash
-bench/orchestrator_client.py --server 10.0.0.10:9077 --mode udp --goroutines 1,2,4,6,8,12,16,24,32,48,64 \\
-  --udp 10.0.0.10:1337 --http 10.0.0.10:1337 --stats-url http://10.0.0.10:1337/stats \\
+bench/orchestrator_client.py --server 192.168.1.100:9077 --mode udp --goroutines 1,2,4,6,8,12,16,24,32,48,64 \
+  --udp 192.168.1.100:1337 --http 192.168.1.100:1337 --stats-url http://192.168.1.100:1337/stats \
   --duration 2m --warmup 20s --concurrency 512 --scrape-ratio 0.03
 ```
 
 The client will create `bench/results/<timestamp>-udp/` with one JSON file per run.
 
 If you need extra `trakxbench` flags, pass them through with `--bench-args`.
+The client script builds `trakxbench` every run (override path with `--bench-bin`).
 
 ### Manual step-through (optional)
 
@@ -99,13 +101,13 @@ Common flags you will use:
 UDP example (closed-loop, max throughput):
 
 ```bash
-bench/trakxbench --mode udp --udp 10.0.0.10:1337 --duration 2m --warmup 20s --concurrency 512 --scrape-ratio 0.03 --stats http://10.0.0.10:1337/stats --out results-udp.json
+bench/trakxbench --mode udp --udp 192.168.1.100:1337 --duration 2m --warmup 20s --concurrency 512 --scrape-ratio 0.03 --stats http://192.168.1.100:1337/stats --out results-udp.json
 ```
 
 HTTP example (open-loop, 1000 req/s target):
 
 ```bash
-bench/trakxbench --mode http --http 10.0.0.10:1337 --rate 1000 --duration 2m --warmup 20s --concurrency 256 --scrape-ratio 0.03 --stats http://10.0.0.10:1337/stats --out results-http.json
+bench/trakxbench --mode http --http 192.168.1.100:1337 --rate 1000 --duration 2m --warmup 20s --concurrency 256 --scrape-ratio 0.03 --stats http://192.168.1.100:1337/stats --out results-http.json
 ```
 
 ## Seeding the DB (optional but recommended)

bench/orchestrator_client.py

@@ -7,6 +7,7 @@
 import subprocess
 import sys
 import time
+from pathlib import Path
 from typing import List, Tuple, Optional
 
 
@@ -46,13 +47,44 @@ def run_trakxbench(cmd: List[str]) -> int:
     return proc.returncode
 
 
+def resolve_repo_root(value: Optional[str]) -> Path:
+    if value:
+        return Path(value).expanduser().resolve()
+    return Path(__file__).resolve().parents[1]
+
+
+def has_path_sep(value: str) -> bool:
+    return os.sep in value or (os.altsep and os.altsep in value)
+
+
+def ensure_trakxbench_binary(bin_arg: str, repo_root: Path) -> str:
+    if not has_path_sep(bin_arg) and not bin_arg.startswith("."):
+        bin_path = repo_root / bin_arg
+    else:
+        bin_path = Path(bin_arg)
+        if not bin_path.is_absolute():
+            bin_path = repo_root / bin_path
+
+    bin_path.parent.mkdir(parents=True, exist_ok=True)
+    build = subprocess.run(
+        ["go", "build", "-o", str(bin_path), "./cmd/trakxbench"],
+        cwd=str(repo_root),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+    if build.returncode != 0:
+        raise RuntimeError(f"failed to build trakxbench: {build.stdout.strip()}")
+    return str(bin_path)
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Trakx benchmark client orchestrator")
     parser.add_argument("--server", default="127.0.0.1:9077", help="server control address")
     parser.add_argument("--mode", choices=["udp", "http"], required=True, help="benchmark mode")
     parser.add_argument("--goroutines", nargs="+", default=["1,2,4,6,8,12,16,24,32,48,64"], help="goroutine sweep list")
     parser.add_argument("--other-routines", type=int, default=1, help="routines for the non-tested tracker")
-    parser.add_argument("--bench-bin", default="bench/trakxbench", help="path to trakxbench binary")
+    parser.add_argument("--bench-bin", default="bench/bin/trakxbench", help="path to trakxbench binary (auto-built every run)")
     parser.add_argument("--out-dir", default="", help="output directory (default: bench/results/<timestamp>-<mode>)")
     parser.add_argument("--udp", default="", help="udp target host:port (default: <server>:1337)")
     parser.add_argument("--http", default="", help="http target host:port (default: <server>:1337)")
@@ -73,8 +105,12 @@ def main() -> int:
     parser.add_argument("--label", default="", help="label prefix")
     parser.add_argument("--pause", action="store_true", help="pause for Enter between runs")
     parser.add_argument("--bench-args", default="", help="extra args passed to trakxbench")
+    parser.add_argument("--repo-root", default="", help="repo root (default: inferred)")
     args = parser.parse_args()
 
+    repo_root = resolve_repo_root(args.repo_root)
+    args.bench_bin = ensure_trakxbench_binary(args.bench_bin, repo_root)
+
     goroutines = parse_list(args.goroutines)
     if not goroutines:
         print("no goroutine values provided", file=sys.stderr)
@@ -96,7 +132,7 @@ def main() -> int:
 
     if not args.out_dir:
         ts = time.strftime("%Y%m%d-%H%M%S")
-        args.out_dir = f"bench/results/{ts}-{args.mode}"
+        args.out_dir = str((repo_root / "bench" / "results" / f"{ts}-{args.mode}").resolve())
     os.makedirs(args.out_dir, exist_ok=True)
 
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

bench/orchestrator_server.py

@@ -2,11 +2,11 @@
 import argparse
 import json
 import os
-import shutil
 import socket
 import subprocess
 import sys
 import time
+from pathlib import Path
 from typing import Optional, Tuple
 
 HEARTBEAT_REQUEST = bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0])
@@ -33,8 +33,33 @@ def recv_msg(fileobj) -> Optional[dict]:
     return json.loads(line.decode("utf-8"))
 
 
-def run_cmd(args, env=None) -> subprocess.CompletedProcess:
-    return subprocess.run(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+def run_cmd(args, env=None, cwd=None) -> subprocess.CompletedProcess:
+    return subprocess.run(args, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+
+
+def resolve_repo_root(value: Optional[str]) -> Path:
+    if value:
+        return Path(value).expanduser().resolve()
+    return Path(__file__).resolve().parents[1]
+
+
+def has_path_sep(value: str) -> bool:
+    return os.sep in value or (os.altsep and os.altsep in value)
+
+
+def ensure_trakx_binary(bin_arg: str, repo_root: Path) -> str:
+    if not has_path_sep(bin_arg) and not bin_arg.startswith("."):
+        bin_path = repo_root / bin_arg
+    else:
+        bin_path = Path(bin_arg)
+        if not bin_path.is_absolute():
+            bin_path = repo_root / bin_path
+
+    bin_path.parent.mkdir(parents=True, exist_ok=True)
+    build = run_cmd(["go", "build", "-o", str(bin_path), "./cli"], env=os.environ.copy(), cwd=str(repo_root))
+    if build.returncode != 0:
+        raise RuntimeError(f"failed to build trakx: {build.stdout.strip()}")
+    return str(bin_path)
 
 
 def wait_for_http(host: str, port: int, timeout: float) -> bool:
@@ -79,7 +104,7 @@ def clear_cache(cache_dir: str) -> None:
 def main() -> int:
     parser = argparse.ArgumentParser(description="Trakx benchmark server orchestrator")
     parser.add_argument("--listen", default="0.0.0.0:9077", help="control listen address")
-    parser.add_argument("--trakx-bin", default="trakx", help="path to trakx binary")
+    parser.add_argument("--trakx-bin", default="bench/bin/trakx", help="path to trakx binary (auto-built every run)")
     parser.add_argument("--config", default="bench/trakx.yaml", help="trakx config path")
     parser.add_argument("--cache-dir", default="/tmp/trakx-bench-cache", help="cache dir to reset between runs")
     parser.add_argument("--http-host", default="127.0.0.1", help="host for readiness check")
@@ -88,8 +113,14 @@ def main() -> int:
     parser.add_argument("--udp-port", type=int, default=1337, help="udp port for udp heartbeat")
     parser.add_argument("--ready-timeout", type=float, default=8.0, help="seconds to wait for readiness")
     parser.add_argument("--manual", action="store_true", help="pause for Enter before each start")
+    parser.add_argument("--repo-root", default="", help="repo root (default: inferred)")
     args = parser.parse_args()
 
+    repo_root = resolve_repo_root(args.repo_root)
+    args.trakx_bin = ensure_trakx_binary(args.trakx_bin, repo_root)
+    if not os.path.isabs(args.config):
+        args.config = str((repo_root / args.config).resolve())
+
     listen_host, listen_port = parse_hostport(args.listen)
     env_base = os.environ.copy()
     env_base["TRAKX_CACHE"] = args.cache_dir