Merge pull request #19 from allenai/cross-bench-eval-v2

Cross-benchmark evaluation: DimSpec system + LIBERO/CALVIN reproduction fixes

Author: MilkClouds

.claude/skills/run-evaluation/SKILL.md

@@ -249,6 +249,19 @@ vla-eval merge -c configs/libero_spatial.yaml -o results/libero_spatial.json
 vla-eval test --all
 ```
 
+### Parallel evaluations of different models
+
+Shard result files are named by benchmark + shard ID (e.g.
+`LIBEROBenchmark_libero_spatial_shard0of10.json`). If two evals use the
+same benchmark config, shard count, and output directory, they will
+collide. The orchestrator prevents this with a file lock — the second
+eval will **fail immediately** with `FileExistsError` rather than
+silently overwriting results.
+
+If you hit this error, either:
+- Use **different output directories** (modify `output_dir` in the config), or
+- Use **different shard counts** (e.g. `--num-shards 10` vs `--num-shards 8`).
+
 ### Troubleshooting
 
 | Problem | Solution |

.dockerignore

@@ -7,4 +7,5 @@
 !docker/calvin_validation_data/
 !docker/init_states/
 !docker/*_entrypoint.sh
+!docker/*.patch
 

configs/model_servers/cogact/cogact.yaml

@@ -2,7 +2,6 @@
 # Weight: CogACT/CogACT-Base (HuggingFace)
 # Output: 16 actions × 7-DoF (future_action_window_size=15 → 16 steps)
 #
-# Usage: vla-eval serve --config configs/model_servers/cogact/cogact.yaml
 #
 # Available checkpoints:
 #   CogACT/CogACT-Small  (action_model_type: DiT-S)

configs/model_servers/groot/groot.yaml

@@ -2,7 +2,6 @@
 # Weight: nvidia/GR00T-N1.6-3B (HuggingFace)
 # Action chunking enabled (chunk_size=16).
 #
-# Usage: vla-eval serve --config configs/model_servers/groot/groot.yaml
 #
 # Available embodiment_tags for foundation model: GR1, ROBOCASA_PANDA_OMRON, BEHAVIOR_R1_PRO
 # Fine-tuned checkpoints may support: LIBERO_PANDA, OXE_GOOGLE, OXE_WIDOWX, UNITREE_G1

configs/model_servers/groot/simpler_google_robot.yaml

@@ -0,0 +1,7 @@
+# GR00T N1.6 — SimplerEnv Google Robot (official NVIDIA checkpoint)
+script: "src/vla_eval/model_servers/groot.py"
+args:
+  model_path: nvidia/GR00T-N1.6-fractal
+  embodiment_tag: OXE_GOOGLE
+  chunk_size: 16
+  port: 8000

configs/model_servers/groot/simpler_widowx.yaml

@@ -0,0 +1,9 @@
+# GR00T N1.6 — SimplerEnv WidowX (official NVIDIA checkpoint)
+script: "src/vla_eval/model_servers/groot.py"
+args:
+  model_path: nvidia/GR00T-N1.6-bridge
+  embodiment_tag: OXE_WIDOWX
+  image_resolution: 256
+  chunk_size: 16
+  bridge_rotation: true
+  port: 8000

configs/model_servers/oft/libero_spatial.yaml

@@ -2,7 +2,6 @@
 # Weight: moojink/openvla-7b-oft-finetuned-libero-spatial (HuggingFace)
 # Action chunking enabled (parallel decoding, 26× faster than OpenVLA).
 #
-# Usage: vla-eval serve --config configs/model_servers/oft/libero_spatial.yaml
 extends: _base.yaml
 args:
   pretrained_checkpoint: moojink/openvla-7b-oft-finetuned-libero-spatial

configs/model_servers/openvla/openvla.yaml

@@ -2,7 +2,6 @@
 # Weight: openvla/openvla-7b (HuggingFace)
 # No action chunking (chunk_size=1), returns 7-dim actions.
 #
-# Usage: vla-eval serve --config configs/model_servers/openvla/openvla.yaml
 #
 # For task-specific unnormalization, set unnorm_key to match the
 # training dataset (e.g. "bridge_orig" for BridgeData V2).

configs/model_servers/pi0/libero.yaml

@@ -1,7 +1,6 @@
 # π₀.5 model server — LIBERO (direct OpenPI inference)
 # Loads the policy checkpoint directly; no external server needed.
 #
-# Usage: vla-eval serve --config configs/model_servers/pi0/libero.yaml
 #
 # Available config_name values (see openpi repo):
 #   pi0_fast_libero, pi05_libero, pi0_fast_droid, pi05_droid, ...

configs/model_servers/pi0/libero_fast.yaml

@@ -1,6 +1,5 @@
 # π₀-FAST model server — LIBERO (direct OpenPI inference)
 #
-# Usage: vla-eval serve --config configs/model_servers/pi0/libero_fast.yaml
 #
 # Pi0-FAST uses FAST tokenizer — a different, lower-performing variant than Pi0.5.
 # For Pi0.5 (96.85% on LIBERO), use libero.yaml instead.