publish five-trial Daytona leaderboards

Author: bingran-you

docs/article-suite-scoring.md

@@ -30,6 +30,13 @@ Each model runs the complete nine-task suite five independent times. For task
 trial scores. The task leaderboard also publishes their sample standard
 deviation.
 
+Task plots use native raw environment scores. If a task's verifier intentionally
+fails closed with `normalized_score = 0` and no observed raw score, the plot uses
+the frozen starter anchor as the raw-score equivalent. The trial artifact keeps
+`observed_raw_score = null` and records
+`raw_score_source = starter_anchor_equivalent_for_fail_closed_timeout`; this
+case is never presented as an observed environment return.
+
 ## Final normalized score
 
 The primary cross-task metric is the interquartile mean (IQM), implemented as

experiments/article_suite/README.md

@@ -157,3 +157,8 @@ matrix, removes the lowest 11 and highest 11 values, and averages the middle
 the five trial-level IQMs remain visible diagnostics. The additive display
 offset does not affect ranking or score gaps; raw IQM remains available in the
 JSON.
+
+For task-defined fail-closed verifier timeouts where no native raw score exists,
+the raw-score plot uses the frozen starter anchor as the equivalent of
+normalized `0`. Trial metadata preserves the missing observation and labels the
+display source explicitly.

leaderboard/ARTICLE_SUITE.md

@@ -2,15 +2,15 @@
 
 The first image contains the nine independently ranked task leaderboards. The second image contains the final cross-task ranking.
 
-The nine task panels use each environment's native raw score. The final scientific metric remains unbounded IQM.
+The nine task panels report five-trial mean ± sample standard deviation for each environment's native raw score.
 
-## Nine task leaderboards
+## Leaderboards
 
 ![Nine task-specific GenesisBench leaderboards](article_suite_task_leaderboards.png)
 
 ## Final normalized score
 
-The primary score is the interquartile mean (IQM): sort the nine task scores, remove the lowest two and highest two, then average the middle five. The image uses a plot-only positive display index equal to `IQM + 100`; raw IQM, arithmetic mean, and median remain in the JSON.
+The final score is RLiable-style IQM over the complete 5 × 9 trial-task score matrix: flatten all 45 normalized scores, trim the lowest 11 and highest 11, and average the middle 23. The displayed ± value is the sample standard deviation of the five per-trial IQMs. The image uses a plot-only positive display index equal to `IQM + 100`; raw metrics remain in the JSON.
 
 ![Final GenesisBench article-suite leaderboard](article_suite_final_leaderboard.png)
 

leaderboard/article_suite.json

@@ -1,29 +1,29 @@
 {
   "benchmark": "learning_beyond_gradients_article_suite",
-  "current_task_digest": "sha256:5cf3805d59b03cb9d0bebb6f154eab38beb655918b8321500d692f5010a00cb2",
-  "digest_compatibility_note": "Score-equivalent to the current task. The later change adds only a CI smoke config and documentation; the publication scoring config is unchanged.",
-  "finished_at": 1783956862.2606888,
+  "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974",
   "harness": "opencode",
   "model": {
-    "agent_idle_timeout_sec": 0,
+    "agent_idle_timeout_sec": 3600,
     "daytona_pty_readline_timeout_sec": 3600,
     "display_name": "Claude Opus 4.8",
     "id": "claude-opus-4.8",
     "model": "anthropic/claude-opus-4-8",
     "provider": "claude_oauth",
     "provider_reasoning_effort": "max"
   },
-  "normalized_score": 14.050353128,
+  "normalized_score": 1.820348169,
+  "normalized_score_stddev": 4.070422248601275,
   "provider_reasoning_effort": "max",
-  "source_run_id": "leaderboard/runs/article_suite/20260713T142919Z/claude-opus-4.8",
+  "raw_score": 2507.501036532837,
+  "raw_score_imputation_count": 4,
+  "raw_score_stddev": 142.40443086118393,
+  "source_run_ids": [
+    "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01/tasks-81ec8d98a4",
+    "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f",
+    "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03",
+    "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04",
+    "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05"
+  ],
   "task": "simulation_heuristics_ant_v1",
-  "task_digest": "sha256:9da0e00147cf66804e6c2fc17869606bea8f260850c5447989a8880eef940d45",
-  "token_usage": {
-    "final_input_tokens": 0.0,
-    "final_output_tokens": 0.0,
-    "input_tokens": 0.0,
-    "output_tokens": 0.0,
-    "total_tokens": 0.0
-  },
-  "tool_calls": 48
+  "trial_count": 5
 }

leaderboard/article_suite_final_leaderboard.png

@@ -1,141 +1,59 @@
 {
-  "hidden_nominal": {
-    "episodes": [
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.025934887526091188,
-        "policy_error": null,
-        "return": 2947.879693227018,
-        "seed": 101,
-        "terminated": false,
-        "truncated": true,
-        "variant": "nominal",
-        "x_position": 102.77571899817438,
-        "x_velocity": 1.8673987220154231
-      },
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.02599245577584952,
-        "policy_error": null,
-        "return": 3000.163186740716,
-        "seed": 211,
-        "terminated": false,
-        "truncated": true,
-        "variant": "nominal",
-        "x_position": 105.33522389819196,
-        "x_velocity": 2.1574081280110136
-      },
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.02603654988342896,
-        "policy_error": null,
-        "return": 2772.4088822782287,
-        "seed": 307,
-        "terminated": false,
-        "truncated": true,
-        "variant": "nominal",
-        "x_position": 93.7636515447122,
-        "x_velocity": 2.4267700657179603
-      },
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.02577067888341844,
-        "policy_error": null,
-        "return": 2990.8335078285263,
-        "seed": 401,
-        "terminated": false,
-        "truncated": true,
-        "variant": "nominal",
-        "x_position": 104.90485217547173,
-        "x_velocity": 2.078351224632513
-      }
-    ],
-    "fall_rate": 0.0,
-    "invalid_episode_rate": 0.0,
-    "max_return": 3000.163186740716,
-    "max_steps": 1000,
-    "mean_action_latency_ms": 0.025933643017197028,
-    "mean_return": 2927.821317518622,
-    "mean_x_position": 101.69486165413757,
-    "min_return": 2772.4088822782287,
-    "policy_path": "submitted_artifact",
-    "score": 2927.821317518622
-  },
-  "hidden_nominal_weight": 0.7,
-  "hidden_robustness": {
-    "episodes": [
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.025812971696723253,
-        "policy_error": null,
-        "return": 3166.4558227561265,
-        "seed": 503,
-        "terminated": false,
-        "truncated": true,
-        "variant": "light_low_friction",
-        "x_position": 113.74752702527248,
-        "x_velocity": 3.0189141499053562
-      },
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.02593717718264088,
-        "policy_error": null,
-        "return": 3177.1212044211834,
-        "seed": 601,
-        "terminated": false,
-        "truncated": true,
-        "variant": "light_low_friction",
-        "x_position": 114.37538391255823,
-        "x_velocity": 2.4397722759977114
-      },
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.026038738433271646,
-        "policy_error": null,
-        "return": 2839.9126867592117,
-        "seed": 503,
-        "terminated": false,
-        "truncated": true,
-        "variant": "heavy_weak_actuators",
-        "x_position": 97.73666401480078,
-        "x_velocity": 0.6070665236407535
-      },
-      {
-        "invalid_action": false,
-        "length": 1000,
-        "mean_action_latency_ms": 0.026594983239192516,
-        "policy_error": null,
-        "return": 2628.443172724322,
-        "seed": 601,
-        "terminated": false,
-        "truncated": true,
-        "variant": "heavy_weak_actuators",
-        "x_position": 87.07088591700881,
-        "x_velocity": 1.804739145616736
-      }
-    ],
-    "fall_rate": 0.0,
-    "invalid_episode_rate": 0.0,
-    "max_return": 3177.1212044211834,
-    "max_steps": 1000,
-    "mean_action_latency_ms": 0.026095967637957074,
-    "mean_return": 2952.983221665211,
-    "mean_x_position": 103.23261521741007,
-    "min_return": 2628.443172724322,
-    "policy_path": "submitted_artifact",
-    "score": 2952.983221665211
-  },
-  "hidden_robustness_weight": 0.3,
-  "normalized_score": 14.050353128,
+  "benchmark": "learning_beyond_gradients_article_suite",
+  "normalized_score": 1.820348169,
+  "normalized_score_stddev": 4.070422248601275,
+  "raw_score_imputation_count": 4,
   "reference_score": 5942.333244755838,
-  "score": 2935.3698887625987,
+  "score": 2507.501036532837,
+  "score_stddev": 142.40443086118393,
   "starter_score": 2443.815838992282,
-  "unique_policy_evaluations": 3
+  "task": "simulation_heuristics_ant_v1",
+  "trial_count": 5,
+  "trials": [
+    {
+      "normalized_score": 0.0,
+      "observed_raw_score": null,
+      "raw_score": 2443.815838992282,
+      "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout",
+      "score_path": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-01/score.json",
+      "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01/tasks-81ec8d98a4",
+      "trial": 1
+    },
+    {
+      "normalized_score": 9.101740845,
+      "observed_raw_score": 2762.241826695058,
+      "raw_score": 2762.241826695058,
+      "raw_score_source": "observed",
+      "score_path": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-02/score.json",
+      "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f",
+      "trial": 2
+    },
+    {
+      "normalized_score": 0.0,
+      "observed_raw_score": null,
+      "raw_score": 2443.815838992282,
+      "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout",
+      "score_path": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-03/score.json",
+      "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03",
+      "trial": 3
+    },
+    {
+      "normalized_score": 0.0,
+      "observed_raw_score": null,
+      "raw_score": 2443.815838992282,
+      "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout",
+      "score_path": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-04/score.json",
+      "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04",
+      "trial": 4
+    },
+    {
+      "normalized_score": 0.0,
+      "observed_raw_score": null,
+      "raw_score": 2443.815838992282,
+      "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout",
+      "score_path": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-05/score.json",
+      "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05",
+      "trial": 5
+    }
+  ]
 }

leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/metadata.json

@@ -0,0 +1,33 @@
+{
+  "benchmark": "learning_beyond_gradients_article_suite",
+  "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974",
+  "digest_compatibility_note": null,
+  "finished_at": 1784105005.326783,
+  "harness": "opencode",
+  "model": {
+    "agent_idle_timeout_sec": 3600,
+    "daytona_pty_readline_timeout_sec": 3600,
+    "display_name": "Claude Opus 4.8",
+    "id": "claude-opus-4.8",
+    "model": "anthropic/claude-opus-4-8",
+    "provider": "claude_oauth",
+    "provider_reasoning_effort": "max"
+  },
+  "normalized_score": 0.0,
+  "observed_raw_score": null,
+  "provider_reasoning_effort": "max",
+  "raw_score": 2443.815838992282,
+  "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout",
+  "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01/tasks-81ec8d98a4",
+  "task": "simulation_heuristics_ant_v1",
+  "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974",
+  "token_usage": {
+    "final_input_tokens": 0.0,
+    "final_output_tokens": 0.0,
+    "input_tokens": 0.0,
+    "output_tokens": 0.0,
+    "total_tokens": 0.0
+  },
+  "tool_calls": 52,
+  "trial": 1
+}

leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/score.json

@@ -0,0 +1,6 @@
+{
+  "normalized_score": 0.0,
+  "score": null,
+  "timeout_seconds": 3900,
+  "verifier_timeout": true
+}

leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-01/metadata.json

@@ -0,0 +1,33 @@
+{
+  "benchmark": "learning_beyond_gradients_article_suite",
+  "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974",
+  "digest_compatibility_note": null,
+  "finished_at": 1784103481.173677,
+  "harness": "opencode",
+  "model": {
+    "agent_idle_timeout_sec": 3600,
+    "daytona_pty_readline_timeout_sec": 3600,
+    "display_name": "Claude Opus 4.8",
+    "id": "claude-opus-4.8",
+    "model": "anthropic/claude-opus-4-8",
+    "provider": "claude_oauth",
+    "provider_reasoning_effort": "max"
+  },
+  "normalized_score": 9.101740845,
+  "observed_raw_score": 2762.241826695058,
+  "provider_reasoning_effort": "max",
+  "raw_score": 2762.241826695058,
+  "raw_score_source": "observed",
+  "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f",
+  "task": "simulation_heuristics_ant_v1",
+  "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974",
+  "token_usage": {
+    "final_input_tokens": 0.0,
+    "final_output_tokens": 0.0,
+    "input_tokens": 0.0,
+    "output_tokens": 0.0,
+    "total_tokens": 0.0
+  },
+  "tool_calls": 75,
+  "trial": 2
+}