analysis notebooks

Author: zhudotexe

analysis/commitment_metrics.ipynb

@@ -12,8 +12,8 @@
    "metadata": {
     "collapsed": true,
     "ExecuteTime": {
-     "end_time": "2024-12-02T14:41:25.972082Z",
-     "start_time": "2024-12-02T14:41:25.749960Z"
+     "end_time": "2025-12-03T18:34:12.116429Z",
+     "start_time": "2025-12-03T18:34:11.848275Z"
     }
    },
    "source": [
@@ -38,8 +38,8 @@
   {
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2024-12-02T14:41:25.974662Z",
-     "start_time": "2024-12-02T14:41:25.973144Z"
+     "end_time": "2025-12-03T18:34:12.121834Z",
+     "start_time": "2025-12-03T18:34:12.120497Z"
     }
    },
    "cell_type": "code",
@@ -56,8 +56,8 @@
   {
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2024-12-02T14:41:25.979243Z",
-     "start_time": "2024-12-02T14:41:25.975169Z"
+     "end_time": "2025-12-03T18:34:14.487901Z",
+     "start_time": "2025-12-03T18:34:14.484170Z"
     }
    },
    "cell_type": "code",
@@ -126,8 +126,8 @@
   {
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2024-12-02T14:41:27.035496Z",
-     "start_time": "2024-12-02T14:41:25.980586Z"
+     "end_time": "2025-12-03T18:34:16.768570Z",
+     "start_time": "2025-12-03T18:34:16.405943Z"
     }
    },
    "cell_type": "code",
@@ -166,27 +166,13 @@
       "Avg prompt tokens del: 28171.0 ($0.0140855)\n",
       "Avg output tokens del: 509.6818181818182 ($0.0007645227272727273)\n",
       "N: 308\n",
-      "Total cost: $9.200377000000001\n",
-      "========== /Users/andrew/Desktop/Code/_penn/redel-experiments/experiments/travelplanner/validation/small-leaf ==========\n",
-      "Avg prompt tokens root: 7912.888888888889 ($0.039564444444444444)\n",
-      "Avg output tokens root: 1018.7333333333333 ($0.015281)\n",
-      "Avg prompt tokens del: 9192.044444444444 ($0.0045960222222222215)\n",
-      "Avg output tokens del: 1121.0833333333333 ($0.001681625)\n",
-      "N: 180\n",
-      "Total cost: $11.0021565\n",
-      "========== /Users/andrew/Desktop/Code/_penn/redel-experiments/experiments/webarena/test/small-leaf ==========\n",
-      "Avg prompt tokens root: 55456.268 ($0.27728134)\n",
-      "Avg output tokens root: 171.884 ($0.0025782599999999998)\n",
-      "Avg prompt tokens del: 222370.916 ($0.111185458)\n",
-      "Avg output tokens del: 740.764 ($0.001111146)\n",
-      "N: 250\n",
-      "Total cost: $98.039051\n"
+      "Total cost: $9.200377000000001\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "(98.039051, 250)"
+       "(0, 0)"
       ]
      },
      "execution_count": 4,

sandbox/foqa_scores_to_csv.py analysis/foqa_scores_to_csv.pysandbox/foqa_scores_to_csv.py renamed to analysis/foqa_scores_to_csv.py

@@ -0,0 +1,41 @@
+family,system,delivery rate,cs micro,cs macro,hard micro,hard macro,hard micro fix,hard macro fix,final
+openai,full,100.000,69.931,3.889,9.524,7.778,80.714,66.667,2.778
+openai,root-fc,100.000,66.250,0.556,21.905,14.444,49.524,31.667,0.000
+openai,baseline,99.444,66.389,0.556,23.333,13.333,52.143,37.778,0.000
+openai,small-leaf,100.000,68.264,0.556,2.143,1.111,74.762,55.000,0.556
+openai,small-all,99.444,55.000,0.000,0.000,0.000,88.095,73.333,0.000
+openai,small-baseline,91.111,49.861,0.000,0.238,0.000,53.333,35.000,0.000
+openai,short-context,99.444,61.667,1.667,7.857,6.667,,,1.111
+openai,short-baseline,98.889,64.861,2.222,25.476,13.889,,,0.556
+cohere-hf,full,84.444,52.153,1.111,1.190,1.111,,,0.000
+cohere-hf,root-fc,92.778,64.931,3.333,7.619,2.222,,,0.000
+cohere-hf,baseline,93.333,65.069,5.556,5.476,2.778,,,0.556
+cohere-hf,small-leaf,,,,,,,,
+cohere-hf,small-all,53.889,30.764,0.556,0.952,1.111,,,0.556
+cohere-hf,small-baseline,97.222,60.556,1.667,1.905,1.667,,,0.000
+cohere-hf,short-context,76.111,46.181,0.000,0.238,0.556,,,0.000
+cohere-hf,short-baseline,95.000,63.889,5.000,6.429,4.444,,,2.222
+glm,full,100.000,72.778,0.000,0.000,0.000,,,0.000
+glm,root-fc,100.000,65.417,6.111,28.571,25.000,,,2.778
+glm,baseline,100.000,67.917,8.889,39.048,27.778,,,5.556
+glm,small-leaf,,,,,,,,
+glm,small-all,,,,,,,,
+glm,small-baseline,,,,,,,,
+glm,short-context,100.000,72.778,0.000,0.000,0.000,,,0.000
+glm,short-baseline,100.000,56.250,5.556,17.619,12.222,,,2.778
+gpt-oss,full,100.000,57.569,0.000,0.000,0.000,,,0.000
+gpt-oss,root-fc,100.000,46.042,0.000,0.714,0.556,,,0.000
+gpt-oss,baseline,100.000,45.764,0.000,1.667,0.556,,,0.000
+gpt-oss,small-leaf,100.000,54.722,0.000,0.000,0.000,,,0.000
+gpt-oss,small-all,100.000,61.181,0.000,0.000,0.000,,,0.000
+gpt-oss,small-baseline,100.000,47.778,0.000,6.429,3.333,,,0.000
+gpt-oss,short-context,100.000,60.139,0.000,0.000,0.000,,,0.000
+gpt-oss,short-baseline,61.667,32.361,0.000,0.714,0.000,,,0.000
+qwen3,full,100.000,63.125,0.000,0.000,0.000,,,0.000
+qwen3,root-fc,100.000,57.083,3.889,17.381,7.222,,,2.222
+qwen3,baseline,100.000,58.958,1.111,15.238,6.111,,,0.556
+qwen3,small-leaf,100.000,63.056,0.000,0.000,0.000,,,0.000
+qwen3,small-all,100.000,55.764,0.000,0.000,0.000,,,0.000
+qwen3,small-baseline,100.000,56.597,0.000,5.476,3.333,,,0.000
+qwen3,short-context,100.000,62.500,0.000,0.000,0.000,,,0.000
+qwen3,short-baseline,100.000,56.944,0.000,11.667,5.556,,,0.000