Merge branch 'main' into pdl-250

Author: mandel

.github/workflows/tauri-cli.yml

@@ -51,7 +51,7 @@ jobs:
           # 3. `run` subcommand works with UI demos (json source)
           # demo8 uses a private model? not sure
           for i in ./src/demos/*.json
-          do if [[ $(basename $i) != "demo8.json" ]]; then pdl run $i; fi
+          do if [[ $(basename $i) != "demo8.json" ]] && [[  $(basename $i) != "demo9.json" ]]; then pdl run $i; fi
           done
 
       - name: Tear down xvfb

examples/gsm8k/gsm8.pdl

@@ -8,7 +8,7 @@
 #
 #
 
-description: Grade School Math example
+description: Grade School Math
 defs:
   # The Grade School Math Dataset
   ALL_TESTS:

examples/gsm8k/gsm8k-plan.pdl

@@ -0,0 +1,120 @@
+description: Grade School Math -- for every problem we generate a plan, then exectute and evaluate it.
+defs:
+  problems:
+    read: ./test.jsonl
+    parser: jsonl
+
+  MAX_ITERATIONS: 50
+
+  planning:
+    function:
+      problem: str
+    return:
+      text:
+      - >
+        Please generate a high-level plan for solving the following question. 
+        As the first step, just say what method and idea you will use to solve the question. 
+        You can reorganize the information in the question. Do not do the actual calculation. 
+        Keep your response concise and within 80 words. 
+        Question:
+      - ${ problem }
+      - "\nThe plan is:\n"
+      - model: ollama/granite3.2:8b
+  
+  solve:
+    function:
+      plan: str
+    return:
+      text:
+      - ${ plan }
+      - >
+        The plan looks good! Now, use real numbers and do the calculation. Please solve the question 
+        step-by-step according to the high-level plan. Give me the final answer. Make your response short.
+      - "\nThe answer is:\n"
+      - model: ollama/granite3.2:8b
+
+  extract_final_answer:
+    function:
+      solution: str
+    return:
+      lastOf:
+      - ${ solution }
+      - Extract the result from the above solution into a JSON object with field "result" and a float as value. Remove any dollar signs or other symbols.
+      - model: ollama/granite3.2:8b
+        parser: json
+        def: result
+        spec: { "result": float }
+        fallback:
+          data:
+            result: 0
+
+  compare_to_ground_truth:
+    function:
+      result: obj
+      truth: str
+    return:
+      lastOf:
+      - data: ${ truth }
+        parser:
+          regex: "(.|\n)*#### (?P<answer>([0-9])*)\n*"
+          spec:
+            answer: str
+        def: ground_truth
+      - if: ${ result.result|float == ground_truth.answer|float}
+        then:
+          1
+        else:
+          0
+
+text:
+- for:
+    problem: ${ problems }
+  repeat:
+    call: ${ planning }
+    args:
+      pdl_context: []
+      problem: ${ problem.question }
+  max_iterations: ${ MAX_ITERATIONS }
+  def: plans
+  join:
+    as: array
+
+- for:
+    plan: ${ plans }
+  repeat:
+    call: ${ solve }
+    args:
+      pdl_context: []
+      plan: ${ plan }
+  max_iterations: ${ MAX_ITERATIONS }
+  def: solutions
+  join:
+    as: array
+
+- for:
+    solution: ${ solutions }
+  repeat:
+    call: ${ extract_final_answer }
+    args:
+      pdl_context: []
+      solution: ${ solution }
+  max_iterations: ${ MAX_ITERATIONS }
+  def: results
+  join:
+    as: array
+
+- for:
+    result: ${ results }
+    problem: ${ problems[:MAX_ITERATIONS] }
+  repeat:
+    call: ${ compare_to_ground_truth }
+    args:
+      pdl_context: []
+      result: ${ result }
+      truth: ${ problem.answer }
+  max_iterations: ${ MAX_ITERATIONS }
+  def: stats
+  join:
+    as: array
+
+- "\nAccuracy: ${ stats|sum / MAX_ITERATIONS * 100}% "

examples/tools/calc.pdl

@@ -20,6 +20,8 @@ text:
   model: replicate/ibm-granite/granite-3.1-8b-instruct
   parser: json
   spec: [{ name: str, arguments: { expr: str }}]  
+  parameters:
+    drop_params: true  # This is needed because the model does not support structured decoding. It directs LiteLLM to ignore parameters sent for structured decoding.
 - "\n"
 - if: ${ actions[0].name == "calc" }
   then:

pdl-live-react/demos/run.sh

@@ -17,3 +17,6 @@ EOF
 pdl --trace "$T"/demo5.json "$TOP"/examples/talk/6-code-json.pdl
 pdl --trace "$T"/demo6.json "$UI"/demos/error.pdl || true
 pdl --trace "$T"/demo7.json "$TOP"/examples/talk/4-function.pdl
+# TODO demo8...
+exit
+pdl --stream none --trace "$T"/demo9.json "$TOP"/examples/gsm8k/gsm8.pdl