Add optimizer test PDL

Signed-off-by: Claudio Spiess <[email protected]>

Author: claudiosv

src/pdl/optimize/pdl_optimizer.py

@@ -161,7 +161,8 @@ def sample_candidates(
 
         if (
             "prompt_pattern" in self.config.variables
-            and "cot" in self.config.variables["prompt_pattern"]
+            and "cot" in self.config.variables.get("prompt_pattern", [])
+            and 0 in self.config.variables.get("num_demonstrations", [])
         ):
             cot_candidate = {
                 k: self.sample_random_index(v) for k, v in self.config.variables.items()
@@ -183,9 +184,8 @@ def sample_candidates(
                 k: self.sample_random_index(v) for k, v in self.config.variables.items()
             }
             if (
-                "num_demonstrations" in variable_instance
-                and variable_instance["num_demonstrations"] == 0
-                and variable_instance["prompt_pattern"] == "cot"
+                variable_instance.get("num_demonstrations") == 0
+                and variable_instance.get("prompt_pattern") == "cot"
             ):
                 if variable_instance["prompt_pattern"] in zero_shots_seen:
                     continue

src/pdl/pdl_dumper.py

@@ -416,9 +416,7 @@ def build_exclude(obj: Any, regex: re.Pattern[str]) -> Any:
         for k, v in obj.items():
             nested = build_exclude(v, regex)
             if nested:
-                out[k] = nested
-                if k == "root":
-                    out |= out["root"]
+                out[k] = {**nested, "__all__": nested}
 
         return out or None
 
@@ -440,6 +438,7 @@ def dump_program_exclude_internals(program: Program) -> str:
     # pattern for internal pdl__ fields
     regex = re.compile(r"^pdl__.*")
     exclude = build_exclude(program, regex)
+    exclude |= exclude.get("root", {})
     return dump_program(program, exclude=exclude)
 
 

tests/data/optimizer_gsm8k.pdl

@@ -0,0 +1,76 @@
+description: Demo of template
+defs:
+  cot:
+    import: ../../contrib/prompt_library/CoT
+  react:
+    import: ../../contrib/prompt_library/ReAct
+  rewoo:
+    import: ../../contrib/prompt_library/ReWoo
+  tools:
+    import: ../../contrib/prompt_library/tools
+
+  chain_of_thought:
+    function:
+      question: str
+      model: str
+      examples:
+        { list: { obj: { question: str, reasoning: str, answer: str } } }
+    return:
+      lastOf:
+        - call: ${ cot.fewshot_cot }
+          args:
+            examples: ${ examples }
+        - "Question: ${ question }\n"
+        - "Answer: Let's think step by step. "
+        - model: ${ model }
+          def: answer
+          parameters:
+            max_tokens: 1024
+            temperature: 0
+            stop:
+              - "<|endoftext|>"
+              - "Question:"
+            include_stop_sequence: false
+            mock_response: "144"
+        - data:
+            answer: ${ answer|trim }
+match: ${ prompt_pattern }
+with:
+  # CoT
+  - case: cot
+    then:
+      text:
+        - "Answer the questions to the best of your abilities.\n\n"
+        - call: ${ chain_of_thought }
+          def: ANSWER
+          contribute: []
+          args:
+            examples: "${ demonstrations }"
+            question: "${ question|trim }"
+            model: "${ model }"
+        - "\nThe answer is ${ ANSWER.answer|trim }"
+
+  # ReAct
+  - case: react
+    then:
+      text:
+        call: ${ react.react }
+        args:
+          task: "Question: ${ question|trim }"
+          model: ${ model }
+          tool_schema: ${ tools.tool_schema }
+          tools: ${ tools.tools }
+          trajectories: ${ demonstrations }
+
+  # ReWOO
+  - case: rewoo
+    then:
+      text:
+        call: ${ rewoo.rewoo }
+        args:
+          task: ${ question|trim }
+          model: ${ model }
+          tool_schema: ${ tools.tool_schema }
+          tools: ${ tools.tools }
+          trajectories: ${ demonstrations }
+          show_plans: false