PDL Optimizer

.pre-commit-config.yaml

@@ -56,6 +56,7 @@ repos:
     rev: 'v1.15.0'
     hooks:
       - id: mypy
+        args: [--explicit-package-bases]
         verbose: true
         additional_dependencies: ['types-PyYAML']
   # type check the Python code using pyright

contrib/prompt_library/ReAct.pdl

@@ -10,26 +10,28 @@ defs:
             trajectory: ${ trajectory }
           repeat:
             text:
-              - def: type
-                text: ${ trajectory.keys()|first }
-                contribute: []
-              - if: ${ type == 'question'}
-                then: |
-                  Question: ${ trajectory[type]|trim }
-              - if: ${ type == 'task'}
-                then: |
-                  Task: ${ trajectory[type]|trim }
-              - if: ${ type == 'thought'}
-                then: |
-                  Tho: ${ trajectory[type]|trim }
-              - if: ${ type == 'action'}
-                then: |
-                  Act: ${ trajectory[type]|trim }
-              - if: ${ type == 'observation'}
-                then: |
-                  Obs: ${ trajectory[type]|trim }
-              - if: ${ type not in ['question', 'task', 'thought', 'action', 'observation'] }
-                then: "${ type }: ${ trajectory[type]|trim }"
+              - defs:
+                  type:
+                    text: ${ trajectory.keys()|first }
+              - match: ${ type }
+                with:
+                  - case: question
+                    then: |
+                      Question: ${ trajectory[type]|trim }
+                  - case: task
+                    then: |
+                      Task: ${ trajectory[type]|trim }
+                  - case: thought
+                    then: |
+                      Tho: ${ trajectory[type]|trim }
+                  - case: action
+                    then: |
+                      Act: ${ trajectory[type]|trim }
+                  - case: observation
+                    then: |
+                      Obs: ${ trajectory[type]|trim }
+                  - if: ${ type not in ['question', 'task', 'thought', 'action', 'observation'] }
+                    then: "${ type }: ${ trajectory[type]|trim }"
         - "\n"
 
   react:
@@ -101,13 +103,12 @@ defs:
                   then:
                     text:
                       - "\nObs: "
-                      - if: ${ action.name in tools }
+                      - if: ${ action.name.lower() in tools }
                         then:
-                          call: ${ tools[action.name] }
+                          call: ${ tools[action.name.lower()] }
                           args:
                             arguments: ${ action.arguments }
                         else: "Invalid action. Valid actions are ${ tool_names[:-1]|join(', ') }, and ${ tool_names[-1] }."
-                      # - "\n"
                 else:
                   def: exit
                   contribute: []

contrib/prompt_library/ReWoo.pdl

@@ -20,24 +20,29 @@ defs:
                     text: ${ trajectory.keys()|first }
                   content:
                     text: ${ trajectory.values()|first }
-              - if: ${ type in ['task', 'question'] }
-                then: |-
-                  Task: ${ content|trim }
-              - if: ${ type == 'thought'}
-                then: |-
+              - match: ${ type }
+                with:
+                  - case: task
+                    then: |-
+                      Task: ${ content|trim }
+                  - case: question
+                    then: |-
+                      Task: ${ content|trim }
+                  - case: thought
+                    then: |-
 
-                  Plan: ${ content|trim }
-              - if: ${ type == 'action'}
-                then:
-                  text:
-                    - " #E${ i } = ${ content|trim }"
-                    - defs:
-                        i:
-                          data: ${ i+1 }
-              - if: ${ type == 'observation'}
-                then: ""
-              - if: ${ type not in ['question', 'task', 'thought', 'action', 'observation'] }
-                then: "${ type }: ${ content|trim }\n"
+                      Plan: ${ content|trim }
+                  - case: action
+                    then:
+                      text:
+                        - " #E${ i } = ${ content|trim }"
+                        - defs:
+                            i:
+                              data: ${ i+1 }
+                  - case: observation
+                    then: ""
+                  - if: ${ type not in ['question', 'task', 'thought', 'action', 'observation'] }
+                    then: "${ type }: ${ content|trim }\n"
         - "\n"
 
   rewoo:
@@ -120,9 +125,9 @@ defs:
                               ACTION_RAW = ACTION_RAW.replace(k, v)
                           result = ACTION_RAW
                       tool_output:
-                        if: ${ ACTION.name in tools }
+                        if: ${ ACTION.name.lower() in tools }
                         then:
-                          call: ${ tools[ACTION.name] }
+                          call: ${ tools[ACTION.name.lower()] }
                           args:
                             arguments: ${ ACTION.arguments }
                         else: "Invalid action. Valid actions are ${ tools.keys() }"

docs/autopdl.md

@@ -0,0 +1,4 @@
+
+# AutoPDL Tutorial
+
+The following sections show how to use the AutoPDL optimizer to produce optimized PDL programs for specific tasks.

examples/optimizer/fever.pdl

@@ -0,0 +1,63 @@
+description: Demo of ReAct template fever
+defs:
+  cot:
+    import: ../../contrib/prompt_library/CoT
+  react:
+    import: ../../contrib/prompt_library/ReAct
+  rewoo:
+    import: ../../contrib/prompt_library/ReWoo
+  tools:
+    import: ../../contrib/prompt_library/tools
+
+  search_tools:
+    data:
+      - name: Search
+        description: Search Wikipedia for a summary
+        parameters:
+          type: object
+          properties:
+            topic:
+              type: string
+              description: The topic of interest
+          required:
+            - topic
+
+  task: |-
+    Task: On June 2017, the following claim was made: ${ claim }
+    Q: Was this claim true or false?
+match: ${ prompt_pattern }
+with:
+  # CoT
+  - case: cot
+    then:
+      text:
+        call: ${ cot.chain_of_thought }
+        args:
+          examples: "${ demonstrations }"
+          question: "${ task }"
+          model: "${ model }"
+
+  # ReAct
+  - case: react
+    then:
+      text:
+        call: ${ react.react }
+        args:
+          task: ${ task }
+          model: ${ model }
+          tool_schema: ${ search_tools }
+          tools: ${ tools.tools }
+          trajectories: ${ demonstrations }
+
+  # ReWOO
+  - case: rewoo
+    then:
+      text:
+        call: ${ rewoo.rewoo }
+        args:
+          task: ${ task }
+          model: ${ model }
+          tool_schema: ${ search_tools }
+          tools: ${ tools.tools }
+          trajectories: ${ demonstrations }
+          show_plans: false

examples/optimizer/fever_evaluator.py

@@ -0,0 +1,86 @@
+from typing import Any
+
+from pdl.optimize.optimizer_evaluator import OptimizerEvaluator
+from pdl.pdl_ast import ScopeType
+from pdl.pdl_interpreter import empty_scope
+
+
+class FEVEREvaluator(OptimizerEvaluator):
+    def __init__(
+        self,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.answer_key = "label"
+
+    def get_scope(self) -> ScopeType:
+        demo_var = self.config.demonstrations_variable_name
+
+        scope = {}
+
+        for k in self.config.variables:
+            if k in self.candidate:
+                scope[k] = self.candidate[k]
+
+        match self.candidate.get("prompt_pattern", None):
+            case "cot":
+                scope[demo_var] = [
+                    {
+                        "question": q["claim"],
+                        "reasoning": q["cot"].strip(),
+                        "answer": str(q[self.answer_key]).lower(),
+                    }
+                    for q in self.candidate[demo_var]
+                ]
+            case "react":
+                scope[demo_var] = [
+                    [
+                        {key: value}
+                        for key, value in zip(
+                            q["traj_keys"],
+                            q["traj_values"],
+                            strict=True,
+                        )
+                    ]
+                    for q in self.candidate[demo_var]
+                ]
+            case "rewoo":
+                scope[demo_var] = [
+                    [
+                        {key: value}
+                        for key, value in zip(
+                            q["rewoo_traj_keys"],
+                            q["rewoo_traj_values"],
+                            strict=True,
+                        )
+                    ]
+                    for q in self.candidate[demo_var]
+                ]
+            case _:
+                pass
+
+        scope["claim"] = self.example["claim"]
+        return empty_scope | scope
+
+    def extract_answer(self, document: str) -> bool | None:
+        #  "SUPPORTS", and otherwise with "REFUTES"
+        response = document.splitlines()[-1].lower()
+        if "```" in response:
+            response = response.split("```")[1]
+        supports = "true" in response
+        refutes = "false" in response
+
+        if (supports and refutes) or not (supports or refutes):
+            return None  # ""
+
+        if supports:
+            return True  # "true"
+
+        if refutes:
+            return False  # "false"
+
+        return None
+
+    def answer_correct(self, document: str, answer: Any, truth: Any) -> bool:
+        return answer == truth or document.lower().endswith(str(truth).lower())

examples/optimizer/gsm8k.pdl

@@ -0,0 +1,50 @@
+description: Demo of template
+defs:
+  cot:
+    import: ../../contrib/prompt_library/CoT
+  react:
+    import: ../../contrib/prompt_library/ReAct
+  rewoo:
+    import: ../../contrib/prompt_library/ReWoo
+  tools:
+    import: ../../contrib/prompt_library/tools
+match: ${ prompt_pattern }
+with:
+  # CoT
+  - case: cot
+    then:
+      text:
+        - "Answer the questions to the best of your abilities.\n\n"
+        - call: ${ cot.chain_of_thought }
+          def: ANSWER
+          contribute: []
+          args:
+            examples: "${ demonstrations }"
+            question: "${ question|trim }"
+            model: "${ model }"
+        - "\nThe answer is ${ ANSWER.answer|trim }"
+
+  # ReAct
+  - case: react
+    then:
+      text:
+        call: ${ react.react }
+        args:
+          task: "Question: ${ question|trim }"
+          model: ${ model }
+          tool_schema: ${ tools.tool_schema }
+          tools: ${ tools.tools }
+          trajectories: ${ demonstrations }
+
+  # ReWOO
+  - case: rewoo
+    then:
+      text:
+        call: ${ rewoo.rewoo }
+        args:
+          task: ${ question|trim }
+          model: ${ model }
+          tool_schema: ${ tools.tool_schema }
+          tools: ${ tools.tools }
+          trajectories: ${ demonstrations }
+          show_plans: false