Changes to requirements feature to support ppdl (#1191)

Signed-off-by: Mandana Vaziri <[email protected]>

Author: vazirim

examples/requirements/email.pdl

@@ -12,7 +12,9 @@ defs:
           Does the following email end with Kind regards. Answer with a JSON object and a result field with value True or False only.
           Email: ${ response }
         parser: json
-      - ${ result.result }
+      - if: ${ result.result }
+        then: 0
+        else: -1000000
 
   fix:
     function:
@@ -36,7 +38,7 @@ text:
 - "Write an email to ${ name } using the notes following: ${ notes }"
 - model: ollama_chat/granite3.2:2b
   requirements:
-  - description: The email should end with Kind regards
+  - expect: The email should end with Kind regards
     evaluate:  ${ eval }
     transformContext: ${ fix }
   retry: 5

examples/requirements/gsm8k.pdl

@@ -0,0 +1,52 @@
+defs:
+  model: ollama_chat/granite3.3:8b
+  llm_as_judge: ollama_chat/gpt-oss:20b
+
+  problem: |
+    Carla is downloading a 200 GB file. Normally she can download 2 GB/minute, 
+    but 40% of the way through the download, Windows forces a restart to install updates, 
+    which takes 20 minutes. Then Carla has to restart the download from the beginning. 
+    How load does it take to download the file?
+
+  extract_answer:
+    function:
+      solution: string
+    return:
+      lastOf:
+      - ${ solution }
+      - Extract the result from the above solution into a JSON object with field "answer" and a float as value. Remove any dollar signs or other symbols.
+      - model: ${ model }
+        parser: json
+        spec: { "answer": number }
+
+  solve:
+    function:
+      problem: string
+    return:
+      defs:
+        solution:
+          text:
+          - ${ problem }
+          - "\n\n"
+          - model: ${ model }
+            parameters:
+              temperature: 0.1
+            requirements:
+            - expect: "The solution to this problem should be correct. Problem: ${ problem }"
+            retry: 1
+        answer_obj:
+          call: ${ extract_answer }
+          args:
+            solution: ${ solution }
+            pdl_context: []
+        debug:
+          lang: python
+          code: |
+            print(answer_obj)    
+            result = ""
+      data: ${ answer_obj.answer }
+
+call: ${ solve }
+args:
+  problem: ${ problem }
+  pdl_context: []

examples/requirements/gsm8k_short.pdl

@@ -0,0 +1,21 @@
+defs:
+  problem: |+
+    Carla is downloading a 200 GB file. Normally she can download 2 GB/minute, 
+    but 40% of the way through the download, Windows forces a restart to install updates, 
+    which takes 20 minutes. Then Carla has to restart the download from the beginning. 
+    How load does it take to download the file?
+
+
+lastOf:
+- ${ problem }
+- model: ollama_chat/granite3.3:8b
+  parameters:
+    temperature: 0.2
+- Extract the result from the above solution into a JSON object with field "answer" and a float as value. Remove any dollar signs or other symbols.
+- model: ollama_chat/granite3.3:8b
+  def: result
+  parser: json
+  spec: { "answer": number }
+  requirements:
+    - expect: "This solution to the following math problem is correct: ${ problem }"
+- ${ result.answer }

pyproject.toml

@@ -87,7 +87,7 @@ version_file = "src/pdl/_version.py"
 where = ["src"]
 
 [tool.setuptools.package-data]
-pdl = ["pdl-schema.json"]
+pdl = ["pdl-schema.json", "pdl_stlib.pdl"]
 
 [tool.pyright]
 include = ["src", "tests", "examples", "docs"]

src/pdl/pdl-schema.json

@@ -4504,7 +4504,7 @@
       "additionalProperties": false,
       "description": "Single requirement definition.",
       "properties": {
-        "description": {
+        "expect": {
           "anyOf": [
             {
               "$ref": "#/$defs/LocalizedExpression_TypeVar_"
@@ -4514,7 +4514,7 @@
               "type": "string"
             }
           ],
-          "title": "Description"
+          "title": "Expect"
         },
         "evaluate": {
           "anyOf": [
@@ -4531,6 +4531,7 @@
               "type": "null"
             }
           ],
+          "default": null,
           "title": "Evaluate"
         },
         "transformContext": {
@@ -4548,13 +4549,12 @@
               "type": "null"
             }
           ],
+          "default": null,
           "title": "Transformcontext"
         }
       },
       "required": [
-        "description",
-        "evaluate",
-        "transformContext"
+        "expect"
       ],
       "title": "RequirementType",
       "type": "object"

src/pdl/pdl_ast.py

@@ -338,13 +338,13 @@ class RequirementType(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    description: ExpressionType
+    expect: ExpressionType
     """English description of the requirement"""
 
-    evaluate: Optional[ExpressionType["FunctionBlock"]]
+    evaluate: Optional[ExpressionType["FunctionBlock"]] = None
     """Evaluation function for the requirement"""
 
-    transformContext: Optional[ExpressionType["FunctionBlock"]]
+    transformContext: Optional[ExpressionType["FunctionBlock"]] = None
     """Function to transform the context for the requirement"""
 
 

src/pdl/pdl_dumper.py

@@ -412,9 +412,11 @@ def usage_to_dict(usage: PdlUsage) -> dict:
 
 def requirement_to_dict(req: RequirementType, json_compatible: bool) -> dict:
     d: dict = {}
-    d["description"] = req.description
-    d["evaluate"] = expr_to_dict(req.evaluate, json_compatible)
-    d["transformContext"] = expr_to_dict(req.transformContext, json_compatible)
+    d["expect"] = req.expect
+    if req.evaluate is not None:
+        d["evaluate"] = expr_to_dict(req.evaluate, json_compatible)
+    if req.transformContext is not None:
+        d["transformContext"] = expr_to_dict(req.transformContext, json_compatible)
     return d
 
 

src/pdl/pdl_interpreter.py

@@ -304,8 +304,21 @@ def process_prog(
         PDLRuntimeError: If the program raises an error.
     """
     scope = empty_scope | scope
+
+    # Process stdlib
+    stdlib_file = Path(__file__).parent / "pdl_stdlib.pdl"
+    stdlib, _ = parse_file(stdlib_file)
+    _, _, stdlib_dict, _ = process_block(
+        state.with_yield_background(False).with_yield_result(False),
+        empty_scope,
+        stdlib.root,
+        loc,
+    )
+
+    stdlib_scope = scope | PdlDict({"stdlib": stdlib_dict})
+
     result, document, final_scope, trace = process_block(
-        state, scope, block=prog.root, loc=loc
+        state, stdlib_scope, block=prog.root, loc=loc
     )
     return result, document, final_scope, trace
 
@@ -441,7 +454,7 @@ def set_error_to_scope_for_retry(
     return scope
 
 
-def process_advanced_block(
+def process_advanced_block(  # noqa: C901
     state: InterpreterState,
     scope: ScopeType,
     block: AdvancedBlockType,
@@ -471,7 +484,7 @@ def process_advanced_block(
     return result, background, new_scope, trace
 
 
-def process_advance_block_retry(
+def process_advance_block_retry(  # noqa: C901
     state: InterpreterState,
     scope: ScopeType,
     block: AdvancedBlockType,
@@ -494,31 +507,39 @@ def process_advance_block_retry(
 
     max_retry = block.retry if block.retry else 0
     trial_total = max_retry + 1
-    for trial_idx in range(trial_total):
+    for trial_idx in range(trial_total):  # pylint: disable=too-many-nested-blocks
         try:
             result, background, new_scope, trace = process_block_body(
                 state, scope, block, loc
             )
             if block.requirements != []:
                 requirements_satisfied = True
                 for req in block.requirements:
-                    evalfn, _ = process_expr(scope, getattr(req, "evaluate"), loc)
-                    evaluation = evalfn(
-                        requirement=getattr(req, "description"), response=result
-                    )
-                    if evaluation.result() is False:
+                    evaluate = getattr(req, "evaluate", None)
+                    stdlib_dict: Any = scope["stdlib"]
+                    if evaluate is None:
+                        evaluate = stdlib_dict["requirements"]["evaluation"]
+                    evalfn: Any
+                    evalfn, _ = process_expr(scope, evaluate, loc)
+                    requirement, _ = process_expr(scope, getattr(req, "expect"), loc)
+                    evaluation = evalfn(requirement=requirement, response=result)
+                    if evaluation < -0.3:
                         requirements_satisfied = False
-                        transfn, _ = process_expr(
-                            scope, getattr(req, "transformContext"), loc
-                        )
+                        transform_context = getattr(req, "transformContext", None)
+                        if transform_context is None:
+                            transform_context = stdlib_dict["requirements"][
+                                "transformContext"
+                            ]
+                        transfn: Any
+                        transfn, _ = process_expr(scope, transform_context, loc)
                         new_context = transfn(
                             pdl_context=scope["pdl_context"],
-                            requirement=getattr(req, "description"),
+                            requirement=requirement,
                             response=result,
                         )
-                        scope = scope | {"pdl_context": new_context}
+                        if trial_idx < max_retry:
+                            scope = scope | {"pdl_context": new_context}
                 if requirements_satisfied is False:
-                    print("\nTrying again!")
                     continue
 
             result = lazy_apply(id_with_set_first_use_nanos(block.pdl__timing), result)
@@ -680,7 +701,7 @@ def process_block_body(
                 yield_result(result.result(), block.kind)
             if state.yield_background:
                 yield_background(background)
-        case TextBlock():  # HERE
+        case TextBlock():
             result, background, scope, trace = process_blocks_of(
                 block,
                 "text",

src/pdl/pdl_stdlib.pdl

@@ -0,0 +1,69 @@
+
+defs: 
+  reward:
+    function:
+      response: 
+    return:
+      defs:
+        top_logprobs: ${ response.choices[0].logprobs.content[0].top_logprobs}
+      lastOf:
+      - for: 
+          tp: ${ top_logprobs }
+        repeat: 
+          match: ${ tp.token }
+          with:
+          - case: "Yes"
+            then:
+              data: ${ tp.logprob }
+              def: lp_y
+          - case: "No"
+            then: 
+              data: ${ tp.logprob }
+              def: lp_n
+      - lang: python  
+        code: |
+          import math 
+          result = math.log(math.exp(lp_y) / (math.exp(lp_y) + math.exp(lp_n)))
+
+  requirements:
+    object:
+      evaluation:
+        function:
+          requirement: string
+          response: string
+          llm_as_judge: {optional: string}
+        return:
+          lastOf:
+          - model: ${ llm_as_judge | default('watsonx/meta-llama/llama-3-3-70b-instruct') }
+            def: evaluation
+            input: |
+                  Is the following requirement satisfied in the solution below? Requirement: ${ requirement }
+                  ${ response }
+
+                  Respond with only 'Yes' or 'No'.
+            modelResponse: out
+            parameters:
+              temperature: 0
+              logprobs: true
+              top_logprobs: 5
+          - ${ reward(out) }
+          
+          
+      transformContext:
+        function:
+          requirement: string
+          response: string
+          model: {optional: string}
+        return: 
+          lastOf:
+          - model: ${ model | default('ollama_chat/granite3.3:8b') }
+            input: |
+              The following requirement is not satisfied, what instruction can be added to get the correct answer?
+              Requirement: ${ requirement }
+              Answer with only the instruction.
+          - ${ pdl_context }
+
+
+
+  
+

tests/test_examples_run.yaml

@@ -34,8 +34,11 @@ skip:
   - examples/optimizer/optimized_grammar_correction.pdl
   - examples/optimizer/eval_levenshtein.pdl
   - examples/requirements/email.pdl
+  - examples/requirements/gsm8k.pdl
+  - examples/requirements/gsm8k_short.pdl
   - examples/skeleton-of-thought/tips.pdl
   - examples/tutorial/sdk/lib.pdl
+  - src/pdl/pdl_stdlib.pdl
 with_inputs:
   examples/tutorial/programs/chatbot.pdl:
     stdin: |