Fix implicit unused analysis (#158)

Author: mandel

examples/hello/hello-type.pdl

@@ -9,7 +9,8 @@ text:
     language: str
   spec: int
   return:
-    - "\nTranslate the sentence '${ sentence }' to ${ language }\n"
+    lastOf:
+    - "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
     - model: replicate/ibm-granite/granite-3.0-8b-instruct
       parameters:
         stop_sequences: "\n"

examples/talk/4-function.pdl

@@ -5,11 +5,12 @@ text:
     sentence: str
     language: str
   return:
-  - "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
-  - model: replicate/ibm-granite/granite-3.0-8b-instruct
-    parameters:
-      stop_sequences: "\n"
-      temperature: 0
+    lastOf:
+    - "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
+    - model: replicate/ibm-granite/granite-3.0-8b-instruct
+      parameters:
+        stop_sequences: "\n"
+        temperature: 0
 - call: translate
   args:
     sentence: I love Paris!

examples/talk/8-tools.pdl

@@ -31,6 +31,7 @@ text:
 - "\n"
 - if: ${ action.name == "Calc" }
   then:
-  - "Obs: "
-  - lang: python
-    code: result = ${ action.arguments.expr }
+    text:
+    - "Obs: "
+    - lang: python
+      code: result = ${ action.arguments.expr }

examples/tools/calc.pdl

@@ -31,6 +31,7 @@ text:
 - "\n"
 - if: ${ action.name == "Calc" }
   then:
-  - "Obs: "
-  - lang: python
-    code: result = ${ action.arguments.expr }
+    text:
+    - "Obs: "
+    - lang: python
+      code: result = ${ action.arguments.expr }

examples/tutorial/function_definition.pdl

@@ -5,7 +5,8 @@ text:
     sentence: str
     language: str
   return:
-  - "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
+  - text: "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
+    contribute: [context]
   - model: replicate/ibm-granite/granite-3.0-8b-instruct
     parameters:
       stop_sequences: "\n"

examples/tutorial/grouping_definitions.pdl

@@ -5,7 +5,8 @@ defs:
       sentence: str
       language: str
     return:
-    - "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
+    - text: "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
+      contribute: [context]
     - model: replicate/ibm-granite/granite-3.0-8b-instruct
       parameters:
         stop_sequences: "\n"

examples/tutorial/muting_block_output.pdl

@@ -5,7 +5,8 @@ defs:
       sentence: str
       language: str
     return:
-    - "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
+    - text: "\nTranslate the sentence '${ sentence }' to ${ language }.\n"
+      contribute: [context]
     - model: replicate/ibm-granite/granite-3.0-8b-instruct
       parameters:
         stop_sequences: "\n"

src/pdl/pdl_analysis.py

@@ -6,6 +6,7 @@
     AdvancedBlockType,
     ArrayBlock,
     Block,
+    BlocksType,
     BlockType,
     CallBlock,
     CodeBlock,
@@ -29,41 +30,66 @@
     TextBlock,
 )
 from .pdl_ast_utils import iter_block_children
+from .pdl_dumper import blocks_to_dict, dump_yaml
 
 
 @dataclass
 class UnusedConfig:
     implicit_ignore: bool
+    implicit_lastOf: bool  # pylint: disable=invalid-name
 
     def with_implicit_ignore(self, b):
-        return UnusedConfig(implicit_ignore=b)
+        return UnusedConfig(implicit_ignore=b, implicit_lastOf=self.implicit_lastOf)
+
+    def with_implicit_lastOf(self, b):  # pylint: disable=invalid-name
+        return UnusedConfig(implicit_ignore=self.implicit_ignore, implicit_lastOf=b)
 
 
 _DISPLAY_UNUSED_HINT = True
 
 
 def unused_warning(block: BlockType):
-    global _DISPLAY_UNUSED_HINT  # pylint: disable= global-statement
-    print(f"Warning: the result of block `{block}` is not used.", file=sys.stderr)
+    global _DISPLAY_UNUSED_HINT  # pylint: disable=global-statement
+    print(
+        f"Warning: the result of block `{dump_yaml(blocks_to_dict(block, json_compatible=True))}` is not used.",
+        file=sys.stderr,
+    )
     if _DISPLAY_UNUSED_HINT:
         _DISPLAY_UNUSED_HINT = False
         print(
-            "         You might want to use a `text` block around the list or explicitly ignore the result with `contribute: [context]`.",
+            "         You might want to use a `text` block around the list or explicitly ignore the result with a `lastOf` block or `contribute: [context]`.",
             file=sys.stderr,
         )
 
 
 def unused_program(prog: Program) -> None:
-    state = UnusedConfig(implicit_ignore=False)
-    unused_advanced_block(state, LastOfBlock(lastOf=prog.root))
+    try:
+        state = UnusedConfig(implicit_ignore=False, implicit_lastOf=True)
+        unused_blocks(state, prog.root)
+    except Exception as exc:
+        print(f"Unexpected error in implicit ignored analysis: {exc}")
+
+
+def unused_blocks(state: UnusedConfig, blocks: BlocksType) -> None:
+    if not isinstance(blocks, str) and isinstance(blocks, Sequence):
+        if state.implicit_lastOf:
+            state_with_ignore = state.with_implicit_ignore(True)
+            for b in blocks[:-1]:
+                unused_block(state_with_ignore, b)
+            unused_block(state, blocks[-1])
+        else:
+            for b in blocks:
+                unused_block(state, b)
+    else:
+        unused_block(state, blocks)
 
 
-def unused_block(state, block: BlockType) -> None:
-    if not isinstance(block, Block):
+def unused_block(state, blocks: BlockType) -> None:
+    if isinstance(blocks, Block):
+        unused_advanced_block(state, blocks)
+    else:
         if state.implicit_ignore:
-            unused_warning(block)
-        return
-    unused_advanced_block(state, block)
+            unused_warning(blocks)
 
 
 def unused_advanced_block(state: UnusedConfig, block: AdvancedBlockType) -> None:
@@ -72,34 +98,54 @@ def unused_advanced_block(state: UnusedConfig, block: AdvancedBlockType) -> None
     if ContributeTarget.RESULT not in block.contribute:
         state = state.with_implicit_ignore(False)
     match block:
-        case LastOfBlock():
-            if not isinstance(block.lastOf, str) and isinstance(block.lastOf, Sequence):
-                state_with_ignore = state.with_implicit_ignore(True)
-                for b in block.lastOf[:-1]:
-                    unused_block(state_with_ignore, b)
-                unused_block(state, block.lastOf[-1])
-            else:
-                unused_block(state, block.lastOf)
-        # Leaf blocks without side effects
-        case DataBlock() | FunctionBlock() | GetBlock() | ModelBlock() | ReadBlock():
+        case ArrayBlock() | LastOfBlock() | ObjectBlock() | TextBlock():
             if state.implicit_ignore:
                 unused_warning(block)
-            return
-        # Leaf blocks with side effects
-        case CallBlock() | CodeBlock() | EmptyBlock() | ErrorBlock():
-            return
-        # Non-leaf blocks
+            state = state.with_implicit_lastOf(False)
+            iter_block_children(
+                (lambda blocks: used_blocks(state, blocks)),
+                block,
+            )
+        # Leaf blocks
         case (
-            ArrayBlock()
-            | ForBlock()
-            | IfBlock()
-            | IncludeBlock()
+            DataBlock()
+            | FunctionBlock()
+            | GetBlock()
             | MessageBlock()
-            | ObjectBlock()
-            | RepeatBlock()
-            | RepeatUntilBlock()
-            | TextBlock()
+            | ModelBlock()
+            | CallBlock()
+            | CodeBlock()
+            | ReadBlock()
         ):
-            iter_block_children((lambda b: unused_block(state, b)), block)
+            if state.implicit_ignore:
+                unused_warning(block)
+            state = state.with_implicit_ignore(False).with_implicit_lastOf(True)
+            iter_block_children(
+                (lambda blocks: unused_blocks(state, blocks)),
+                block,
+            )
+        case EmptyBlock():
+            state = state.with_implicit_ignore(False).with_implicit_lastOf(True)
+            iter_block_children(
+                (lambda blocks: unused_blocks(state, blocks)),
+                block,
+            )
+        # Non-leaf blocks
+        case IfBlock() | IncludeBlock():
+            state = state.with_implicit_lastOf(True)
+            iter_block_children((lambda blocks: unused_blocks(state, blocks)), block)
+        # Loops blocks
+        case ForBlock() | RepeatBlock() | RepeatUntilBlock():
+            iter_block_children((lambda blocks: unused_blocks(state, blocks)), block)
+        case ErrorBlock():
+            pass
         case _:
             assert False
+
+
+def used_blocks(state: UnusedConfig, blocks: BlocksType) -> None:
+    if not isinstance(blocks, str) and isinstance(blocks, Sequence):
+        for block in blocks:
+            unused_block(state.with_implicit_ignore(False), block)
+    else:
+        unused_block(state.with_implicit_ignore(False), blocks)

src/pdl/pdl_ast_utils.py

@@ -32,69 +32,69 @@
 )
 
 
-def iter_block_children(f: Callable[[BlockType], None], block: BlockType) -> None:
+def iter_block_children(f: Callable[[BlocksType], None], block: BlockType) -> None:
     if not isinstance(block, Block):
         return
     for blocks in block.defs.values():
-        iter_blocks(f, blocks)
+        f(blocks)
     match block:
         case FunctionBlock():
             if block.returns is not None:
-                iter_blocks(f, block.returns)
+                f(block.returns)
         case CallBlock():
             if block.trace is not None:
-                iter_blocks(f, block.trace)
+                f(block.trace)
         case ModelBlock():
             if block.input is not None:
-                iter_blocks(f, block.input)
+                f(block.input)
             if block.trace is not None:
-                iter_blocks(f, block.trace)
+                f(block.trace)
         case CodeBlock():
-            iter_blocks(f, block.code)
+            f(block.code)
         case GetBlock():
             pass
         case DataBlock():
             pass
         case TextBlock():
-            iter_blocks(f, block.text)
+            f(block.text)
         case LastOfBlock():
-            iter_blocks(f, block.lastOf)
+            f(block.lastOf)
         case ArrayBlock():
-            iter_blocks(f, block.array)
+            f(block.array)
         case ObjectBlock():
             if isinstance(block.object, dict):
                 for blocks in block.object.values():
-                    iter_blocks(f, blocks)
+                    f(blocks)
             else:
-                iter_blocks(f, block.object)
+                f(block.object)
         case MessageBlock():
-            iter_blocks(f, block.content)
+            f(block.content)
         case IfBlock():
-            iter_blocks(f, block.then)
+            f(block.then)
             if block.elses is not None:
-                iter_blocks(f, block.elses)
+                f(block.elses)
         case RepeatBlock():
-            iter_blocks(f, block.repeat)
+            f(block.repeat)
             if block.trace is not None:
                 for trace in block.trace:
-                    iter_blocks(f, trace)
+                    f(trace)
         case RepeatUntilBlock():
-            iter_blocks(f, block.repeat)
+            f(block.repeat)
             if block.trace is not None:
                 for trace in block.trace:
-                    iter_blocks(f, trace)
+                    f(trace)
         case ForBlock():
-            iter_blocks(f, block.repeat)
+            f(block.repeat)
             if block.trace is not None:
                 for trace in block.trace:
-                    iter_blocks(f, trace)
+                    f(trace)
         case ErrorBlock():
-            iter_blocks(f, block.program)
+            f(block.program)
         case ReadBlock():
             pass
         case IncludeBlock():
             if block.trace is not None:
-                iter_blocks(f, block.trace)
+                f(block.trace)
         case EmptyBlock():
             pass
         case _:
@@ -105,17 +105,9 @@ def iter_block_children(f: Callable[[BlockType], None], block: BlockType) -> Non
         case "json" | "yaml" | RegexParser():
             pass
         case PdlParser():
-            iter_blocks(f, block.parser.pdl)
+            f(block.parser.pdl)
     if block.fallback is not None:
-        iter_blocks(f, block.fallback)
-
-
-def iter_blocks(f: Callable[[BlockType], None], blocks: BlocksType) -> None:
-    if not isinstance(blocks, str) and isinstance(blocks, Sequence):
-        for block in blocks:
-            f(block)
-    else:
-        f(blocks)
+        f(block.fallback)
 
 
 class MappedFunctions: