eval improvements, text block def, test for this

starpit · starpit · commit ebd6511645f6 · 2025-04-01T06:13:07.000-04:00
Signed-off-by: Nick Mitchell &lt;nickm@us.ibm.com&gt;
diff --git a/.github/workflows/rust-interpreter.yml b/.github/workflows/rust-interpreter.yml
@@ -0,0 +1,36 @@
+name: Rust Interpreter Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# cancel any prior runs for this workflow and this PR (or branch)
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  rust-interpreter:
+    name: Test Rust interpreter
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./pdl-live-react
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - name: Install dependencies
+        # sleep 2 to wait for ollama to be running... hack warning
+        run: |
+          npm ci & sudo apt update && sudo apt install -y libgtk-3-dev libwebkit2gtk-4.1-dev librsvg2-dev patchelf at-spi2-core &
+            (curl -fsSL https://ollama.com/install.sh | sudo -E sh && sleep 2)
+          wait
+          # todo: do this in rust
+          ollama pull granite3.2:2b
+      - name: Run interpreter tests
+        run: npm run test:interpreter
diff --git a/.github/workflows/tauri-cli.yml b/.github/workflows/tauri-cli.yml
@@ -45,7 +45,7 @@ jobs:
           for i in ./demos/beeai/*.py
           do pdl compile beeai $i -g -o /tmp/z.json && jq .description /tmp/z.json
           done
-          
+
       - name: Test pdl run against production build
         env:
           DISPLAY: :1
diff --git a/pdl-live-react/package.json b/pdl-live-react/package.json
@@ -17,7 +17,7 @@
     "test:bee": "until [ -f ./src-tauri/target/debug/pdl ]; do sleep 1; done; for i in ./demos/beeai/*.py; do ./src-tauri/target/debug/pdl compile beeai $i -g --output - | jq; done",
     "test:interpreter": "cd src-tauri && cargo test",
     "types": "(cd .. && python -m src.pdl.pdl --schema > src/pdl/pdl-schema.json) && json2ts ../src/pdl/pdl-schema.json src/pdl_ast.d.ts --unreachableDefinitions && npm run format",
-    "test": "concurrently -n 'quality,playwright,interpreter' 'npm run test:quality' 'npm run test:ui' 'npm run test:interpreter'",
+    "test": "concurrently -n 'quality,playwright' 'npm run test:quality' 'npm run test:ui'",
     "pdl": "./src-tauri/target/debug/pdl",
     "view": "npm run pdl view",
     "start": "npm run tauri dev"
diff --git a/pdl-live-react/src-tauri/src/compile/beeai.rs b/pdl-live-react/src-tauri/src/compile/beeai.rs
@@ -167,6 +167,7 @@ fn with_tools(
 
 fn call_tools(model: &String, parameters: &HashMap<String, Value>) -> PdlBlock {
     let repeat = PdlBlock::Text(PdlTextBlock {
+        def: None,
         defs: None,
         role: None,
         parser: None,
@@ -437,6 +438,7 @@ asyncio.run(invoke())
                     model_call.push(PdlBlock::Text(PdlTextBlock {
                         role: Some(Role::System),
                         text: vec![PdlBlock::String(instructions)],
+                        def: None,
                         defs: None,
                         parser: None,
                         description: Some("Model instructions".into()),
@@ -476,6 +478,7 @@ asyncio.run(invoke())
                     PdlBlock::Function(PdlFunctionBlock {
                         function: HashMap::new(),
                         return_: Box::new(PdlBlock::Text(PdlTextBlock {
+                            def: None,
                             defs: None,
                             role: None,
                             parser: None,
@@ -485,15 +488,15 @@ asyncio.run(invoke())
                     }),
                 );
                 PdlBlock::Text(PdlTextBlock {
+                    def: None,
                     defs: Some(defs),
                     role: None,
                     parser: None,
                     description: Some("Model call wrapper".to_string()),
-                    text: vec![PdlBlock::Call(PdlCallBlock {
-                        call: format!("${{ {} }}", closure_name),
-                        defs: None,
-                        args: None,
-                    })],
+                    text: vec![PdlBlock::Call(PdlCallBlock::new(format!(
+                        "${{ {} }}",
+                        closure_name
+                    )))],
                 })
             },
         )
@@ -504,6 +507,7 @@ asyncio.run(invoke())
         .collect::<Vec<_>>();
 
     let pdl: PdlBlock = PdlBlock::Text(PdlTextBlock {
+        def: None,
         defs: if tool_declarations.len() == 0 {
             None
         } else {
diff --git a/pdl-live-react/src-tauri/src/pdl/ast.rs b/pdl-live-react/src-tauri/src/pdl/ast.rs
@@ -55,6 +55,16 @@ pub struct PdlCallBlock {
     pub defs: Option<HashMap<String, PdlBlock>>,
 }
 
+impl PdlCallBlock {
+    pub fn new(call: String) -> Self {
+        PdlCallBlock {
+            call: call,
+            args: None,
+            defs: None,
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct PdlTextBlock {
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -66,11 +76,14 @@ pub struct PdlTextBlock {
     pub defs: Option<HashMap<String, PdlBlock>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub parser: Option<PdlParser>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub def: Option<String>,
 }
 
 impl PdlTextBlock {
     pub fn new(text: Vec<PdlBlock>) -> Self {
         PdlTextBlock {
+            def: None,
             defs: None,
             description: None,
             role: None,
@@ -79,6 +92,11 @@ impl PdlTextBlock {
         }
     }
 
+    pub fn def(&mut self, def: &str) -> &mut Self {
+        self.def = Some(def.into());
+        self
+    }
+
     pub fn description(&mut self, description: String) -> &mut Self {
         self.description = Some(description);
         self
diff --git a/pdl-live-react/src-tauri/src/pdl/interpreter.rs b/pdl-live-react/src-tauri/src/pdl/interpreter.rs
@@ -5,8 +5,8 @@ use ::std::error::Error;
 use ::std::fs::File;
 // use ::std::path::PathBuf;
 
-use minijinja::value::ValueKind;
 use minijinja::Environment;
+
 use ollama_rs::{
     generation::{
         chat::{request::ChatMessageRequest, ChatMessage, MessageRole},
@@ -15,37 +15,38 @@ use ollama_rs::{
     models::ModelOptions,
     Ollama,
 };
+
 use serde_json::{from_str, to_string, Value};
 use serde_norway::from_reader;
 use tokio::runtime::Runtime;
 
 use crate::pdl::ast::{
-    PdlBlock, PdlCallBlock, PdlModelBlock, PdlRepeatBlock, PdlTextBlock, PdlUsage, Role,
+    PdlBlock, PdlCallBlock, PdlModelBlock, PdlParser, PdlRepeatBlock, PdlTextBlock, PdlUsage, Role,
 };
 
 type Context = Vec<ChatMessage>;
-type Scope = HashMap<String, String>;
+type Scope = HashMap<String, Value>;
 type Interpretation = Result<(Context, PdlBlock), Box<dyn Error>>;
 
-struct Interpreter<'a> {
+struct Interpreter {
     // batch: u32,
     // role: Role,
     // cwd: Box<PathBuf>,
     // id_stack: Vec<String>,
-    jinja_env: Environment<'a>,
+    // jinja_env: Environment<'a>,
     rt: Runtime,
     scope: Vec<Scope>,
     debug: bool,
 }
 
-impl<'a> Interpreter<'a> {
+impl Interpreter {
     fn new() -> Self {
         Self {
             // batch: 0,
             // role: Role::User,
             // cwd: Box::new(current_dir().unwrap_or(PathBuf::from("/"))),
             // id_stack: vec![],
-            jinja_env: Environment::new(),
+            // jinja_env: Environment::new(),
             rt: Runtime::new().unwrap(),
             scope: vec![Scope::new()],
             debug: false,
@@ -66,26 +67,41 @@ impl<'a> Interpreter<'a> {
         Ok((messages, trace))
     }
 
-    fn eval(&self, expr: &String) -> Result<PdlBlock, Box<dyn Error>> {
-        let e = expr.replace("${ ", "").replace(" }", "");
-        let jexpr = self.jinja_env.compile_expression(&e)?;
-        let result = jexpr.eval(self.scope.last().unwrap_or(&HashMap::new()))?;
+    fn eval(&self, pdl_expr: &String) -> Result<PdlBlock, Box<dyn Error>> {
+        let expr = pdl_expr.replace("${ ", "{{").replace(" }", "}}"); // FIXME regexp
+        let mut env = Environment::new();
+        env.add_template(pdl_expr, expr.as_str())?;
+        let tmpl = env.get_template(pdl_expr)?;
+        let result = tmpl.render(self.scope.last().unwrap_or(&HashMap::new()))?;
+        eprintln!("Eval '{}' -> {}", &expr, &result);
+
+        match from_str(&result) {
+            Err(_) => {
+                eprintln!("Plain string {}", &result);
+                Ok(PdlBlock::String(result))
+            }
+            Ok(x) => Ok(x),
+        }
+        //let jexpr = self.jinja_env.compile_expression(&e)?;
+        //let result = jexpr.eval(self.scope.last().unwrap_or(&HashMap::new()))?;
         //dbg!("Eval '{}' -> {:?}", e, result);
 
-        match result.kind() {
-            ValueKind::String => Ok(from_str(&result.as_str().unwrap())?),
-            t => Err(Box::from(format!("Unexpected jinja result type {}", t))),
-        }
+        //match result.kind() {
+        //            ValueKind::String => Ok(from_str(&result.as_str().unwrap())?),
+        //            t => Err(Box::from(format!("Unexpected jinja result type: {} -> {}. {:?}", expr, t, result))),
+        //        }
     }
 
     fn run_string(&self, msg: &String, _context: Context) -> Interpretation {
+        let trace = self.eval(msg)?;
         if self.debug {
-            eprintln!("String {}", msg);
+            eprintln!("String {} -> {:?}", msg, trace);
         }
-        //TODO? self.eval(msg)
 
-        let messages = vec![ChatMessage::user(msg.clone())];
-        let trace = PdlBlock::String(msg.to_string());
+        let messages = vec![ChatMessage::user(match &trace {
+            PdlBlock::String(s) => s.clone(),
+            x => to_string(&x)?,
+        })];
 
         Ok((messages, trace))
     }
@@ -222,6 +238,12 @@ impl<'a> Interpreter<'a> {
         }
     }
 
+    fn parse_result(&self, parser: &PdlParser, result: &String) -> Result<Value, Box<dyn Error>> {
+        match parser {
+            PdlParser::Json => Ok(from_str(result)?),
+        }
+    }
+
     fn run_text(&mut self, block: &PdlTextBlock, context: Context) -> Interpretation {
         if self.debug {
             eprintln!(
@@ -237,10 +259,12 @@ impl<'a> Interpreter<'a> {
             Some(defs) => {
                 // this is all non-optimal
                 let mut scope: Scope = HashMap::from(cur_scope);
-                scope.extend(
-                    defs.iter()
-                        .map(|(var, def)| (var.clone(), to_string(def).unwrap())),
-                );
+                scope.extend(defs.iter().map(|(var, def)| {
+                    (
+                        var.clone(),
+                        from_str(to_string(def).unwrap().as_str()).unwrap(),
+                    )
+                }));
                 scope
             }
             None => cur_scope,
@@ -264,16 +288,28 @@ impl<'a> Interpreter<'a> {
         let mut trace = block.clone();
         trace.text = output_blocks;
 
+        let result_string = output_messages
+            .iter()
+            .map(|m| m.content.clone())
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        if let Some(def) = &block.def {
+            let result = if let Some(parser) = &block.parser {
+                self.parse_result(parser, &result_string)?
+            } else {
+                Value::from(result_string.clone()) // TODO
+            };
+
+            if let Some(scope) = self.scope.last_mut() {
+                eprintln!("DEF {} -> {}", def, result);
+                scope.insert(def.clone(), result);
+            }
+        }
+
         Ok((
             match &block.role {
-                Some(role) => vec![ChatMessage::new(
-                    self.to_ollama_role(role),
-                    output_messages
-                        .into_iter()
-                        .map(|m| m.content)
-                        .collect::<Vec<_>>()
-                        .join("\n"),
-                )],
+                Some(role) => vec![ChatMessage::new(self.to_ollama_role(role), result_string)],
                 None => output_messages,
             },
             PdlBlock::Text(trace),
diff --git a/pdl-live-react/src-tauri/src/pdl/interpreter_tests.rs b/pdl-live-react/src-tauri/src/pdl/interpreter_tests.rs
@@ -4,10 +4,7 @@ mod tests {
     use ::std::error::Error;
 
     use crate::pdl::{
-        ast::{
-            PdlBlock,      /*PdlCallBlock,*/
-            PdlModelBlock, /*PdlRepeatBlock,*/ /*PdlTextBlock,*/ /*, PdlUsage, Role,*/
-        },
+        ast::{PdlBlock, PdlModelBlock, PdlParser, PdlTextBlock},
         interpreter::run,
     };
 
@@ -55,4 +52,30 @@ mod tests {
         assert!(messages[1].content.contains("Hello!"));
         Ok(())
     }
+
+    #[test]
+    fn text_parser_json() -> Result<(), Box<dyn Error>> {
+        let json = "{\"key\": \"value\"}";
+        let program = PdlBlock::Text(
+            vec![
+                PdlBlock::Text(
+                    PdlTextBlock::new(vec![json.into()])
+                        .def(&"foo")
+                        .parser(PdlParser::Json)
+                        .build(),
+                ),
+                "${ foo.key }".into(),
+            ]
+            .into(),
+        );
+        println!("{}", serde_json::to_string(&program)?);
+
+        let (messages, _) = run(&program, false)?;
+        assert_eq!(messages.len(), 2);
+        assert_eq!(messages[0].role, MessageRole::User);
+        assert_eq!(messages[0].content, json);
+        assert_eq!(messages[1].role, MessageRole::User);
+        assert_eq!(messages[1].content, "value");
+        Ok(())
+    }
 }
