fix: begin phasing in Metadata (common defs, etc. attrs) into rust AST

Rather than duplicating this "metadata" logic -- the stuff common to all non-literal blocks. This just starts the migration. The main trick is to use the serde `flatten` capability, so that we can maintain a separate Metadata struct in Rust, but have it flattened into the enclosing object for serde.

Signed-off-by: Nick Mitchell <[email protected]>

Author: starpit

pdl-live-react/src-tauri/src/compile/beeai.rs

@@ -12,9 +12,9 @@ use serde_json::{from_reader, json, to_string, Map, Value};
 use tempfile::Builder;
 
 use crate::pdl::ast::{
-    ArrayBlock, CallBlock, FunctionBlock, ListOrString, MessageBlock, ModelBlock, ObjectBlock,
-    PdlBaseType, PdlBlock, PdlOptionalType, PdlParser, PdlType, PythonCodeBlock, RepeatBlock, Role,
-    TextBlock,
+    ArrayBlock, CallBlock, FunctionBlock, ListOrString, MessageBlock, Metadata, ModelBlock,
+    ObjectBlock, PdlBaseType, PdlBlock, PdlOptionalType, PdlParser, PdlType, PythonCodeBlock,
+    RepeatBlock, Role, TextBlock,
 };
 use crate::pdl::pip::pip_install_if_needed;
 use crate::pdl::requirements::BEEAI_FRAMEWORK;
@@ -192,7 +192,7 @@ fn with_tools(
 fn call_tools(model: &String, parameters: &HashMap<String, Value>) -> PdlBlock {
     let repeat = PdlBlock::Text(TextBlock {
         def: None,
-        defs: None,
+        metadata: None,
         role: None,
         parser: None,
         description: Some("Calling tool ${ tool.function.name }".to_string()),
@@ -206,11 +206,13 @@ fn call_tools(model: &String, parameters: &HashMap<String, Value>) -> PdlBlock {
                         name: Some("${ tool.function.name }".to_string()),
                         tool_call_id: Some("${ tool.id }".to_string()),
                         content: Box::new(PdlBlock::Call(CallBlock {
-                            defs: json_loads(
-                                &"args",
-                                &"pdl__args",
-                                &"${ tool.function.arguments }",
-                            ),
+                            metadata: Some(Metadata {
+                                defs: json_loads(
+                                    &"args",
+                                    &"pdl__args",
+                                    &"${ tool.function.arguments }",
+                                ),
+                            }),
                             call: "${ pdl__tools[tool.function.name] }".to_string(), // look up tool in tool_declarations def (see below)
                             args: Some("${ args }".into()), // invoke with arguments as specified by the model
                         })),
@@ -464,7 +466,7 @@ asyncio.run(invoke())
                         role: Some(Role::System),
                         text: vec![PdlBlock::String(instructions)],
                         def: None,
-                        defs: None,
+                        metadata: None,
                         parser: None,
                         description: Some("Model instructions".into()),
                     }));
@@ -504,7 +506,7 @@ asyncio.run(invoke())
                         function: HashMap::new(),
                         return_: Box::new(PdlBlock::Text(TextBlock {
                             def: None,
-                            defs: None,
+                            metadata: None,
                             role: None,
                             parser: None,
                             description: Some(format!("Model call {}", &model)),
@@ -514,7 +516,7 @@ asyncio.run(invoke())
                 );
                 PdlBlock::Text(TextBlock {
                     def: None,
-                    defs: Some(defs),
+                    metadata: Some(Metadata { defs: Some(defs) }),
                     role: None,
                     parser: None,
                     description: Some("Model call wrapper".to_string()),
@@ -533,7 +535,7 @@ asyncio.run(invoke())
 
     let pdl: PdlBlock = PdlBlock::Text(TextBlock {
         def: None,
-        defs: if tool_declarations.len() == 0 {
+        metadata: if tool_declarations.len() == 0 {
             None
         } else {
             let mut m = indexmap::IndexMap::new();
@@ -543,7 +545,7 @@ asyncio.run(invoke())
                     object: tool_declarations,
                 }),
             );
-            Some(m)
+            Some(Metadata { defs: Some(m) })
         },
         description: Some(bee.workflow.workflow.name),
         role: None,

pdl-live-react/src-tauri/src/pdl/ast.rs

@@ -51,6 +51,13 @@ pub enum PdlType {
     Object(HashMap<String, PdlType>),
 }
 
+/// Common metadata of blocks
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Metadata {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub defs: Option<IndexMap<String, PdlBlock>>,
+}
+
 /// Call a function
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(tag = "kind", rename = "call")]
@@ -62,16 +69,17 @@ pub struct CallBlock {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub args: Option<Value>,
 
+    #[serde(flatten)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub defs: Option<IndexMap<String, PdlBlock>>,
+    pub metadata: Option<Metadata>,
 }
 
 impl CallBlock {
     pub fn new(call: String) -> Self {
         CallBlock {
             call: call,
             args: None,
-            defs: None,
+            metadata: None,
         }
     }
 }
@@ -81,7 +89,7 @@ pub trait SequencingBlock {
     fn description(&self) -> &Option<String>;
     fn role(&self) -> &Option<Role>;
     fn def(&self) -> &Option<String>;
-    fn defs(&self) -> &Option<IndexMap<String, PdlBlock>>;
+    fn metadata(&self) -> &Option<Metadata>;
     fn items(&self) -> &Vec<PdlBlock>;
     fn with_items(&self, items: Vec<PdlBlock>) -> Self;
     fn parser(&self) -> &Option<PdlParser>;
@@ -104,8 +112,9 @@ pub struct LastOfBlock {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub role: Option<Role>,
 
+    #[serde(flatten)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub defs: Option<IndexMap<String, PdlBlock>>,
+    pub metadata: Option<Metadata>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub parser: Option<PdlParser>,
@@ -126,8 +135,8 @@ impl SequencingBlock for LastOfBlock {
     fn def(&self) -> &Option<String> {
         return &self.def;
     }
-    fn defs(&self) -> &Option<IndexMap<String, PdlBlock>> {
-        &self.defs
+    fn metadata(&self) -> &Option<Metadata> {
+        &self.metadata
     }
     fn items(&self) -> &Vec<PdlBlock> {
         &self.last_of
@@ -171,8 +180,9 @@ pub struct TextBlock {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub role: Option<Role>,
 
+    #[serde(flatten)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub defs: Option<IndexMap<String, PdlBlock>>,
+    pub metadata: Option<Metadata>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub parser: Option<PdlParser>,
@@ -193,8 +203,8 @@ impl SequencingBlock for TextBlock {
     fn def(&self) -> &Option<String> {
         return &self.def;
     }
-    fn defs(&self) -> &Option<IndexMap<String, PdlBlock>> {
-        &self.defs
+    fn metadata(&self) -> &Option<Metadata> {
+        &self.metadata
     }
     fn items(&self) -> &Vec<PdlBlock> {
         &self.text
@@ -228,7 +238,7 @@ impl TextBlock {
     pub fn new(text: Vec<PdlBlock>) -> Self {
         TextBlock {
             def: None,
-            defs: None,
+            metadata: None,
             description: None,
             role: None,
             parser: None,
@@ -525,8 +535,9 @@ pub struct IfBlock {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub else_: Option<Box<PdlBlock>>,
 
+    #[serde(flatten)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub defs: Option<IndexMap<String, PdlBlock>>,
+    pub metadata: Option<Metadata>,
 }
 
 /// Return the array of values computed by each block of the list of blocks

pdl-live-react/src-tauri/src/pdl/extract.rs

@@ -1,4 +1,4 @@
-use crate::pdl::ast::PdlBlock;
+use crate::pdl::ast::{Metadata, PdlBlock};
 
 /// Extract models referenced by the programs
 pub fn extract_models(program: &PdlBlock) -> Vec<String> {
@@ -35,7 +35,10 @@ fn extract_values_iter(program: &PdlBlock, field: &str, values: &mut Vec<String>
             b.text
                 .iter()
                 .for_each(|p| extract_values_iter(p, field, values));
-            if let Some(defs) = &b.defs {
+            if let Some(Metadata {
+                defs: Some(defs), ..
+            }) = &b.metadata
+            {
                 defs.values()
                     .for_each(|p| extract_values_iter(p, field, values));
             }
@@ -44,7 +47,10 @@ fn extract_values_iter(program: &PdlBlock, field: &str, values: &mut Vec<String>
             b.last_of
                 .iter()
                 .for_each(|p| extract_values_iter(p, field, values));
-            if let Some(defs) = &b.defs {
+            if let Some(Metadata {
+                defs: Some(defs), ..
+            }) = &b.metadata
+            {
                 defs.values()
                     .for_each(|p| extract_values_iter(p, field, values));
             }
@@ -54,11 +60,19 @@ fn extract_values_iter(program: &PdlBlock, field: &str, values: &mut Vec<String>
             if let Some(else_) = &b.else_ {
                 extract_values_iter(else_, field, values);
             }
-            if let Some(defs) = &b.defs {
+            if let Some(Metadata {
+                defs: Some(defs), ..
+            }) = &b.metadata
+            {
                 defs.values()
                     .for_each(|p| extract_values_iter(p, field, values));
             }
         }
+        PdlBlock::Empty(b) => {
+            b.defs
+                .values()
+                .for_each(|p| extract_values_iter(p, field, values));
+        }
         PdlBlock::Object(b) => b
             .object
             .values()

pdl-live-react/src-tauri/src/pdl/interpreter.rs

@@ -1,4 +1,3 @@
-// use ::std::cell::LazyCell;
 use ::std::collections::HashMap;
 use ::std::error::Error;
 use ::std::path::PathBuf;
@@ -91,7 +90,6 @@ impl State {
 
 struct Interpreter<'a> {
     // batch: u32,
-    // role: Role,
     // id_stack: Vec<String>,
     options: RunOptions<'a>,
     jinja_env: Environment<'a>,
@@ -110,7 +108,6 @@ impl<'a> Interpreter<'a> {
 
         Self {
             // batch: 0,
-            // role: Role::User,
             // id_stack: vec![],
             jinja_env: jinja_env,
             options: options,
@@ -415,8 +412,9 @@ impl<'a> Interpreter<'a> {
             eprintln!("Empty");
         }
 
-        let trace = block.clone();
         self.process_defs(&Some(block.defs.clone()), state).await?;
+
+        let trace = block.clone();
         Ok((
             PdlResult::Dict(state.scope.clone()),
             vec![],
@@ -430,7 +428,9 @@ impl<'a> Interpreter<'a> {
             eprintln!("If {:?}({:?})", block.condition, block.then);
         }
 
-        self.process_defs(&block.defs, state).await?;
+        if let Some(meta) = &block.metadata {
+            self.process_defs(&meta.defs, state).await?;
+        }
 
         let cond = match &block.condition {
             StringOrBoolean::Boolean(b) => PdlResult::Bool(*b),
@@ -893,7 +893,9 @@ impl<'a> Interpreter<'a> {
         let mut output_messages = vec![];
         let mut output_blocks = vec![];
 
-        self.process_defs(block.defs(), state).await?;
+        if let Some(meta) = block.metadata() {
+            self.process_defs(&meta.defs, state).await?;
+        }
 
         // here is where we iterate over the sequence items
         let mut iter = block.items().iter();