fix: rust ast support for gsm8k, including jsonl parser

The program may not run 100% yet, but it parses now.

Signed-off-by: Nick Mitchell <[email protected]>

Author: starpit

pdl-live-react/src-tauri/src/compile/beeai.rs

@@ -270,6 +270,7 @@ fn call_tools(model: &String, parameters: &HashMap<String, Value>) -> PdlBlock {
         body: Repeat(RepeatBlock {
             for_: for_,
             repeat: Box::new(repeat),
+            join: None,
         }),
     })
 }

pdl-live-react/src-tauri/src/pdl/ast.rs

@@ -19,14 +19,40 @@ pub enum Role {
     Tool,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all_fields(serialize = "lowercase"))]
+pub enum RegexMode {
+    Search,
+    Match,
+    Fullmatch,
+    Split,
+    Findall,
+}
+
+/// A regular expression parser
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct RegexParser {
+    /// Regular expression to parse the value
+    pub regex: String,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mode: Option<RegexMode>,
+
+    /// Expected type of the parsed value
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub spec: Option<Value>,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub enum PdlParser {
     #[serde(rename = "json")]
     Json,
-    /*#[serde(rename = "jsonl")]
-    Jsonl,*/
+    #[serde(rename = "jsonl")]
+    Jsonl,
     #[serde(rename = "yaml")]
     Yaml,
+    #[serde(untagged)]
+    Regex(RegexParser),
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -89,15 +115,26 @@ impl Timing {
 #[serde(default)]
 #[builder(setter(into, strip_option), default)]
 pub struct Metadata {
+    /// Documentation associated to the block
     #[serde(skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
 
+    /// Set of definitions executed before the execution of the block
     #[serde(skip_serializing_if = "Option::is_none")]
     pub defs: Option<IndexMap<String, PdlBlock>>,
 
+    /// Name of the variable used to store the result of the execution of the block
     #[serde(skip_serializing_if = "Option::is_none")]
     pub def: Option<String>,
 
+    /// Indicate if the block contributes to the result and background context
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub contribute: Option<Vec<String>>, // TODO
+
+    /// Type specification of the result of the block
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub spec: Option<Value>,
+
     #[serde(rename = "pdl__id", skip_serializing_if = "Option::is_none")]
     pub pdl_id: Option<String>,
 
@@ -284,11 +321,9 @@ pub struct ModelBlock {
     pub input: Option<Box<PdlBlock>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub platform: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(rename = "modelResponse")]
+    #[serde(rename = "modelResponse", skip_serializing_if = "Option::is_none")]
     pub model_response: Option<String>,
-    #[serde(rename = "pdl__usage")]
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "pdl__usage", skip_serializing_if = "Option::is_none")]
     pub pdl_usage: Option<PdlUsage>,
 
     /// The result of evaluating the `input` field (if given)
@@ -307,6 +342,37 @@ pub enum ListOrString {
     List(Vec<Value>),
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum IterationType {
+    #[serde(rename = "lastOf")]
+    LastOf,
+    #[serde(rename = "array")]
+    Array,
+    #[serde(rename = "object")]
+    Object,
+    #[serde(rename = "text")]
+    Text,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct JoinAs {
+    #[serde(rename = "as")]
+    as_: IterationType,
+}
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct JoinAsWith {
+    #[serde(rename = "as")]
+    as_: IterationType,
+    with: String,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum JoinType {
+    AsWith(JoinAsWith),
+    As(JoinAs),
+}
+
 /// Repeat the execution of a block.
 ///
 /// For loop example:
@@ -326,6 +392,10 @@ pub struct RepeatBlock {
 
     /// Body of the loop
     pub repeat: Box<PdlBlock>,
+
+    /// Define how to combine the result of each iteration
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub join: Option<JoinType>,
 }
 
 /// Create a message

pdl-live-react/src-tauri/src/pdl/interpreter.rs

@@ -1189,8 +1189,14 @@ impl<'a> Interpreter<'a> {
 
     fn parse_result(&self, parser: &PdlParser, result: &String) -> Result<PdlResult, PdlError> {
         match parser {
-            PdlParser::Json => from_str(result).map_err(|e| Box::from(e)),
+            PdlParser::Json => from_str(result).map_err(|e| Box::from(e)), // .map_err(|e| PdlError::from(e))
+            PdlParser::Jsonl => Ok(PdlResult::List(
+                serde_json::Deserializer::from_str(result)
+                    .into_iter::<PdlResult>()
+                    .collect::<Result<_, _>>()?,
+            )),
             PdlParser::Yaml => from_yaml_str(result).map_err(|e| Box::from(e)),
+            PdlParser::Regex(_) => todo!(),
         }
     }
 

pdl-live-react/src-tauri/src/pdl/interpreter_tests.rs

@@ -205,6 +205,29 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn text_parser_jsonl() -> Result<(), Box<dyn Error>> {
+        let json = "{\"key\":\"value\"}
+{\"key2\":\"value2\"}";
+        let program = json!({
+            "text": [
+                { "def": "foo", "parser": "jsonl", "text": [json] },
+                "${ foo[0].key }",
+                "${ foo[1].key2 }"
+            ]
+        });
+
+        let (_, messages, _) = run_json(program, streaming(), initial_scope())?;
+        assert_eq!(messages.len(), 3);
+        assert_eq!(messages[0].role, MessageRole::User);
+        assert_eq!(messages[0].content, json);
+        assert_eq!(messages[1].role, MessageRole::User);
+        assert_eq!(messages[1].content, "value");
+        assert_eq!(messages[2].role, MessageRole::User);
+        assert_eq!(messages[2].content, "value2");
+        Ok(())
+    }
+
     #[test]
     fn last_of_parser_json() -> Result<(), Box<dyn Error>> {
         let json = "{\"key\":\"value\"}";

pdl-live-react/src-tauri/tests/cli/jsonl-parser.pdl

@@ -0,0 +1,9 @@
+text:
+  - text:
+      - |
+          {"key": "value"}
+          {"key2": "value2"}
+    parser: jsonl
+    def: foo
+  - ${ foo[0].key }
+  - ${ foo[1].key2 }