feat!: use expr language for executable templates and request transformations (#204)

Author: jahvon

docs/guide/conditional.md

@@ -5,7 +5,7 @@ using a simple expression language that provides access to system information, e
 
 ### Expression Language
 
-Flow uses the [Expr](https://expr-lang.org) language for evaluating conditions. The language supports common
+flow uses the [Expr](https://expr-lang.org) language for evaluating conditions. The language supports common
 operators and functions while providing access to flow executable-specific context data.
 
 **See the [Expr language documentation](https://expr-lang.org/docs/language-definition) for more information on the

docs/guide/executable.md

@@ -365,9 +365,9 @@ executables:
       url: "http://pi.hole/admin/api.php?disable=$DURATION&auth=$PWHASH"
       logResponse: true # log the response body
       validStatusCodes: [200] # only consider the execution successful if the status code is 200
-      # transform the response body with jq
+      # transform the response body with a Expr expression
       transformResponse: |
-        if .status == "disabled" then .status = "pause" else . end
+        "paused: " + string(fromJSON(body)["status"] == "disabled")
 ```
 
 ##### render

docs/guide/templating.md

@@ -100,18 +100,18 @@ files will only be copied if the `Type` field is set to `Helm`. The `deploy.sh`
 artifacts:
   - srcName: "helm-deploy.sh"
     srcDir: "scripts" # By default, the file will be copied from the template directory. This field can be used to specify a different directory.
-    if: "{{ eq .Type 'Helm' }}" # This will only copy the file if the Helm field is true
+    if: form["Type"] == "Helm" # This will only copy the file if the Helm field is true
   - srcName: "deploy.sh"
     srcDir: "scripts"
-    if: "{{ eq .Type 'K8s' }}" # This will only copy the file if the K8s field is true
+    if: form["Type"] == "K8s" # This will only copy the file if the K8s field is true
   - srcName: "values.yaml.tmpl"
     asTemplate: true
     dstName: "values.yaml"
-    if: "{{ eq .Type 'Helm' }}" # This will only copy the file if the Helm field is true
+    if: form["Type"] == "Helm" # This will only copy the file if the Helm field is true
   - srcName: "resources.yaml.tmpl"
     asTemplate: true
     dstName: "resources.yaml"
-    if: "{{ eq .Type 'K8s' }}" # This will only copy the file if the K8s field is true
+    if: form["Type"] == "K8s" # This will only copy the file if the K8s field is true
 ```
 
 ### flowfile template string
@@ -126,22 +126,22 @@ template: |
   tags: [k8s]
   executables:
     - verb: deploy
-      name: "{{ .FlowFileName }}"
+      name: "{{ name }}"
       exec:
-        file: "{{ if eq .Type 'Helm' }}helm-deploy.sh{{ else }}deploy.sh{{ end }}"
+        file: "{{ if form["Type"] == 'Helm' }}helm-deploy.sh{{ else }}deploy.sh{{ end }}"
         params:
           - envKey: "NAMESPACE"
-            text: "{{ .Namespace }}"
+            text: "{{ form["Namespace"] }}"
           - envKey: "APP_NAME"
-            text: "{{ .FlowFileName }}"
+            text: "{{ name }}"
     - verb: restart
-      name: "{{ .FlowFileName }}"
+      name: "{{ name }}"
       exec:
-        cmd: "kubectl rollout restart deployment/{{ .FlowFileName }} -n {{ .Namespace }}"
+        cmd: "kubectl rollout restart deployment/{{ name }} -n {{ form["Namespace"] }}"
     - verb: open
-      name: "{{ .FlowFileName }}"
+      name: "{{ name }}"
       launch:
-        uri: "https://{{ .FlowFileName }}.my.haus"
+        uri: "https://{{ name }}.my.haus"
 ```
 
 ### Pre- and post- run executables
@@ -156,14 +156,15 @@ Before exiting, it will also run a simple command and either open the flowfile i
 preRun:
   - ref: "validate k8s/validation:context" # You can reference other executables that you have on your system
     args: ["homelab"]
-    if: "{{ .Deploy }}"
+    if: form["Deploy"]
 postRun:
-  - cmd: "echo 'Rendered {{ if .Helm }}Helm values{{ else }}k8s manifest{{end}}'; ls -al"
+  - cmd: |
+      echo 'Rendered {{ if form["Helm"] }}Helm values{{ else }}k8s manifest{{end}}'; ls -al
   - ref: "edit vscode"
-    args: ["{{ .FlowFilePath }}"]
-    if: "{{ not .Deploy }}"
-  - ref: "deploy {{ .FlowFileName }}"
-    if: "{{ .Deploy }}"
+    args: ["{{ flowFilePath }}"]
+    if: not form["Deploy"]
+  - ref: "deploy {{ name }}"
+    if: form["Deploy"]
 ```
 
 **Note**: preRun executables are run from the template directory, while postRun executables are run from the output directory.
@@ -215,36 +216,56 @@ postRun:
   - ref: "edit vscode"
     args: ["{{ .FlowFilePath }}"]
     if: "{{ not .Deploy }}"
-  - ref: "deploy {{ .FlowFileName }}"
+  - ref: "deploy {{ .name }}"
     if: "{{ .Deploy }}"
 template: |
   tags: [k8s]
   executables:
     - verb: deploy
-      name: "{{ .FlowFileName }}"
+      name: "{{ name }}"
       exec:
         file: "{{ if eq .Type 'Helm' }}helm-deploy.sh{{ else }}deploy.sh{{ end }}"
         params:
           - envKey: "NAMESPACE"
-            text: "{{ .Namespace }}"
+            text: "{{ .form.namespace }}"
           - envKey: "APP_NAME"
-            text: "{{ .FlowFileName }}"
+            text: "{{ name }}"
     - verb: restart
-      name: "{{ .FlowFileName }}"
+      name: "{{ name }}"
       exec:
         cmd: "kubectl rollout restart deployment/{{ .FlowFileName }} -n {{ .Namespace }}"
     - verb: open
-      name: "{{ .FlowFileName }}"
+      name: "{{ name }}"
       launch:
-        uri: "https://{{ .FlowFileName }}.my.haus"
-``` 
+        uri: "https://{{ name }}.my.haus"
+```
+### Templating language
+
+flow uses a hybrid of [Go text templating](https://pkg.go.dev/text/template) and [Expr](https://expr-lang.org) language for
+rendering templates.
+
+Aside from the `if` field in the artifacts, preRun, and postRun sections, all other fields that use templating
+will need to include the `{{` and `}}` delimiters to indicate that the text should be rendered as a template.
+
+**Template Variables**
+
+The following variables are automatically available in all template expressions:
 
-### Template helpers
+| Variable        | Type                   | Description                                                      |
+|-----------------|------------------------|------------------------------------------------------------------|
+| `os`            | string                 | Operating system identifier (e.g., "linux", "darwin", "windows") |
+| `arch`          | string                 | System architecture (e.g., "amd64", "arm64")                     |
+| `workspace`     | string                 | Target workspace name                                            |
+| `workspacePath` | string                 | Full path to the target workspace root directory                 |
+| `name`          | string                 | Name provided for the newly rendered flow file                   |
+| `directory`     | string                 | Target directory that the template will be render in to          |
+| `flowFilePath`  | string                 | Full path to the target flow file                                |
+| `templatePath`  | string                 | Path to the template file being rendered                         |
+| `env`           | map (string -> string) | Environment variables accessible to the template                 |
+| `form`          | map (string -> any)    | Values provided through template form inputs                     |
 
-Templates can use the [Sprig functions](https://masterminds.github.io/sprig/) to manipulate data during the rendering process.
-Additionally, the following keys are available to the template:
+**See the [Expr language documentation](https://expr-lang.org/docs/language-definition) for more information on the
+additional expression functions and syntax.**
 
-- `FlowFileName`: The name of the flowfile being rendered. This is the argument passed into the `generate` command.
-- `FlowFilePath`: The output path to the flowfile being rendered.
-- `FlowWorkspace`: The name of the workspace that the template is rendering into.
-- `FlowWorkspacePath`: The path to the workspace root directory.
+> [!NOTE]
+> The `env` map contains environment variables that were present when the template was rendered. The `form` map contains values from any form inputs defined in the template configuration.

docs/schemas/flowfile_schema.json

@@ -390,7 +390,7 @@
           "default": "30m0s"
         },
         "transformResponse": {
-          "description": "JQ query to transform the response before saving it to a file or outputting it.",
+          "description": "[Expr](https://expr-lang.org/docs/language-definition) expression used to transform the response before \nsaving it to a file or outputting it.\n\nThe following variables are available in the expression:\n  - `status`: The response status string.\n  - `code`: The response status code.\n  - `body`: The response body.\n  - `headers`: The response headers.\n\nFor example, to capitalize a JSON body field's value, you can use `upper(fromJSON(body)[\"field\"])`.\n",
           "type": "string",
           "default": ""
         },

docs/schemas/template_schema.json

@@ -30,7 +30,7 @@
           "default": ""
         },
         "if": {
-          "description": "A condition to determine if the artifact should be copied. The condition is evaluated using Go templating \nfrom the form data. If the condition is not met, the artifact will not be copied.\n[Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition.\n\nFor example, to copy the artifact only if the `name` field is set:\n```\n{{ if .name }}true{{ end }}\n```\n",
+          "description": "An expression that determines whether the the artifact should be copied, using the Expr language syntax. \nThe expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, \nthe artifact will not be copied.\n\nThe expression has access to OS/architecture information (os, arch), environment variables (env), form input \n(form), and context information (name, workspace, directory, etc.).\n\nSee the [flow documentation](https://flowexec.io/#/guide/templating) for more information.\n",
           "type": "string",
           "default": ""
         },
@@ -121,7 +121,7 @@
           "default": ""
         },
         "if": {
-          "description": "A condition to determine if the executable should be run. The condition is evaluated using Go templating \nfrom the form data. If the condition is not met, the executable run will be skipped.\n[Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition.\n\nFor example, to run a command only if the `name` field is set:\n```\n{{ if .name }}true{{ end }}\n```\n",
+          "description": "An expression that determines whether the executable should be run, using the Expr language syntax. \nThe expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, \nthe executable will be skipped.\n\nThe expression has access to OS/architecture information (os, arch), environment variables (env), form input \n(form), and context information (name, workspace, directory, etc.).\n\nSee the [flow documentation](https://flowexec.io/#/guide/templating) for more information.\n",
           "type": "string",
           "default": ""
         },

docs/types/flowfile.md

@@ -302,7 +302,7 @@ Makes an HTTP request.
 | `params` |  | [ExecutableParameterList](#ExecutableParameterList) | <no value> |  |
 | `responseFile` |  | [ExecutableRequestResponseFile](#ExecutableRequestResponseFile) | <no value> |  |
 | `timeout` | The timeout for the request in Go duration format (e.g. 30s, 5m, 1h). | `string` | 30m0s |  |
-| `transformResponse` | JQ query to transform the response before saving it to a file or outputting it. | `string` |  |  |
+| `transformResponse` | [Expr](https://expr-lang.org/docs/language-definition) expression used to transform the response before  saving it to a file or outputting it.  The following variables are available in the expression:   - `status`: The response status string.   - `code`: The response status code.   - `body`: The response body.   - `headers`: The response headers.  For example, to capitalize a JSON body field's value, you can use `upper(fromJSON(body)["field"])`.  | `string` |  |  |
 | `url` | The URL to make the request to. | `string` |  | ✘ |
 | `validStatusCodes` | A list of valid status codes. If the response status code is not in this list, the executable will fail. If not set, the response status code will not be checked.  | `array` (`integer`) | [] |  |
 

docs/types/template.md

@@ -39,7 +39,7 @@ Go templating from form data is supported in all fields.
 | `asTemplate` | If true, the artifact will be copied as a template file. The file will be rendered using Go templating from  the form data. [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the template.  | `boolean` | false |  |
 | `dstDir` | The directory to copy the file to. If not set, the file will be copied to the root of the flow file directory. The directory will be created if it does not exist.  | `string` |  |  |
 | `dstName` | The name of the file to copy to. If not set, the file will be copied with the same name. | `string` |  |  |
-| `if` | A condition to determine if the artifact should be copied. The condition is evaluated using Go templating  from the form data. If the condition is not met, the artifact will not be copied. [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition.  For example, to copy the artifact only if the `name` field is set: ``` {{ if .name }}true{{ end }} ```  | `string` |  |  |
+| `if` | An expression that determines whether the the artifact should be copied, using the Expr language syntax.  The expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met,  the artifact will not be copied.  The expression has access to OS/architecture information (os, arch), environment variables (env), form input  (form), and context information (name, workspace, directory, etc.).  See the [flow documentation](https://flowexec.io/#/guide/templating) for more information.  | `string` |  |  |
 | `srcDir` | The directory to copy the file from.  If not set, the file will be copied from the directory of the template file.  | `string` |  |  |
 | `srcName` | The name of the file to copy. | `string` | <no value> |  |
 
@@ -94,7 +94,7 @@ Configuration for a template executable.
 | ----- | ----------- | ---- | ------- | :--------: |
 | `args` | Arguments to pass to the executable. | `array` (`string`) | [] |  |
 | `cmd` | The command to execute. One of `cmd` or `ref` must be set.  | `string` |  |  |
-| `if` | A condition to determine if the executable should be run. The condition is evaluated using Go templating  from the form data. If the condition is not met, the executable run will be skipped. [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition.  For example, to run a command only if the `name` field is set: ``` {{ if .name }}true{{ end }} ```  | `string` |  |  |
+| `if` | An expression that determines whether the executable should be run, using the Expr language syntax.  The expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met,  the executable will be skipped.  The expression has access to OS/architecture information (os, arch), environment variables (env), form input  (form), and context information (name, workspace, directory, etc.).  See the [flow documentation](https://flowexec.io/#/guide/templating) for more information.  | `string` |  |  |
 | `ref` | A reference to another executable to run in serial. One of `cmd` or `ref` must be set.  | [ExecutableRef](#ExecutableRef) |  |  |
 
 

examples/exec-template.flow.tmpl

@@ -10,7 +10,7 @@ template: |
   namespace: examples
   executables:
     - verb: run
-      name: "{{ .Color }}-msg"
-      description: Created from a template in {{ .FlowWorkspace }}
+      name: {{ form["Color"] }}-msg
+      description: Created from a template in {{ workspace }}
       exec:
         cmd: cat message.txt

examples/request.flow

@@ -41,7 +41,8 @@ executables:
       url: https://httpbin.org/post
       body: '{"hello": "world"}'
       logResponse: true
-      transformResponse: .args.hello = "universe" | .args
+      transformResponse: |
+        "Hello, " + upper(fromJSON(body)["json"]["hello"])
 
   - verb: send
     name: param-request

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/charmbracelet/lipgloss v1.0.0
 	github.com/expr-lang/expr v1.16.9
 	github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4
-	github.com/itchyny/gojq v0.12.17
 	github.com/jahvon/glamour v0.8.1-patch3
 	github.com/jahvon/open-golang v0.0.0-20240522004812-68511c3bc9ef
 	github.com/jahvon/tuikit v0.0.27
@@ -58,7 +57,6 @@ require (
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect