Merge pull request #514 from SteveL-MSFT/array-index

Add array index support to expression grammar

Author: SteveL-MSFT

dsc/src/subcommand.rs

@@ -49,7 +49,7 @@ pub fn config_get(configurator: &mut Configurator, format: &Option<OutputFormat>
             }
         },
         Err(err) => {
-            error!("Error: {err}");
+            error!("{err}");
             exit(EXIT_DSC_ERROR);
         }
     }

dsc/tests/dsc_expressions.tests.ps1

@@ -0,0 +1,66 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+Describe 'Expressions tests' {
+    It 'Accessors work: <text>' -TestCases @(
+        @{ text = "[parameters('test').hello]"; expected = '@{world=there}' }
+        @{ text = "[parameters('test').hello.world]"; expected = 'there' }
+        @{ text = "[parameters('test').array[0]]"; expected = 'one' }
+        @{ text = "[parameters('test').array[1][1]]"; expected = 'three' }
+        @{ text = "[parameters('test').objectArray[0].name]"; expected = 'one' }
+        @{ text = "[parameters('test').objectArray[1].value[0]]"; expected = '2' }
+        @{ text = "[parameters('test').objectArray[1].value[1].name]"; expected = 'three' }
+    ) {
+        param($text, $expected)
+        $yaml = @"
+`$schema: https://raw.githubusercontent.com/PowerShell/DSC/main/schemas/2024/04/config/document.json
+parameters:
+  test:
+    type: object
+    defaultValue:
+      hello:
+        world: there
+      array:
+      - one
+      - [ 'two', 'three' ]
+      objectArray:
+      - name: one
+        value: 1
+      - name: two
+        value:
+        - 2
+        - nestedObject:
+          name: three
+          value: 3
+resources:
+- name: echo
+  type: Test/Echo
+  properties:
+    output: "$text"
+"@
+        $debug = $yaml | dsc -l debug config get -f yaml 2>&1 | Out-String
+        $out = $yaml | dsc config get | ConvertFrom-Json
+        $LASTEXITCODE | Should -Be 0
+        $out.results[0].result.actualState.output | Should -Be $expected -Because $debug
+    }
+
+    It 'Invalid expressions: <expression>' -TestCases @(
+        @{ expression = "[concat('A','B')].hello" }
+        @{ expression = "[concat('A','B')](0)" }
+        @{ expression = "[concat('a','b').hello]" }
+        @{ expression = "[concat('a','b')[0]]" }
+    ) {
+        param($expression)
+        $yaml = @"
+`$schema: https://raw.githubusercontent.com/PowerShell/DSC/main/schemas/2024/04/config/document.json
+resources:
+- name: echo
+  type: Test/Echo
+  properties:
+    output: "$expression"
+"@
+        $out = dsc config get -d $yaml 2>&1
+        $LASTEXITCODE | Should -Be 2
+        $out | Should -BeLike "*ERROR*"
+    }
+}

dsc_lib/src/parser/expressions.rs

@@ -10,10 +10,16 @@ use crate::dscerror::DscError;
 use crate::functions::FunctionDispatcher;
 use crate::parser::functions::Function;
 
+#[derive(Clone)]
+pub enum Accessor {
+    Member(String),
+    Index(Value),
+}
+
 #[derive(Clone)]
 pub struct Expression {
     function: Function,
-    member_access: Option<Vec<String>>,
+    accessors: Vec<Accessor>,
 }
 
 impl Expression {
@@ -32,64 +38,115 @@ impl Expression {
         let Some(function) = expression.child_by_field_name("function") else {
             return Err(DscError::Parser("Function node not found".to_string()));
         };
+        debug!("Parsing function '{:?}'", function);
         let function = Function::new(statement_bytes, &function)?;
-        let member_access = if let Some(members) = expression.child_by_field_name("members") {
-            if members.is_error() {
-                return Err(DscError::Parser("Error parsing dot-notation".to_string()));
+        let mut accessors = Vec::<Accessor>::new();
+        if let Some(accessor) = expression.child_by_field_name("accessor") {
+            debug!("Parsing accessor '{:?}'", accessor);
+            if accessor.is_error() {
+                return Err(DscError::Parser("Error parsing accessor".to_string()));
             }
-            let mut result = vec![];
-            let mut cursor = members.walk();
-            for member in members.named_children(&mut cursor) {
-                if member.is_error() {
-                    return Err(DscError::Parser("Error parsing dot-notation member".to_string()));
+            let mut cursor = accessor.walk();
+            for accessor in accessor.named_children(&mut cursor) {
+                if accessor.is_error() {
+                    return Err(DscError::Parser("Error parsing accessor".to_string()));
                 }
-                let value = member.utf8_text(statement_bytes)?;
-                result.push(value.to_string());
+                let accessor_kind = accessor.kind();
+                let value = match accessor_kind {
+                    "memberAccess" => {
+                        debug!("Parsing member accessor '{:?}'", accessor);
+                        let Some(member_name) = accessor.child_by_field_name("name") else {
+                            return Err(DscError::Parser("Member name not found".to_string()));
+                        };
+                        let member = member_name.utf8_text(statement_bytes)?;
+                        Accessor::Member(member.to_string())
+                    },
+                    "index" => {
+                        debug!("Parsing index accessor '{:?}'", accessor);
+                        let Some(index_value) = accessor.child_by_field_name("indexValue") else {
+                            return Err(DscError::Parser("Index value not found".to_string()));
+                        };
+                        match index_value.kind() {
+                            "number" => {
+                                let value = index_value.utf8_text(statement_bytes)?;
+                                let value = serde_json::from_str(value)?;
+                                Accessor::Index(value)
+                            },
+                            "expression" => {
+                                return Err(DscError::Parser("Expression index not supported".to_string()));
+                            },
+                            _ => {
+                                return Err(DscError::Parser(format!("Invalid accessor kind: '{accessor_kind}'")));
+                            },
+                        }
+                    },
+                    _ => {
+                        return Err(DscError::Parser(format!("Invalid accessor kind: '{accessor_kind}'")));
+                    },
+                };
+                accessors.push(value);
             }
-            Some(result)
         }
-        else {
-            None
-        };
+
         Ok(Expression {
             function,
-            member_access,
+            accessors,
         })
     }
 
     /// Invoke the expression.
     ///
+    /// # Arguments
+    ///
+    /// * `function_dispatcher` - The function dispatcher to use.
+    /// * `context` - The context to use.
+    ///
+    /// # Returns
+    ///
+    /// The result of the expression.
+    ///
     /// # Errors
     ///
     /// This function will return an error if the expression fails to execute.
     pub fn invoke(&self, function_dispatcher: &FunctionDispatcher, context: &Context) -> Result<Value, DscError> {
         let result = self.function.invoke(function_dispatcher, context)?;
         trace!("Function result: '{:?}'", result);
-        if let Some(member_access) = &self.member_access {
-            debug!("Evaluating member access '{:?}'", member_access);
-            if !result.is_object() {
-                return Err(DscError::Parser("Member access on non-object value".to_string()));
-            }
-
+        if self.accessors.is_empty() {
+            Ok(result)
+        }
+        else {
+            debug!("Evaluating accessors");
             let mut value = result;
-            for member in member_access {
-                if !value.is_object() {
-                    return Err(DscError::Parser(format!("Member access '{member}' on non-object value")));
-                }
-
-                if let Some(object) = value.as_object() {
-                    if !object.contains_key(member) {
-                        return Err(DscError::Parser(format!("Member '{member}' not found")));
-                    }
-
-                    value = object[member].clone();
+            for accessor in &self.accessors {
+                match accessor {
+                    Accessor::Member(member) => {
+                        if let Some(object) = value.as_object() {
+                            if !object.contains_key(member) {
+                                return Err(DscError::Parser(format!("Member '{member}' not found")));
+                            }
+                            value = object[member].clone();
+                        } else {
+                            return Err(DscError::Parser("Member access on non-object value".to_string()));
+                        }
+                    },
+                    Accessor::Index(index) => {
+                        if let Some(array) = value.as_array() {
+                            let Some(index) = index.as_u64() else {
+                                return Err(DscError::Parser("Index is not a valid number".to_string()));
+                            };
+                            let index = usize::try_from(index)?;
+                            if index >= array.len() {
+                                return Err(DscError::Parser("Index out of bounds".to_string()));
+                            }
+                            value = array[index].clone();
+                        } else {
+                            return Err(DscError::Parser("Index access on non-array value".to_string()));
+                        }
+                    },
                 }
             }
 
             Ok(value)
         }
-        else {
-            Ok(result)
-        }
     }
 }

dsc_lib/src/parser/mod.rs

@@ -52,43 +52,45 @@ impl Statement {
         if root_node.is_error() {
             return Err(DscError::Parser(format!("Error parsing statement root: {statement}")));
         }
-        let root_node_kind = root_node.kind();
-        if root_node_kind != "statement" {
+        if root_node.kind() != "statement" {
             return Err(DscError::Parser(format!("Invalid statement: {statement}")));
         }
-        let Some(child_node) = root_node.named_child(0) else {
-            return Err(DscError::Parser("Child node not found".to_string()));
-        };
-        if child_node.is_error() {
-            return Err(DscError::Parser("Error parsing statement child".to_string()));
-        }
-        let kind = child_node.kind();
         let statement_bytes = statement.as_bytes();
-        match kind {
-            "stringLiteral" | "bracketInStringLiteral" => {
-                let Ok(value) = child_node.utf8_text(statement_bytes) else {
-                    return Err(DscError::Parser("Error parsing string literal".to_string()));
-                };
-                debug!("Parsing string literal: {0}", value.to_string());
-                Ok(Value::String(value.to_string()))
-            },
-            "escapedStringLiteral" => {
-                // need to remove the first character: [[ => [
-                let Ok(value) = child_node.utf8_text(statement_bytes) else {
-                    return Err(DscError::Parser("Error parsing escaped string literal".to_string()));
-                };
-                debug!("Parsing escaped string literal: {0}", value[1..].to_string());
-                Ok(Value::String(value[1..].to_string()))
-            },
-            "expression" => {
-                debug!("Parsing expression");
-                let expression = Expression::new(statement_bytes, &child_node)?;
-                Ok(expression.invoke(&self.function_dispatcher, context)?)
-            },
-            _ => {
-                Err(DscError::Parser(format!("Unknown expression type {0}", child_node.kind())))
+        let mut cursor = root_node.walk();
+        let mut return_value = Value::Null;
+        for child_node in root_node.named_children(&mut cursor) {
+            if child_node.is_error() {
+                return Err(DscError::Parser(format!("Error parsing statement: {statement}")));
+            }
+
+            match child_node.kind() {
+                "stringLiteral" => {
+                    let Ok(value) = child_node.utf8_text(statement_bytes) else {
+                        return Err(DscError::Parser("Error parsing string literal".to_string()));
+                    };
+                    debug!("Parsing string literal: {0}", value.to_string());
+                    return_value = Value::String(value.to_string());
+                },
+                "escapedStringLiteral" => {
+                    // need to remove the first character: [[ => [
+                    let Ok(value) = child_node.utf8_text(statement_bytes) else {
+                        return Err(DscError::Parser("Error parsing escaped string literal".to_string()));
+                    };
+                    debug!("Parsing escaped string literal: {0}", value[1..].to_string());
+                    return_value = Value::String(value[1..].to_string());
+                },
+                "expression" => {
+                    debug!("Parsing expression");
+                    let expression = Expression::new(statement_bytes, &child_node)?;
+                    return_value = expression.invoke(&self.function_dispatcher, context)?;
+                },
+                _ => {
+                    return Err(DscError::Parser(format!("Unknown expression type {0}", child_node.kind())));
+                }
             }
         }
+
+        Ok(return_value)
     }
 }
 
@@ -113,8 +115,8 @@ mod tests {
     #[test]
     fn bracket_in_string() {
         let mut parser = Statement::new().unwrap();
-        let result = parser.parse_and_execute("[this] is a string", &Context::new()).unwrap();
-        assert_eq!(result, "[this] is a string");
+        let result = parser.parse_and_execute("[this] is a string", &Context::new());
+        assert!(result.is_err());
     }
 
     #[test]

tree-sitter-dscexpression/.npmrc

@@ -1,3 +1,7 @@
+; We generally want to save install/update commands
+save=true
+; We use a public Azure Artifacts mirror
 registry=https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/npm/registry/
-
 always-auth=true
+; But we don't want references to it in the lockfile
+omit-lockfile-registry-resolved=true

tree-sitter-dscexpression/build.ps1

@@ -1,7 +1,9 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-# check if tools are installed
+param(
+    [switch]$UpdatePackages
+)
 
 function Invoke-NativeCommand($cmd) {
     Invoke-Expression $cmd
@@ -10,6 +12,8 @@ function Invoke-NativeCommand($cmd) {
     }
 }
 
+$env:TREE_SITTER_VERBOSE=1
+
 if ($null -eq (Get-Command npm -ErrorAction Ignore)) {
     Write-Host 'Installing Node'
 
@@ -30,5 +34,18 @@ if ($LASTEXITCODE -ne 0) {
     npm ci tree-sitter-cli --omit=optional
 }
 
-Invoke-NativeCommand 'npx tree-sitter generate'
+if ($UpdatePackages) {
+    if (!$IsWindows) {
+        throw "This switch only works on Windows"
+    }
+
+    rm ./package-lock.json
+    rm -r ./node_modules
+    npm cache clean --force
+    npm logout
+    vsts-npm-auth -config .npmrc -F -V
+    npm install --force --verbose
+}
+
+Invoke-NativeCommand 'npx tree-sitter generate --build'
 Invoke-NativeCommand 'npx tree-sitter test'