feat: support JSONPath (#182)

Use library `jsonpath-ng` to replace own but limited JSONPath implementation. Especially the filter expressions allow more complex replacements within lists. Example: `backend.env[?name=='TEST'].value`

Resolves #181.

Author: christiansiegel

docs/commands/deploy.md

@@ -15,10 +15,10 @@ backend:
   tag: 1.0.0 # <- and this one
   env:
   - name: TEST
-    value: foo # <- and even one in a list
+    value: foo # <- and this one in a list, selected via sibling value 'TEST'
 ```
 
-With the following command GitOps CLI will update both values to `1.1.0` on the default branch.
+With the following command GitOps CLI will update all values on the default branch.
 
 ```bash
 gitopscli deploy \
@@ -30,9 +30,11 @@ gitopscli deploy \
   --organisation "deployment" \
   --repository-name "myapp-non-prod" \
   --file "example/values.yaml" \
-  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env.[0].value': bar}"
+  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env[?name==''TEST''].value': bar}"
 ```
 
+You could also use the list index to replace the latter (`my-app.env.[0].value`). For more details on the underlying *JSONPath* syntax, please refer to the [documenatation of the used library *jsonpath-ng*](https://github.com/h2non/jsonpath-ng#jsonpath-syntax).
+
 ### Number Of Commits
 
 Note that by default GitOps CLI will create a separate commit for every value change:
@@ -42,7 +44,7 @@ commit 0dcaa136b4c5249576bb1f40b942bff6ac718144
 Author: GitOpsCLI <[email protected]>
 Date:   Thu Mar 12 15:30:32 2020 +0100
 
-    changed 'backend.env.[0].value' to 'bar' in example/values.yaml
+    changed 'backend.env[?name=='TEST'].value' to 'bar' in example/values.yaml
 
 commit d98913ad8fecf571d5f8c3635f8070b05c43a9ca
 Author: GitOpsCLI <[email protected]>
@@ -64,11 +66,11 @@ commit 3b96839e90c35b8decf89f34a65ab6d66c8bab28
 Author: GitOpsCLI <[email protected]>
 Date:   Thu Mar 12 15:30:00 2020 +0100
 
-    updated 2 values in example/values.yaml
+    updated 3 values in example/values.yaml
 
     frontend.tag: '1.1.0'
     backend.tag: '1.1.0'
-    backend.env.[0].value: 'bar'
+    'backend.env[?name==''TEST''].value': 'bar'
 ```
 
 ### Specific Commit Message
@@ -88,7 +90,7 @@ gitopscli deploy \
   --repository-name "myapp-non-prod" \
   --commit-message "test commit message" \
   --file "example/values.yaml" \
-  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env.[0].value': bar}"
+  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env[?name==''TEST''].value': bar}"
 ```
 
 This will end up in one single commit with your specified commit-message.
@@ -107,7 +109,7 @@ gitopscli deploy \
   --organisation "deployment" \
   --repository-name "myapp-non-prod" \
   --file "example/values.yaml" \
-  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env.[0].value': bar}" \
+  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env[?name==''TEST''].value': bar}" \
   --create-pr \
   --auto-merge
 ```

docs/includes/preview-configuration.md

@@ -23,7 +23,7 @@ Make sure that your *app repository* contains a `.gitops.config.yaml` file. This
 
 1. find repository, branch, and folder containing the template
 2. templates for host and namespace name
-3. replace values in template files
+3. replace values in template files (see [`deploy` command](/gitopscli/commands/deploy/) for details on the key syntax)
 4. find repository and branch where the preview should be created (i.e. your *deployment config repository*)
 5. message templates used to comment your pull request
 

gitopscli/commands/deploy.py

@@ -70,7 +70,7 @@ def __update_values(self, git_repo: GitRepo) -> Dict[str, Any]:
             except YAMLException as ex:
                 raise GitOpsException(f"Error loading file: {args.file}") from ex
             except KeyError as ex:
-                raise GitOpsException(f"Key '{key}' not found in file: {args.file}") from ex
+                raise GitOpsException(str(ex)) from ex
 
             if not updated_value:
                 logging.info("Yaml property %s already up-to-date", key)

gitopscli/io_api/yaml_util.py

@@ -1,10 +1,8 @@
-import re
 from io import StringIO
 from typing import Any
 from ruamel.yaml import YAML, YAMLError
-
-
-_ARRAY_KEY_SEGMENT_PATTERN = re.compile(r"\[(\d+)\]")
+from jsonpath_ng.exceptions import JSONPathError
+from jsonpath_ng.ext import parse
 
 YAML_INSTANCE = YAML()
 YAML_INSTANCE.preserve_quotes = True  # type: ignore
@@ -41,30 +39,24 @@ def yaml_dump(yaml: Any) -> str:
 
 
 def update_yaml_file(file_path: str, key: str, value: Any) -> bool:
+    if not key:
+        raise KeyError("Empty key!")
     content = yaml_file_load(file_path)
-
-    key_segments = key.split(".") if key else []
-    current_key_segments = []
-    parent_item = content
-    for current_key_segment in key_segments:
-        current_key_segments.append(current_key_segment)
-        current_key = ".".join(current_key_segments)
-        is_array = _ARRAY_KEY_SEGMENT_PATTERN.match(current_key_segment)
-        if is_array:
-            current_array_index = int(is_array.group(1))
-            if not isinstance(parent_item, list) or current_array_index >= len(parent_item):
-                raise KeyError(f"Key '{current_key}' not found in YAML!")
-        else:
-            if not isinstance(parent_item, dict) or current_key_segment not in parent_item:
-                raise KeyError(f"Key '{current_key}' not found in YAML!")
-        if current_key == key:
-            if parent_item[current_array_index if is_array else current_key_segment] == value:
-                return False  # nothing to update
-            parent_item[current_array_index if is_array else current_key_segment] = value
-            yaml_file_dump(content, file_path)
-            return True
-        parent_item = parent_item[current_array_index if is_array else current_key_segment]
-    raise KeyError(f"Empty key!")
+    try:
+        jsonpath_expr = parse(key)
+    except JSONPathError as ex:
+        raise KeyError(f"Key '{key}' is invalid JSONPath expression: {ex}!") from ex
+    matches = jsonpath_expr.find(content)
+    if not matches:
+        raise KeyError(f"Key '{key}' not found in YAML!")
+    if all(match.value == value for match in matches):
+        return False  # nothing to update
+    try:
+        jsonpath_expr.update(content, value)
+    except TypeError as ex:
+        raise KeyError(f"Key '{key}' cannot be updated: {ex}!") from ex
+    yaml_file_dump(content, file_path)
+    return True
 
 
 def merge_yaml_element(file_path: str, element_path: str, desired_value: Any) -> None:

setup.py

@@ -9,6 +9,7 @@
     install_requires=[
         "GitPython==3.0.6",
         "ruamel.yaml==0.16.5",
+        "jsonpath-ng==1.5.3",
         "atlassian-python-api==1.14.5",
         "PyGithub==1.53",
         "python-gitlab==2.6.0",

tests/commands/test_deploy.py

@@ -405,7 +405,7 @@ def test_key_not_found(self):
         )
         with pytest.raises(GitOpsException) as ex:
             DeployCommand(args).execute()
-        self.assertEqual(str(ex.value), "Key 'a.b.c' not found in file: test/file.yml")
+        self.assertEqual(str(ex.value), "'dummy key error'")
 
         assert self.mock_manager.method_calls == [
             call.GitRepoApiFactory.create(args, "ORGA", "REPO"),

tests/io_api/test_yaml_util.py

@@ -16,6 +16,8 @@
 
 
 class YamlUtilTest(unittest.TestCase):
+    maxDiff = None
+
     @classmethod
     def setUpClass(cls):
         cls.tmp_dir = f"/tmp/gitopscli-test-{uuid.uuid4()}"
@@ -117,7 +119,15 @@ def test_update_yaml_file(self):
     g: 4 # comment 6
   - [hello, world] # comment 7
   - foo: # comment 8
-      bar # comment 9"""
+      bar # comment 9
+  - list: # comment 10
+    - key: k1 # comment 11
+      value: v1 # comment 12
+    - key: k2 # comment 13
+      value: v2 # comment 14
+    - {key: k3+4, value: v3} # comment 15
+    - key: k3+4 # comment 16
+      value: v4 # comment 17"""
         )
 
         self.assertTrue(update_yaml_file(test_file, "a.b.c", "2"))
@@ -132,6 +142,11 @@ def test_update_yaml_file(self):
         self.assertTrue(update_yaml_file(test_file, "a.e.[2]", "replaced object"))
         self.assertFalse(update_yaml_file(test_file, "a.e.[2]", "replaced object"))  # already updated
 
+        self.assertTrue(update_yaml_file(test_file, "a.e.[*].list[?key=='k3+4'].value", "replaced v3 and v4"))
+        self.assertFalse(
+            update_yaml_file(test_file, "a.e.[*].list[?key=='k3+4'].value", "replaced v3 and v4")
+        )  # already updated
+
         expected = """\
 a: # comment 1
 # comment 2
@@ -144,17 +159,25 @@ def test_update_yaml_file(self):
     g: 42 # comment 6
   - [hello, tester] # comment 7
   - replaced object
+  - list: # comment 10
+    - key: k1 # comment 11
+      value: v1 # comment 12
+    - key: k2 # comment 13
+      value: v2 # comment 14
+    - {key: k3+4, value: replaced v3 and v4} # comment 15
+    - key: k3+4 # comment 16
+      value: replaced v3 and v4 # comment 17
 """
         actual = self._read_file(test_file)
         self.assertEqual(expected, actual)
 
         with pytest.raises(KeyError) as ex:
             update_yaml_file(test_file, "x.y", "foo")
-        self.assertEqual("\"Key 'x' not found in YAML!\"", str(ex.value))
+        self.assertEqual("\"Key 'x.y' not found in YAML!\"", str(ex.value))
 
         with pytest.raises(KeyError) as ex:
             update_yaml_file(test_file, "[42].y", "foo")
-        self.assertEqual("\"Key '[42]' not found in YAML!\"", str(ex.value))
+        self.assertEqual("\"Key '[42].y' not found in YAML!\"", str(ex.value))
 
         with pytest.raises(KeyError) as ex:
             update_yaml_file(test_file, "a.x", "foo")
@@ -165,12 +188,25 @@ def test_update_yaml_file(self):
         self.assertEqual("\"Key 'a.[42]' not found in YAML!\"", str(ex.value))
 
         with pytest.raises(KeyError) as ex:
-            update_yaml_file(test_file, "a.e.[3]", "foo")
-        self.assertEqual("\"Key 'a.e.[3]' not found in YAML!\"", str(ex.value))
+            update_yaml_file(test_file, "a.e.[100]", "foo")
+        self.assertEqual("\"Key 'a.e.[100]' not found in YAML!\"", str(ex.value))
+
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "a.e.[*].list[?key=='foo'].value", "foo")
+        self.assertEqual("\"Key 'a.e.[*].list[?key=='foo'].value' not found in YAML!\"", str(ex.value))
 
         with pytest.raises(KeyError) as ex:
             update_yaml_file(test_file, "a.e.[2].[2]", "foo")
-        self.assertEqual("\"Key 'a.e.[2].[2]' not found in YAML!\"", str(ex.value))
+        self.assertEqual(
+            "\"Key 'a.e.[2].[2]' cannot be updated: 'str' object does not support item assignment!\"", str(ex.value)
+        )
+
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "invalid JSONPath", "foo")
+        self.assertEqual(
+            "\"Key 'invalid JSONPath' is invalid JSONPath expression: Parse error at 1:8 near token JSONPath (ID)!\"",
+            str(ex.value),
+        )
 
         with pytest.raises(KeyError) as ex:
             update_yaml_file(test_file, "", "foo")