Feat: Add 'replace_pattern' entry for DictComparator and several related save capabilities (#53)

Author: adrien-berchet

dir_content_diff/base_comparators.py

@@ -2,12 +2,15 @@
 import configparser
 import filecmp
 import json
+import re
 from abc import ABC
 from abc import abstractmethod
 from xml.etree import ElementTree
 
 import dictdiffer
+import jsonpath_ng
 import yaml
+from dicttoxml import dicttoxml
 from diff_pdf_visually import pdf_similar
 
 from dir_content_diff.util import diff_msg_formatter
@@ -282,6 +285,14 @@ class DictComparator(BaseComparator):
         "add": "Added the value(s) '{value}' in the '{key}' key.",
         "change": "Changed the value of '{key}' from {value[0]} to {value[1]}.",
         "remove": "Removed the value(s) '{value}' from '{key}' key.",
+        "missing_ref_entry": (
+            "The path '{key}' is missing in the reference dictionary, please fix the "
+            "'replace_pattern' argument."
+        ),
+        "missing_comp_entry": (
+            "The path '{key}' is missing in the compared dictionary, please fix the "
+            "'replace_pattern' argument."
+        ),
     }
 
     def __init__(self, *args, **kwargs):
@@ -318,6 +329,43 @@ def _format_change_value(value):
                 value[num] = str(i)
         return value
 
+    def format_data(self, data, ref=None, replace_pattern=None, **kwargs):
+        """Format the loaded data."""
+        # pylint: disable=too-many-nested-blocks
+        self.current_state["format_errors"] = errors = []
+
+        if replace_pattern is not None:
+            for pat, paths in replace_pattern.items():
+                pattern = pat[0]
+                new_value = pat[1]
+                count = pat[2] if len(pat) > 2 else 0
+                flags = pat[3] if len(pat) > 3 else 0
+                for raw_path in paths:
+                    path = jsonpath_ng.parse(raw_path)
+                    if ref is not None and len(path.find(ref)) == 0:
+                        errors.append(
+                            (
+                                "missing_ref_entry",
+                                raw_path,
+                                None,
+                            )
+                        )
+                    elif len(path.find(data)) == 0:
+                        errors.append(
+                            (
+                                "missing_comp_entry",
+                                raw_path,
+                                None,
+                            )
+                        )
+                    else:
+                        for i in path.find(data):
+                            if isinstance(i.value, str):
+                                i.full_path.update(
+                                    data, re.sub(pattern, new_value, i.value, count, flags)
+                                )
+        return data
+
     def diff(self, ref, comp, *args, **kwargs):
         """Compare 2 dictionaries.
 
@@ -332,13 +380,16 @@ def diff(self, ref, comp, *args, **kwargs):
             path_limit (list[str]): List of path limit tuples or :class:`dictdiffer.utils.PathLimit`
                 object to limit the diff recursion depth.
         """
+        errors = self.current_state.get("format_errors", [])
+
         if len(args) > 5:
             dot_notation = args[5]
             args = args[:5] + args[6:]
         else:
             dot_notation = kwargs.pop("dot_notation", False)
         kwargs["dot_notation"] = dot_notation
-        return list(dictdiffer.diff(ref, comp, *args, **kwargs))
+        errors.extend(list(dictdiffer.diff(ref, comp, *args, **kwargs)))
+        return errors
 
     def format_diff(self, difference):
         """Format one element difference."""
@@ -361,6 +412,11 @@ def load(self, path):
             data = json.load(file)
         return data
 
+    def save(self, data, path):
+        """Save formatted data into a JSON file."""
+        with open(path, "w", encoding="utf-8") as file:
+            json.dump(data, file)
+
 
 class YamlComparator(DictComparator):
     """Comparator for YAML files.
@@ -374,6 +430,11 @@ def load(self, path):
             data = yaml.full_load(file)
         return data
 
+    def save(self, data, path):
+        """Save formatted data into a YAML file."""
+        with open(path, "w", encoding="utf-8") as file:
+            yaml.dump(data, file)
+
 
 class XmlComparator(DictComparator):
     """Comparator for XML files.
@@ -407,9 +468,14 @@ def load(self, path):  # pylint: disable=arguments-differ
             data = self.xmltodict(file.read())
         return data
 
+    def save(self, data, path):
+        """Save formatted data into a XML file."""
+        with open(path, "w", encoding="utf-8") as file:
+            file.write(dicttoxml(data["root"]).decode())
+
     @staticmethod
     def _cast_from_attribute(text, attr):
-        """Converts XML text into a Python data format based on the tag attribute."""
+        """Convert XML text into a Python data format based on the tag attribute."""
         if "type" not in attr:
             return text
         value_type = attr.get("type", "").lower()
@@ -453,7 +519,7 @@ def add_to_output(obj, child):
 
     @staticmethod
     def xmltodict(obj):
-        """Converts an XML string into a Python object based on each tag's attribute."""
+        """Convert an XML string into a Python object based on each tag's attribute."""
         root = ElementTree.fromstring(obj)
         output = {}
 
@@ -473,11 +539,16 @@ class IniComparator(DictComparator):
     """
 
     def load(self, path, **kwargs):  # pylint: disable=arguments-differ
-        """Open a XML file."""
+        """Open a INI file."""
         data = configparser.ConfigParser(**kwargs)
         data.read(path)
         return self.configparser_to_dict(data)
 
+    def save(self, data, path):
+        """Save formatted data into a INI file."""
+        with open(path, "w", encoding="utf-8") as file:
+            self.dict_to_configparser(data).write(file)
+
     @staticmethod
     def configparser_to_dict(config):
         """Transform a ConfigParser object into a dict."""
@@ -494,6 +565,16 @@ def configparser_to_dict(config):
                 dict_config[section][option] = val
         return dict_config
 
+    @staticmethod
+    def dict_to_configparser(data, **kwargs):
+        """Transform a dict object into a ConfigParser."""
+        config = configparser.ConfigParser(**kwargs)
+        for k, v in data.items():
+            config.add_section(k)
+            for opt, val in v.items():
+                config[k][opt] = json.dumps(val)
+        return config
+
 
 class PdfComparator(BaseComparator):
     """Comparator for PDF files."""

setup.py

@@ -6,7 +6,9 @@
 
 reqs = [
     "dictdiffer>=0.8",
+    "dicttoxml>=1.7.12",
     "diff_pdf_visually>=1.7",
+    "jsonpath-ng>=1.5",
     "PyYaml>=6",
 ]
 

tests/test_base.py

@@ -4,6 +4,7 @@
 # pylint: disable=unused-argument
 # pylint: disable=use-implicit-booleaness-not-comparison
 import configparser
+import copy
 import json
 import re
 
@@ -409,6 +410,83 @@ def report(
         assert kwargs_msg in no_report_diff_default
         assert no_report_diff_default.replace(kwargs_msg, "") == diff
 
+    class TestJsonComparator:
+        """Test the JSON comparator."""
+
+        def test_format_data(self):
+            """Test data formatting."""
+            data = {
+                "a": 1,
+                "b": {
+                    "c": "a string",
+                },
+                "d": [
+                    {"d1": "the d1 string"},
+                    {"d2": "the d2 string"},
+                ],
+                "e": {
+                    "nested_e": {
+                        "nested_e_a": "the nested_e_a string",
+                        "nested_e_b": "the nested_e_b string",
+                    }
+                },
+            }
+            initial_data = copy.deepcopy(data)
+
+            expected_data = {
+                "a": 1,
+                "b": {
+                    "c": "a NEW VALUE",
+                },
+                "d": [
+                    {"d1": "the d1 NEW VALUE"},
+                    {"d2": "the d2 NEW VALUE"},
+                ],
+                "e": {
+                    "nested_e": {
+                        "nested_e_a": "the nested_e_a NEW VALUE",
+                        "nested_e_b": "the nested_e_b NEW VALUE",
+                    }
+                },
+            }
+
+            patterns = {
+                ("string", "NEW VALUE"): [
+                    "b.c",
+                    "d[*].*",
+                    "e.*.*",
+                ]
+            }
+
+            comparator = dir_content_diff.JsonComparator()
+            comparator.format_data(data)
+            assert data == initial_data
+
+            data = copy.deepcopy(initial_data)
+            comparator = dir_content_diff.JsonComparator()
+            comparator.format_data(data, replace_pattern=patterns)
+            assert data == expected_data
+
+            # Missing key in ref
+            comparator = dir_content_diff.JsonComparator()
+            data = copy.deepcopy(initial_data)
+            ref = {"a": 1}
+            comparator.format_data(data, ref, replace_pattern=patterns)
+            assert data == initial_data
+            assert comparator.current_state["format_errors"] == [
+                ("missing_ref_entry", i, None) for i in patterns[("string", "NEW VALUE")]
+            ]
+
+            # Missing key in data
+            comparator = dir_content_diff.JsonComparator()
+            ref = copy.deepcopy(initial_data)
+            data = {"a": 1}
+            comparator.format_data(data, ref, replace_pattern=patterns)
+            assert data == {"a": 1}
+            assert comparator.current_state["format_errors"] == [
+                ("missing_comp_entry", i, None) for i in patterns[("string", "NEW VALUE")]
+            ]
+
     class TestXmlComparator:
         """Test the XML comparator."""
 
@@ -655,22 +733,16 @@ def test_assert_equal_trees(self, ref_tree, res_tree_equal):
 
     def test_assert_equal_trees_export(self, ref_tree, res_tree_equal):
         """Test that the formatted files are properly exported."""
-
-        class JsonComparator(dir_content_diff.base_comparators.JsonComparator):
-            """Compare data from two JSON files."""
-
-            def save(self, data, path):
-                """Save formatted data into a file."""
-                with open(path, "w", encoding="utf-8") as file:
-                    json.dump(data, file)
-
-        comparators = dir_content_diff.get_comparators()
-        comparators[".json"] = JsonComparator()
         assert_equal_trees(
-            ref_tree, res_tree_equal, export_formatted_files=True, comparators=comparators
+            ref_tree,
+            res_tree_equal,
+            export_formatted_files=True,
         )
-        assert list(res_tree_equal.with_name(res_tree_equal.name + "_FORMATTED").iterdir()) == [
-            res_tree_equal.with_name(res_tree_equal.name + "_FORMATTED") / "file.json"
+        assert sorted(res_tree_equal.with_name(res_tree_equal.name + "_FORMATTED").iterdir()) == [
+            (res_tree_equal.with_name(res_tree_equal.name + "_FORMATTED") / "file").with_suffix(
+                suffix
+            )
+            for suffix in [".ini", ".json", ".xml", ".yaml"]
         ]
 
     def test_diff_empty(self, empty_ref_tree, empty_res_tree):
@@ -706,6 +778,30 @@ def test_specific_args(self, ref_tree, res_tree_equal):
 
         assert res == {}
 
+    def test_replace_pattern(self, ref_tree, res_tree_equal):
+        """Test specific args."""
+        specific_args = {
+            "file.yaml": {"args": [None, None, None, False, 0, False]},
+            "file.json": {
+                "format_data_kwargs": {
+                    "replace_pattern": {(".*val.*", "NEW_VAL"): ["*.[*]"]},
+                },
+            },
+        }
+        res = compare_trees(
+            ref_tree, res_tree_equal, specific_args=specific_args, export_formatted_files=True
+        )
+
+        pat = (
+            r"""The files '\S*/ref/file\.json' and '\S*/res/file\.json' are different:\n"""
+            r"""Kwargs used for formatting data: """
+            r"""{'replace_pattern': {\('\.\*val\.\*', 'NEW_VAL'\): \['\*\.\[\*\]'\]}}\n"""
+            r"""Changed the value of '\[nested_list\]\[2\]' from 'str_val' to 'NEW_VAL'\.\n"""
+            r"""Changed the value of '\[simple_list\]\[2\]' from 'str_val' to 'NEW_VAL'\."""
+        )
+
+        assert re.match(pat, res["file.json"]) is not None
+
     def test_specific_comparator(self, ref_tree, res_tree_equal):
         """Test specific args."""
         specific_args = {