From fbed747f9c3dc1ca15d8b0a6bb94f9354060821b Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Wed, 16 Apr 2025 12:29:55 -0500 Subject: [PATCH 01/11] create helpers for json compliance in GC --- jdiff/extract_data.py | 42 +++++++++++---- jdiff/utils/diff_helpers.py | 105 ++++++++++++++++++++++++++++++++++-- tests/test_diff_helpers.py | 98 ++++++++++++++++++++++++++++++--- 3 files changed, 223 insertions(+), 22 deletions(-) diff --git a/jdiff/extract_data.py b/jdiff/extract_data.py index 96188ec..cf36b32 100644 --- a/jdiff/extract_data.py +++ b/jdiff/extract_data.py @@ -1,19 +1,24 @@ """Extract data from JSON. Based on custom JMSPath implementation.""" + import re import warnings -from typing import Mapping, List, Dict, Any, Union, Optional +from typing import Any, Dict, List, Mapping, Optional, Union + import jmespath + from .utils.data_normalization import exclude_filter, flatten_list from .utils.jmespath_parsers import ( - jmespath_value_parser, - jmespath_refkey_parser, associate_key_of_my_value, + jmespath_refkey_parser, + jmespath_value_parser, keys_values_zipper, multi_reference_keys, ) -def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude: Optional[List] = None) -> Any: +def extract_data_from_json( + data: Union[Mapping, List], path: str = "*", exclude: Optional[List] = None +) -> Any: """Return wanted data from outpdevice data based on the check path. See unit test for complete example. Get the wanted values to be evaluated if JMESPath expression is defined, @@ -32,12 +37,16 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude: """ if exclude and isinstance(data, Dict): if not isinstance(exclude, list): - raise ValueError(f"Exclude list must be defined as a list. You have {type(exclude)}") + raise ValueError( + f"Exclude list must be defined as a list. You have {type(exclude)}" + ) # exclude unwanted elements exclude_filter(data, exclude) if not path: - warnings.warn("JMSPath cannot be empty string or type 'None'. Path argument reverted to default value '*'") + warnings.warn( + "JMSPath cannot be empty string or type 'None'. Path argument reverted to default value '*'" + ) path = "*" if path == "*": @@ -48,7 +57,10 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude: if len(re.findall(r"\$.*?\$", path)) > 1: clean_path = path.replace("$", "") values = jmespath.search(f"{clean_path}{' | []' * (path.count('*') - 1)}", data) - return keys_values_zipper(multi_reference_keys(path, data), associate_key_of_my_value(clean_path, values)) + return keys_values_zipper( + multi_reference_keys(path, data), + associate_key_of_my_value(clean_path, values), + ) values = jmespath.search(jmespath_value_parser(path), data) @@ -73,19 +85,27 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude: # Based on the expression or data we might have different data types # therefore we need to normalize. if re.search(r"\$.*\$", path): - paired_key_value = associate_key_of_my_value(jmespath_value_parser(path), values) + paired_key_value = associate_key_of_my_value( + jmespath_value_parser(path), values + ) wanted_reference_keys = jmespath.search(jmespath_refkey_parser(path), data) - if isinstance(wanted_reference_keys, dict): # when wanted_reference_keys is dict() type + if isinstance( + wanted_reference_keys, dict + ): # when wanted_reference_keys is dict() type list_of_reference_keys = list(wanted_reference_keys.keys()) elif any( isinstance(element, list) for element in wanted_reference_keys ): # when wanted_reference_keys is a nested list list_of_reference_keys = flatten_list(wanted_reference_keys)[0] - elif isinstance(wanted_reference_keys, list): # when wanted_reference_keys is a list + elif isinstance( + wanted_reference_keys, list + ): # when wanted_reference_keys is a list list_of_reference_keys = wanted_reference_keys else: - raise ValueError("Reference Key normalization failure. Please verify data type returned.") + raise ValueError( + "Reference Key normalization failure. Please verify data type returned." + ) normalized = keys_values_zipper(list_of_reference_keys, paired_key_value) # Data between pre and post may come in different order, so it needs to be sorted. diff --git a/jdiff/utils/diff_helpers.py b/jdiff/utils/diff_helpers.py index a22c4a6..073fca5 100644 --- a/jdiff/utils/diff_helpers.py +++ b/jdiff/utils/diff_helpers.py @@ -1,8 +1,9 @@ """Diff helpers.""" + import re from collections import defaultdict from functools import partial -from typing import Mapping, Dict, List, DefaultDict +from typing import DefaultDict, Dict, List, Mapping REGEX_PATTERN_RELEVANT_KEYS = r"'([A-Za-z0-9_\./\\-]*)'" @@ -62,8 +63,12 @@ def fix_deepdiff_key_names(obj: Mapping) -> Dict: result = {} # type: Dict for key, value in obj.items(): key_parts = re.findall(REGEX_PATTERN_RELEVANT_KEYS, key) - if not key_parts: # If key parts can't be find, keep original key so data is not lost. - key_parts = [key.replace("root", "index_element")] # replace root from DeepDiff with more meaningful name. + if ( + not key_parts + ): # If key parts can't be find, keep original key so data is not lost. + key_parts = [ + key.replace("root", "index_element") + ] # replace root from DeepDiff with more meaningful name. partial_res = group_value(key_parts, value) dict_merger(result, partial_res) return result @@ -79,9 +84,99 @@ def group_value(tree_list: List, value: Dict) -> Dict: def dict_merger(original_dict: Dict, dict_to_merge: Dict): """Function to merge a dictionary (dict_to_merge) recursively into the original_dict.""" for key in dict_to_merge.keys(): - if key in original_dict and isinstance(original_dict[key], dict) and isinstance(dict_to_merge[key], dict): + if ( + key in original_dict + and isinstance(original_dict[key], dict) + and isinstance(dict_to_merge[key], dict) + ): dict_merger(original_dict[key], dict_to_merge[key]) elif key in original_dict.keys(): - original_dict[key + "_dup!"] = dict_to_merge[key] # avoid overwriting existing keys. + original_dict[key + "_dup!"] = dict_to_merge[ + key + ] # avoid overwriting existing keys. else: original_dict[key] = dict_to_merge[key] + + +def _parse_index_element_string(index_element_string): + """Build out dictionary from the index element string.""" + result = {} + pattern = r"\[\'(.*?)\'\]" + match = re.findall(pattern, index_element_string) + if match: + for inner_key in match[1::]: + result[inner_key] = "" + return result + + +def parse_diff(jdiff_evaluate_response, actual, intended, match_config): + """Parse jdiff evaluate result into missing and extra dictionaries.""" + extra = {} + missing = {} + + def process_diff(_map, extra_map, missing_map): + for key, value in _map.items(): + if ( + isinstance(value, dict) + and "new_value" in value + and "old_value" in value + ): + extra_map[key] = value["old_value"] + missing_map[key] = value["new_value"] + elif isinstance(value, str): + if "missing" in value: + extra_map[key] = actual.get(match_config, {}).get(key) + if "new" in value: + new_key = _parse_index_element_string(key) + new_key[key] = intended.get(match_config, {}).get(key) + missing_map.update(new_key) + elif isinstance(value, dict): + extra_map[key] = {} + missing_map[key] = {} + process_diff(value, extra_map[key], missing_map[key]) + return extra_map, missing_map + + extras, missings = process_diff(jdiff_evaluate_response, extra, missing) + return extras, missings + + +# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing'} +# result = {'domain-name': 'missing'} +# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} +# result = {'domain-name': 'missing','hostname': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} +# result = {'servers': {'server': defaultdict(, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}} + +# ''' +# ``` +# from jdiff import CheckType + +# a = { +# "openconfig-system:ntp": { +# "servers": { +# "server": [ +# { +# "address": "1.us.pool.ntp.org", +# "config": { +# "address": "1.us.pool.ntp.org" +# }, +# "state": { +# "address": "1.us.pool.ntp.org" +# } +# } +# ] +# } +# } +# } + +# i = { +# "openconfig-system:ntp": { +# "servers": { +# "server": [] +# } +# } +# } + +# jdiff_param_match = CheckType.create("exact_match") +# result, compliant = jdiff_param_match.evaluate(a, i) +# ``` +# ''' diff --git a/tests/test_diff_helpers.py b/tests/test_diff_helpers.py index 82a23fa..d4b2a10 100644 --- a/tests/test_diff_helpers.py +++ b/tests/test_diff_helpers.py @@ -1,5 +1,14 @@ """DIff helpers unit tests.""" -from jdiff.utils.diff_helpers import dict_merger, group_value, fix_deepdiff_key_names, get_diff_iterables_items + +import pytest +from jdiff.utils.diff_helpers import ( + _parse_index_element_string, + dict_merger, + fix_deepdiff_key_names, + get_diff_iterables_items, + group_value, + parse_diff, +) def test_dict_merger(): @@ -21,12 +30,16 @@ def test_group_value(): """Tests that nested dict is recursively created.""" tree_list = ["10.1.0.0", "is_enabled"] value = {"new_value": False, "old_value": True} - assert group_value(tree_list, value) == {"10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}}} + assert group_value(tree_list, value) == { + "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} + } def test_fix_deepdiff_key_names(): """Tests that deepdiff return is parsed properly.""" - deepdiff_object = {"root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True}} + deepdiff_object = { + "root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True} + } assert fix_deepdiff_key_names(deepdiff_object) == { "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} } @@ -35,12 +48,85 @@ def test_fix_deepdiff_key_names(): def test_get_diff_iterables_items(): """Tests that deepdiff return is parsed properly.""" diff_result = { - "values_changed": {"root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"}}, + "values_changed": { + "root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"} + }, "iterable_item_added": { - "root['Ethernet3'][1]": {"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"}, + "root['Ethernet3'][1]": { + "hostname": "ios-xrv-unittest", + "port": "Gi0/0/0/0", + }, }, } result = get_diff_iterables_items(diff_result) assert list(dict(result).keys())[0] == "['Ethernet3']" - assert list(list(dict(result).values())[0].values())[0] == [{"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"}] + assert list(list(dict(result).values())[0].values())[0] == [ + {"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"} + ] + + +# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing'} +# result = {'domain-name': 'missing'} +# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} +# result = {'domain-name': 'missing','hostname': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} +# result = {'servers': {'server': defaultdict(, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}} + + +index_element_case_1 = ( + "index_element['foo']['ip name']", + {"ip name": ""}, +) + +index_element_case_2 = ( + "index_element['foo']['ip name']['ip domain']", + {"ip name": "", "ip domain": ""}, +) + + +index_element_tests = [index_element_case_1, index_element_case_2] + + +@pytest.mark.parametrize("index_element, result", index_element_tests) +def test__parse_index_element_string(index_element, result): + """Test that index_element can be unpacked.""" + parsed_result = _parse_index_element_string(index_element) + assert parsed_result == result + + +parse_diff_case_1 = ( + { + "hostname": {"new_value": "veos", "old_value": "veos-0"}, + "domain-name": "missing", + }, + {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, + {"openconfig-system:config": {"hostname": "veos"}}, + "openconfig-system:config", + {"hostname": "veos-0"}, + {"hostname": "veos", "domain-name": "ntc.com"}, +) + + +parse_diff_tests = [parse_diff_case_1] + + +@pytest.mark.parametrize( + "jdiff_evaluate_response, actual, intended, match_config, extra, missing", + parse_diff_tests, +) +def test_parse_diff( + jdiff_evaluate_response, actual, intended, match_config, extra, missing +): + """Test that index_element can be unpacked.""" + parsed_extra, parsed_missing = parse_diff( + jdiff_evaluate_response, + actual, + intended, + match_config, + ) + assert ( + parsed_extra == extra + ) # AssertionError: assert {'hostname': 'veos-0', 'domain-name': 'ntc.com'} == {'hostname': 'veos-0'} + assert ( + parsed_missing == missing + ) # AssertionError: assert {'hostname': 'veos'} == {'hostname': 'veos', 'domain-name': 'ntc.com'} From 90d5283be792df25d698266bd2f713b0de5cf706 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 8 May 2025 09:47:00 -0600 Subject: [PATCH 02/11] prep new jdiff compliance testing --- jdiff/__init__.py | 1 + jdiff/check_types.py | 1 + jdiff/evaluators.py | 1 + jdiff/extract_data.py | 28 ++--- jdiff/operator.py | 1 + jdiff/utils/data_normalization.py | 1 + jdiff/utils/diff_helpers.py | 137 ++++++++++++------------ jdiff/utils/jmespath_parsers.py | 1 + tasks.py | 1 + tests/test_diff_generator.py | 1 + tests/test_diff_helpers.py | 148 ++++++++++++++++++++------ tests/test_get_value.py | 1 + tests/test_jmespath_parsers.py | 1 + tests/test_operators.py | 1 + tests/test_sw_upgrade_device_state.py | 1 + tests/test_type_checks.py | 1 + tests/test_validates.py | 1 + 17 files changed, 208 insertions(+), 119 deletions(-) diff --git a/jdiff/__init__.py b/jdiff/__init__.py index 3d0a62b..37e2b6b 100644 --- a/jdiff/__init__.py +++ b/jdiff/__init__.py @@ -1,4 +1,5 @@ """Pre/Post Check library.""" + from .check_types import CheckType from .extract_data import extract_data_from_json diff --git a/jdiff/check_types.py b/jdiff/check_types.py index 8b1bd0d..a58f70f 100644 --- a/jdiff/check_types.py +++ b/jdiff/check_types.py @@ -1,4 +1,5 @@ """CheckType Implementation.""" + from typing import List, Tuple, Dict, Any, Union from abc import ABC, abstractmethod from .evaluators import diff_generator, parameter_evaluator, regex_evaluator, operator_evaluator diff --git a/jdiff/evaluators.py b/jdiff/evaluators.py index 6b9474a..eac23e6 100644 --- a/jdiff/evaluators.py +++ b/jdiff/evaluators.py @@ -1,4 +1,5 @@ """Evaluators.""" + import re from typing import Any, Mapping, Dict, Tuple, List from deepdiff import DeepDiff diff --git a/jdiff/extract_data.py b/jdiff/extract_data.py index cf36b32..50abc9e 100644 --- a/jdiff/extract_data.py +++ b/jdiff/extract_data.py @@ -16,9 +16,7 @@ ) -def extract_data_from_json( - data: Union[Mapping, List], path: str = "*", exclude: Optional[List] = None -) -> Any: +def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude: Optional[List] = None) -> Any: """Return wanted data from outpdevice data based on the check path. See unit test for complete example. Get the wanted values to be evaluated if JMESPath expression is defined, @@ -37,16 +35,12 @@ def extract_data_from_json( """ if exclude and isinstance(data, Dict): if not isinstance(exclude, list): - raise ValueError( - f"Exclude list must be defined as a list. You have {type(exclude)}" - ) + raise ValueError(f"Exclude list must be defined as a list. You have {type(exclude)}") # exclude unwanted elements exclude_filter(data, exclude) if not path: - warnings.warn( - "JMSPath cannot be empty string or type 'None'. Path argument reverted to default value '*'" - ) + warnings.warn("JMSPath cannot be empty string or type 'None'. Path argument reverted to default value '*'") path = "*" if path == "*": @@ -85,27 +79,19 @@ def extract_data_from_json( # Based on the expression or data we might have different data types # therefore we need to normalize. if re.search(r"\$.*\$", path): - paired_key_value = associate_key_of_my_value( - jmespath_value_parser(path), values - ) + paired_key_value = associate_key_of_my_value(jmespath_value_parser(path), values) wanted_reference_keys = jmespath.search(jmespath_refkey_parser(path), data) - if isinstance( - wanted_reference_keys, dict - ): # when wanted_reference_keys is dict() type + if isinstance(wanted_reference_keys, dict): # when wanted_reference_keys is dict() type list_of_reference_keys = list(wanted_reference_keys.keys()) elif any( isinstance(element, list) for element in wanted_reference_keys ): # when wanted_reference_keys is a nested list list_of_reference_keys = flatten_list(wanted_reference_keys)[0] - elif isinstance( - wanted_reference_keys, list - ): # when wanted_reference_keys is a list + elif isinstance(wanted_reference_keys, list): # when wanted_reference_keys is a list list_of_reference_keys = wanted_reference_keys else: - raise ValueError( - "Reference Key normalization failure. Please verify data type returned." - ) + raise ValueError("Reference Key normalization failure. Please verify data type returned.") normalized = keys_values_zipper(list_of_reference_keys, paired_key_value) # Data between pre and post may come in different order, so it needs to be sorted. diff --git a/jdiff/operator.py b/jdiff/operator.py index 6c07000..ffa3921 100644 --- a/jdiff/operator.py +++ b/jdiff/operator.py @@ -1,4 +1,5 @@ """Operator diff.""" + import operator from typing import Any, List, Tuple diff --git a/jdiff/utils/data_normalization.py b/jdiff/utils/data_normalization.py index 8a5e053..d658fa3 100644 --- a/jdiff/utils/data_normalization.py +++ b/jdiff/utils/data_normalization.py @@ -1,4 +1,5 @@ """Data Normalization utilities.""" + from typing import List, Generator, Union, Dict diff --git a/jdiff/utils/diff_helpers.py b/jdiff/utils/diff_helpers.py index 073fca5..1883c88 100644 --- a/jdiff/utils/diff_helpers.py +++ b/jdiff/utils/diff_helpers.py @@ -2,7 +2,8 @@ import re from collections import defaultdict -from functools import partial +from functools import partial, reduce +from operator import getitem from typing import DefaultDict, Dict, List, Mapping REGEX_PATTERN_RELEVANT_KEYS = r"'([A-Za-z0-9_\./\\-]*)'" @@ -63,12 +64,8 @@ def fix_deepdiff_key_names(obj: Mapping) -> Dict: result = {} # type: Dict for key, value in obj.items(): key_parts = re.findall(REGEX_PATTERN_RELEVANT_KEYS, key) - if ( - not key_parts - ): # If key parts can't be find, keep original key so data is not lost. - key_parts = [ - key.replace("root", "index_element") - ] # replace root from DeepDiff with more meaningful name. + if not key_parts: # If key parts can't be find, keep original key so data is not lost. + key_parts = [key.replace("root", "index_element")] # replace root from DeepDiff with more meaningful name. partial_res = group_value(key_parts, value) dict_merger(result, partial_res) return result @@ -84,16 +81,10 @@ def group_value(tree_list: List, value: Dict) -> Dict: def dict_merger(original_dict: Dict, dict_to_merge: Dict): """Function to merge a dictionary (dict_to_merge) recursively into the original_dict.""" for key in dict_to_merge.keys(): - if ( - key in original_dict - and isinstance(original_dict[key], dict) - and isinstance(dict_to_merge[key], dict) - ): + if key in original_dict and isinstance(original_dict[key], dict) and isinstance(dict_to_merge[key], dict): dict_merger(original_dict[key], dict_to_merge[key]) elif key in original_dict.keys(): - original_dict[key + "_dup!"] = dict_to_merge[ - key - ] # avoid overwriting existing keys. + original_dict[key + "_dup!"] = dict_to_merge[key] # avoid overwriting existing keys. else: original_dict[key] = dict_to_merge[key] @@ -106,7 +97,45 @@ def _parse_index_element_string(index_element_string): if match: for inner_key in match[1::]: result[inner_key] = "" - return result + return match, result + + +def set_nested_value(data, keys, value): + """ + Recursively sets a value in a nested dictionary, given a list of keys. + + Args: + data (dict): The nested dictionary to modify. + keys (list): A list of keys to access the target value. + value: The value to set. + + Returns: + None: The function modifies the dictionary in place. Returns None. + """ + if not keys: + return # Should not happen, but good to have. + if len(keys) == 1: + data[keys[0]] = value + else: + if keys[0] not in data: + data[keys[0]] = {} # Create the nested dictionary if it doesn't exist + set_nested_value(data[keys[0]], keys[1:], value) + + +def all_values_empty(input_dict): + """ + Checks if all values in a dictionary are empty objects (empty string, list, or dictionary). + + Args: + input_dict: The dictionary to check. + + Returns: + True if all values are empty, False otherwise. + """ + for value in input_dict.values(): + if value: # Empty objects evaluate to False in a boolean context + return False + return True def parse_diff(jdiff_evaluate_response, actual, intended, match_config): @@ -116,67 +145,39 @@ def parse_diff(jdiff_evaluate_response, actual, intended, match_config): def process_diff(_map, extra_map, missing_map): for key, value in _map.items(): - if ( - isinstance(value, dict) - and "new_value" in value - and "old_value" in value - ): + if isinstance(value, dict) and "new_value" in value and "old_value" in value: extra_map[key] = value["old_value"] missing_map[key] = value["new_value"] elif isinstance(value, str): if "missing" in value: extra_map[key] = actual.get(match_config, {}).get(key) if "new" in value: - new_key = _parse_index_element_string(key) - new_key[key] = intended.get(match_config, {}).get(key) - missing_map.update(new_key) + key_chain, _ = _parse_index_element_string(key) + new_value = reduce(getitem, key_chain, intended) + set_nested_value(missing_map, key_chain[1::], new_value) + elif isinstance(value, defaultdict): + if dict(value).get("new"): + missing[key] = dict(value).get("new", {}) + if dict(value).get("missing"): + extra_map[key] = dict(value).get("missing", {}) elif isinstance(value, dict): extra_map[key] = {} missing_map[key] = {} process_diff(value, extra_map[key], missing_map[key]) return extra_map, missing_map - extras, missings = process_diff(jdiff_evaluate_response, extra, missing) - return extras, missings - - -# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing'} -# result = {'domain-name': 'missing'} -# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} -# result = {'domain-name': 'missing','hostname': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} -# result = {'servers': {'server': defaultdict(, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}} - -# ''' -# ``` -# from jdiff import CheckType - -# a = { -# "openconfig-system:ntp": { -# "servers": { -# "server": [ -# { -# "address": "1.us.pool.ntp.org", -# "config": { -# "address": "1.us.pool.ntp.org" -# }, -# "state": { -# "address": "1.us.pool.ntp.org" -# } -# } -# ] -# } -# } -# } - -# i = { -# "openconfig-system:ntp": { -# "servers": { -# "server": [] -# } -# } -# } - -# jdiff_param_match = CheckType.create("exact_match") -# result, compliant = jdiff_param_match.evaluate(a, i) -# ``` -# ''' + extras, missing = process_diff(jdiff_evaluate_response, extra, missing) + # Don't like this, but with less the performant way of doing it right now it works to clear out + # Any empty dicts that are left over from the diff. + # This is a bit of a hack, but it works for now. + final_extras = extras.copy() + final_missing = missing.copy() + for key, value in extras.items(): + if isinstance(value, dict): + if not value: + del final_extras[key] + for key, value in missing.items(): + if isinstance(value, dict): + if not value: + del final_missing[key] + return final_extras, final_missing diff --git a/jdiff/utils/jmespath_parsers.py b/jdiff/utils/jmespath_parsers.py index ee1e669..b35b839 100644 --- a/jdiff/utils/jmespath_parsers.py +++ b/jdiff/utils/jmespath_parsers.py @@ -5,6 +5,7 @@ From one expression defined in jdiff, we will derive two expressions: one expression that traverse the json output and get the evaluated bit of it, the second will target the reference key relative to the value to evaluate. More on README.md """ + import re from typing import Mapping, List, Union diff --git a/tasks.py b/tasks.py index c9bc272..635d0a4 100644 --- a/tasks.py +++ b/tasks.py @@ -1,4 +1,5 @@ """Tasks for use with Invoke.""" + import os import sys from distutils.util import strtobool diff --git a/tests/test_diff_generator.py b/tests/test_diff_generator.py index 9c0892d..a9c3412 100644 --- a/tests/test_diff_generator.py +++ b/tests/test_diff_generator.py @@ -1,4 +1,5 @@ """Diff generator tests.""" + import pytest from jdiff.evaluators import diff_generator from jdiff import extract_data_from_json diff --git a/tests/test_diff_helpers.py b/tests/test_diff_helpers.py index d4b2a10..fe8c045 100644 --- a/tests/test_diff_helpers.py +++ b/tests/test_diff_helpers.py @@ -1,6 +1,9 @@ """DIff helpers unit tests.""" +from collections import defaultdict + import pytest + from jdiff.utils.diff_helpers import ( _parse_index_element_string, dict_merger, @@ -30,16 +33,12 @@ def test_group_value(): """Tests that nested dict is recursively created.""" tree_list = ["10.1.0.0", "is_enabled"] value = {"new_value": False, "old_value": True} - assert group_value(tree_list, value) == { - "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} - } + assert group_value(tree_list, value) == {"10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}}} def test_fix_deepdiff_key_names(): """Tests that deepdiff return is parsed properly.""" - deepdiff_object = { - "root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True} - } + deepdiff_object = {"root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True}} assert fix_deepdiff_key_names(deepdiff_object) == { "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} } @@ -48,9 +47,7 @@ def test_fix_deepdiff_key_names(): def test_get_diff_iterables_items(): """Tests that deepdiff return is parsed properly.""" diff_result = { - "values_changed": { - "root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"} - }, + "values_changed": {"root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"}}, "iterable_item_added": { "root['Ethernet3'][1]": { "hostname": "ios-xrv-unittest", @@ -61,16 +58,7 @@ def test_get_diff_iterables_items(): result = get_diff_iterables_items(diff_result) assert list(dict(result).keys())[0] == "['Ethernet3']" - assert list(list(dict(result).values())[0].values())[0] == [ - {"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"} - ] - - -# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing'} -# result = {'domain-name': 'missing'} -# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} -# result = {'domain-name': 'missing','hostname': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'} -# result = {'servers': {'server': defaultdict(, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}} + assert list(list(dict(result).values())[0].values())[0] == [{"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"}] index_element_case_1 = ( @@ -90,7 +78,7 @@ def test_get_diff_iterables_items(): @pytest.mark.parametrize("index_element, result", index_element_tests) def test__parse_index_element_string(index_element, result): """Test that index_element can be unpacked.""" - parsed_result = _parse_index_element_string(index_element) + _, parsed_result = _parse_index_element_string(index_element) assert parsed_result == result @@ -102,12 +90,116 @@ def test__parse_index_element_string(index_element, result): {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, {"openconfig-system:config": {"hostname": "veos"}}, "openconfig-system:config", - {"hostname": "veos-0"}, - {"hostname": "veos", "domain-name": "ntc.com"}, + {"hostname": "veos-0", "domain-name": "ntc.com"}, + {"hostname": "veos"}, +) + +parse_diff_case_2 = ( + { + "hostname": {"new_value": "veos", "old_value": "veos-0"}, + "domain-name": "missing", + "index_element['openconfig-system:config']['ip name']": "new", + }, + {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, + {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, + "openconfig-system:config", + {"domain-name": "ntc.com", "hostname": "veos-0"}, + {"hostname": "veos", "ip name": "ntc.com"}, ) +parse_diff_case_3 = ( + { + "domain-name": "missing", + "hostname": "missing", + "index_element['openconfig-system:config']['ip name']": "new", + }, + {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, + {"openconfig-system:config": {"ip name": "ntc.com"}}, + "openconfig-system:config", + {"domain-name": "ntc.com", "hostname": "veos-0"}, + {"ip name": "ntc.com"}, +) + +parse_diff_case_4 = ( + {"domain-name": "missing"}, + {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos"}}, + {"openconfig-system:config": {"hostname": "veos"}}, + "openconfig-system:config", + {"domain-name": "ntc.com"}, + {}, +) + +parse_diff_case_5 = ( + { + "hostname": {"new_value": "veos", "old_value": "veos-0"}, + "domain-name": "missing", + "index_element['openconfig-system:config']['ip name']": "new", + }, + {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, + {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, + "openconfig-system:config", + {"hostname": "veos-0", "domain-name": "ntc.com"}, + {"ip name": "ntc.com", "hostname": "veos"}, +) + +parse_diff_case_6 = ( + { + "servers": { + "server": defaultdict( + list, + { + "new": [ + { + "address": "1.us.pool.ntp.org", + "config": {"address": "1.us.pool.ntp.org"}, + "state": {"address": "1.us.pool.ntp.org"}, + } + ] + }, + ) + } + }, + {"openconfig-system:ntp": {"servers": {"server": []}}}, + { + "openconfig-system:ntp": { + "servers": { + "server": [ + { + "address": "1.us.pool.ntp.org", + "config": {"address": "1.us.pool.ntp.org"}, + "state": {"address": "1.us.pool.ntp.org"}, + } + ] + } + } + }, + "openconfig-system:ntp", + {}, + { + "servers": { + "server": [ + { + "address": "1.us.pool.ntp.org", + "config": { + "address": "1.us.pool.ntp.org", + }, + "state": { + "address": "1.us.pool.ntp.org", + }, + }, + ], + }, + }, +) -parse_diff_tests = [parse_diff_case_1] +parse_diff_tests = [ + parse_diff_case_1, + parse_diff_case_2, + parse_diff_case_3, + parse_diff_case_4, + parse_diff_case_5, + parse_diff_case_6, +] @pytest.mark.parametrize( @@ -116,7 +208,7 @@ def test__parse_index_element_string(index_element, result): ) def test_parse_diff( jdiff_evaluate_response, actual, intended, match_config, extra, missing -): +): # pylint: disable=too-many-arguments """Test that index_element can be unpacked.""" parsed_extra, parsed_missing = parse_diff( jdiff_evaluate_response, @@ -124,9 +216,5 @@ def test_parse_diff( intended, match_config, ) - assert ( - parsed_extra == extra - ) # AssertionError: assert {'hostname': 'veos-0', 'domain-name': 'ntc.com'} == {'hostname': 'veos-0'} - assert ( - parsed_missing == missing - ) # AssertionError: assert {'hostname': 'veos'} == {'hostname': 'veos', 'domain-name': 'ntc.com'} + assert parsed_extra == extra + assert parsed_missing == missing diff --git a/tests/test_get_value.py b/tests/test_get_value.py index e437c55..55f187d 100644 --- a/tests/test_get_value.py +++ b/tests/test_get_value.py @@ -1,4 +1,5 @@ """Test extract_data_from_json.""" + import pytest from jdiff import extract_data_from_json from .utility import load_json_file, ASSERT_FAIL_MESSAGE diff --git a/tests/test_jmespath_parsers.py b/tests/test_jmespath_parsers.py index 972acc1..c0f6a9a 100644 --- a/tests/test_jmespath_parsers.py +++ b/tests/test_jmespath_parsers.py @@ -1,4 +1,5 @@ """jmespath parser unit tests.""" + import pytest from jdiff.utils.jmespath_parsers import ( jmespath_value_parser, diff --git a/tests/test_operators.py b/tests/test_operators.py index 3d5fd1f..6748752 100644 --- a/tests/test_operators.py +++ b/tests/test_operators.py @@ -1,4 +1,5 @@ """Unit tests for operator check-type.""" + import pytest from jdiff import CheckType, extract_data_from_json from .utility import load_json_file, ASSERT_FAIL_MESSAGE diff --git a/tests/test_sw_upgrade_device_state.py b/tests/test_sw_upgrade_device_state.py index 05c34b8..498c175 100644 --- a/tests/test_sw_upgrade_device_state.py +++ b/tests/test_sw_upgrade_device_state.py @@ -1,4 +1,5 @@ """Tests for typical software upgrade device state check.""" + from copy import deepcopy import pytest from jdiff import CheckType, extract_data_from_json diff --git a/tests/test_type_checks.py b/tests/test_type_checks.py index 7345cde..17d9269 100644 --- a/tests/test_type_checks.py +++ b/tests/test_type_checks.py @@ -1,4 +1,5 @@ """Check Type unit tests.""" + import pytest from jdiff.check_types import ( CheckType, diff --git a/tests/test_validates.py b/tests/test_validates.py index c10a8ff..7ece4ce 100644 --- a/tests/test_validates.py +++ b/tests/test_validates.py @@ -1,4 +1,5 @@ """Unit tests for validator CheckType method.""" + import pytest from jdiff import CheckType From 59e4239d71a651297e89fc2a64275d54e27bd2d8 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 8 May 2025 11:30:59 -0600 Subject: [PATCH 03/11] fix last failing test --- jdiff/utils/diff_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jdiff/utils/diff_helpers.py b/jdiff/utils/diff_helpers.py index 1883c88..0bbbf81 100644 --- a/jdiff/utils/diff_helpers.py +++ b/jdiff/utils/diff_helpers.py @@ -143,7 +143,7 @@ def parse_diff(jdiff_evaluate_response, actual, intended, match_config): extra = {} missing = {} - def process_diff(_map, extra_map, missing_map): + def process_diff(_map, extra_map, missing_map, previous_key=None): for key, value in _map.items(): if isinstance(value, dict) and "new_value" in value and "old_value" in value: extra_map[key] = value["old_value"] @@ -157,13 +157,13 @@ def process_diff(_map, extra_map, missing_map): set_nested_value(missing_map, key_chain[1::], new_value) elif isinstance(value, defaultdict): if dict(value).get("new"): - missing[key] = dict(value).get("new", {}) + missing[previous_key][key] = dict(value).get("new", {}) if dict(value).get("missing"): - extra_map[key] = dict(value).get("missing", {}) + extra_map[previous_key][key] = dict(value).get("missing", {}) elif isinstance(value, dict): extra_map[key] = {} missing_map[key] = {} - process_diff(value, extra_map[key], missing_map[key]) + process_diff(value, extra_map[key], missing_map[key], previous_key=key) return extra_map, missing_map extras, missing = process_diff(jdiff_evaluate_response, extra, missing) From b740207966bb7f101bd35207d6b0ccc9564c56df Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 8 May 2025 11:33:33 -0600 Subject: [PATCH 04/11] fix last failing test --- jdiff/utils/diff_helpers.py | 40 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/jdiff/utils/diff_helpers.py b/jdiff/utils/diff_helpers.py index 0bbbf81..60f190f 100644 --- a/jdiff/utils/diff_helpers.py +++ b/jdiff/utils/diff_helpers.py @@ -64,8 +64,12 @@ def fix_deepdiff_key_names(obj: Mapping) -> Dict: result = {} # type: Dict for key, value in obj.items(): key_parts = re.findall(REGEX_PATTERN_RELEVANT_KEYS, key) - if not key_parts: # If key parts can't be find, keep original key so data is not lost. - key_parts = [key.replace("root", "index_element")] # replace root from DeepDiff with more meaningful name. + if ( + not key_parts + ): # If key parts can't be find, keep original key so data is not lost. + key_parts = [ + key.replace("root", "index_element") + ] # replace root from DeepDiff with more meaningful name. partial_res = group_value(key_parts, value) dict_merger(result, partial_res) return result @@ -81,10 +85,16 @@ def group_value(tree_list: List, value: Dict) -> Dict: def dict_merger(original_dict: Dict, dict_to_merge: Dict): """Function to merge a dictionary (dict_to_merge) recursively into the original_dict.""" for key in dict_to_merge.keys(): - if key in original_dict and isinstance(original_dict[key], dict) and isinstance(dict_to_merge[key], dict): + if ( + key in original_dict + and isinstance(original_dict[key], dict) + and isinstance(dict_to_merge[key], dict) + ): dict_merger(original_dict[key], dict_to_merge[key]) elif key in original_dict.keys(): - original_dict[key + "_dup!"] = dict_to_merge[key] # avoid overwriting existing keys. + original_dict[key + "_dup!"] = dict_to_merge[ + key + ] # avoid overwriting existing keys. else: original_dict[key] = dict_to_merge[key] @@ -122,22 +132,6 @@ def set_nested_value(data, keys, value): set_nested_value(data[keys[0]], keys[1:], value) -def all_values_empty(input_dict): - """ - Checks if all values in a dictionary are empty objects (empty string, list, or dictionary). - - Args: - input_dict: The dictionary to check. - - Returns: - True if all values are empty, False otherwise. - """ - for value in input_dict.values(): - if value: # Empty objects evaluate to False in a boolean context - return False - return True - - def parse_diff(jdiff_evaluate_response, actual, intended, match_config): """Parse jdiff evaluate result into missing and extra dictionaries.""" extra = {} @@ -145,7 +139,11 @@ def parse_diff(jdiff_evaluate_response, actual, intended, match_config): def process_diff(_map, extra_map, missing_map, previous_key=None): for key, value in _map.items(): - if isinstance(value, dict) and "new_value" in value and "old_value" in value: + if ( + isinstance(value, dict) + and "new_value" in value + and "old_value" in value + ): extra_map[key] = value["old_value"] missing_map[key] = value["new_value"] elif isinstance(value, str): From 00ee630471bf7da2fd079f7c5cbf1a57bd3b5d2e Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Fri, 8 Aug 2025 08:08:03 -0600 Subject: [PATCH 05/11] more testing --- tests/test_diff_helpers.py | 75 +++++++++++++++----------------------- 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/tests/test_diff_helpers.py b/tests/test_diff_helpers.py index fe8c045..0bf505d 100644 --- a/tests/test_diff_helpers.py +++ b/tests/test_diff_helpers.py @@ -1,9 +1,8 @@ """DIff helpers unit tests.""" -from collections import defaultdict - import pytest +from jdiff.check_types import CheckType from jdiff.utils.diff_helpers import ( _parse_index_element_string, dict_merger, @@ -33,12 +32,16 @@ def test_group_value(): """Tests that nested dict is recursively created.""" tree_list = ["10.1.0.0", "is_enabled"] value = {"new_value": False, "old_value": True} - assert group_value(tree_list, value) == {"10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}}} + assert group_value(tree_list, value) == { + "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} + } def test_fix_deepdiff_key_names(): """Tests that deepdiff return is parsed properly.""" - deepdiff_object = {"root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True}} + deepdiff_object = { + "root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True} + } assert fix_deepdiff_key_names(deepdiff_object) == { "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} } @@ -47,7 +50,9 @@ def test_fix_deepdiff_key_names(): def test_get_diff_iterables_items(): """Tests that deepdiff return is parsed properly.""" diff_result = { - "values_changed": {"root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"}}, + "values_changed": { + "root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"} + }, "iterable_item_added": { "root['Ethernet3'][1]": { "hostname": "ios-xrv-unittest", @@ -58,7 +63,9 @@ def test_get_diff_iterables_items(): result = get_diff_iterables_items(diff_result) assert list(dict(result).keys())[0] == "['Ethernet3']" - assert list(list(dict(result).values())[0].values())[0] == [{"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"}] + assert list(list(dict(result).values())[0].values())[0] == [ + {"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"} + ] index_element_case_1 = ( @@ -82,11 +89,15 @@ def test__parse_index_element_string(index_element, result): assert parsed_result == result +parse_diff_simple_1 = ( + {"foo": {"bar-1": "baz1"}}, # actual + {"foo": {"bar-2": "baz2"}}, # intended + "foo", # match_config + {"bar-1": "baz1"}, # extra + {"bar-2": "baz2"}, # missing +) + parse_diff_case_1 = ( - { - "hostname": {"new_value": "veos", "old_value": "veos-0"}, - "domain-name": "missing", - }, {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, {"openconfig-system:config": {"hostname": "veos"}}, "openconfig-system:config", @@ -95,11 +106,6 @@ def test__parse_index_element_string(index_element, result): ) parse_diff_case_2 = ( - { - "hostname": {"new_value": "veos", "old_value": "veos-0"}, - "domain-name": "missing", - "index_element['openconfig-system:config']['ip name']": "new", - }, {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, "openconfig-system:config", @@ -108,11 +114,6 @@ def test__parse_index_element_string(index_element, result): ) parse_diff_case_3 = ( - { - "domain-name": "missing", - "hostname": "missing", - "index_element['openconfig-system:config']['ip name']": "new", - }, {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, {"openconfig-system:config": {"ip name": "ntc.com"}}, "openconfig-system:config", @@ -121,7 +122,6 @@ def test__parse_index_element_string(index_element, result): ) parse_diff_case_4 = ( - {"domain-name": "missing"}, {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos"}}, {"openconfig-system:config": {"hostname": "veos"}}, "openconfig-system:config", @@ -130,11 +130,6 @@ def test__parse_index_element_string(index_element, result): ) parse_diff_case_5 = ( - { - "hostname": {"new_value": "veos", "old_value": "veos-0"}, - "domain-name": "missing", - "index_element['openconfig-system:config']['ip name']": "new", - }, {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, "openconfig-system:config", @@ -143,22 +138,6 @@ def test__parse_index_element_string(index_element, result): ) parse_diff_case_6 = ( - { - "servers": { - "server": defaultdict( - list, - { - "new": [ - { - "address": "1.us.pool.ntp.org", - "config": {"address": "1.us.pool.ntp.org"}, - "state": {"address": "1.us.pool.ntp.org"}, - } - ] - }, - ) - } - }, {"openconfig-system:ntp": {"servers": {"server": []}}}, { "openconfig-system:ntp": { @@ -193,6 +172,7 @@ def test__parse_index_element_string(index_element, result): ) parse_diff_tests = [ + parse_diff_simple_1, parse_diff_case_1, parse_diff_case_2, parse_diff_case_3, @@ -203,18 +183,21 @@ def test__parse_index_element_string(index_element, result): @pytest.mark.parametrize( - "jdiff_evaluate_response, actual, intended, match_config, extra, missing", + "actual, intended, match_config, extra, missing", parse_diff_tests, ) -def test_parse_diff( - jdiff_evaluate_response, actual, intended, match_config, extra, missing -): # pylint: disable=too-many-arguments +def test_parse_diff(actual, intended, match_config, extra, missing): # pylint: disable=too-many-arguments """Test that index_element can be unpacked.""" + jdiff_param_match = CheckType.create("exact_match") + jdiff_evaluate_response, _ = jdiff_param_match.evaluate(actual, intended) + print(jdiff_evaluate_response) + parsed_extra, parsed_missing = parse_diff( jdiff_evaluate_response, actual, intended, match_config, ) + print(parsed_extra, parsed_missing) assert parsed_extra == extra assert parsed_missing == missing From 4a776f5225210212f3fe083ca97871602df4dc0b Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 14 Aug 2025 00:06:28 -0500 Subject: [PATCH 06/11] updates --- .github/workflows/ci.yml | 18 ++++++++++++++++ jdiff/utils/diff_helpers.py | 29 +++++++++---------------- tests/test_diff_helpers.py | 22 ++++++------------- towncrier_templates.j2 | 42 +++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 35 deletions(-) create mode 100644 towncrier_templates.j2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccec4f3..63569e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,6 +137,24 @@ jobs: poetry-install-options: "" - name: "Run Tests" run: "poetry run invoke pytest" + changelog: + if: > + contains(fromJson('["develop"]'), github.base_ref) && + (github.head_ref != 'main') && (!startsWith(github.head_ref, 'release')) + runs-on: "ubuntu-22.04" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + with: + fetch-depth: "0" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v6" + with: + poetry-version: "1.8.5" + - name: "Check for changelog entry" + run: | + git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} + poetry run towncrier check --compare-with origin/${{ github.base_ref }} publish_gh: needs: - "pytest" diff --git a/jdiff/utils/diff_helpers.py b/jdiff/utils/diff_helpers.py index 60f190f..90d73cf 100644 --- a/jdiff/utils/diff_helpers.py +++ b/jdiff/utils/diff_helpers.py @@ -64,12 +64,8 @@ def fix_deepdiff_key_names(obj: Mapping) -> Dict: result = {} # type: Dict for key, value in obj.items(): key_parts = re.findall(REGEX_PATTERN_RELEVANT_KEYS, key) - if ( - not key_parts - ): # If key parts can't be find, keep original key so data is not lost. - key_parts = [ - key.replace("root", "index_element") - ] # replace root from DeepDiff with more meaningful name. + if not key_parts: # If key parts can't be find, keep original key so data is not lost. + key_parts = [key.replace("root", "index_element")] # replace root from DeepDiff with more meaningful name. partial_res = group_value(key_parts, value) dict_merger(result, partial_res) return result @@ -85,16 +81,10 @@ def group_value(tree_list: List, value: Dict) -> Dict: def dict_merger(original_dict: Dict, dict_to_merge: Dict): """Function to merge a dictionary (dict_to_merge) recursively into the original_dict.""" for key in dict_to_merge.keys(): - if ( - key in original_dict - and isinstance(original_dict[key], dict) - and isinstance(dict_to_merge[key], dict) - ): + if key in original_dict and isinstance(original_dict[key], dict) and isinstance(dict_to_merge[key], dict): dict_merger(original_dict[key], dict_to_merge[key]) elif key in original_dict.keys(): - original_dict[key + "_dup!"] = dict_to_merge[ - key - ] # avoid overwriting existing keys. + original_dict[key + "_dup!"] = dict_to_merge[key] # avoid overwriting existing keys. else: original_dict[key] = dict_to_merge[key] @@ -132,6 +122,7 @@ def set_nested_value(data, keys, value): set_nested_value(data[keys[0]], keys[1:], value) +# {'foo': {'bar-1': 'missing', 'bar-2': 'new'}} def parse_diff(jdiff_evaluate_response, actual, intended, match_config): """Parse jdiff evaluate result into missing and extra dictionaries.""" extra = {} @@ -139,17 +130,17 @@ def parse_diff(jdiff_evaluate_response, actual, intended, match_config): def process_diff(_map, extra_map, missing_map, previous_key=None): for key, value in _map.items(): - if ( - isinstance(value, dict) - and "new_value" in value - and "old_value" in value - ): + print("value", value) + print("type(value)", type(value)) + if isinstance(value, dict) and "new_value" in value and "old_value" in value: extra_map[key] = value["old_value"] missing_map[key] = value["new_value"] elif isinstance(value, str): if "missing" in value: + print("missing", value) extra_map[key] = actual.get(match_config, {}).get(key) if "new" in value: + print("new", value) key_chain, _ = _parse_index_element_string(key) new_value = reduce(getitem, key_chain, intended) set_nested_value(missing_map, key_chain[1::], new_value) diff --git a/tests/test_diff_helpers.py b/tests/test_diff_helpers.py index 0bf505d..bd9d27e 100644 --- a/tests/test_diff_helpers.py +++ b/tests/test_diff_helpers.py @@ -32,16 +32,12 @@ def test_group_value(): """Tests that nested dict is recursively created.""" tree_list = ["10.1.0.0", "is_enabled"] value = {"new_value": False, "old_value": True} - assert group_value(tree_list, value) == { - "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} - } + assert group_value(tree_list, value) == {"10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}}} def test_fix_deepdiff_key_names(): """Tests that deepdiff return is parsed properly.""" - deepdiff_object = { - "root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True} - } + deepdiff_object = {"root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True}} assert fix_deepdiff_key_names(deepdiff_object) == { "10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}} } @@ -50,9 +46,7 @@ def test_fix_deepdiff_key_names(): def test_get_diff_iterables_items(): """Tests that deepdiff return is parsed properly.""" diff_result = { - "values_changed": { - "root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"} - }, + "values_changed": {"root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"}}, "iterable_item_added": { "root['Ethernet3'][1]": { "hostname": "ios-xrv-unittest", @@ -63,9 +57,7 @@ def test_get_diff_iterables_items(): result = get_diff_iterables_items(diff_result) assert list(dict(result).keys())[0] == "['Ethernet3']" - assert list(list(dict(result).values())[0].values())[0] == [ - {"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"} - ] + assert list(list(dict(result).values())[0].values())[0] == [{"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"}] index_element_case_1 = ( @@ -93,8 +85,8 @@ def test__parse_index_element_string(index_element, result): {"foo": {"bar-1": "baz1"}}, # actual {"foo": {"bar-2": "baz2"}}, # intended "foo", # match_config - {"bar-1": "baz1"}, # extra - {"bar-2": "baz2"}, # missing + {"foo": {"bar-1": "baz1"}}, # extra + {"foo": {"bar-2": "baz2"}}, # missing ) parse_diff_case_1 = ( @@ -190,7 +182,6 @@ def test_parse_diff(actual, intended, match_config, extra, missing): # pylint: """Test that index_element can be unpacked.""" jdiff_param_match = CheckType.create("exact_match") jdiff_evaluate_response, _ = jdiff_param_match.evaluate(actual, intended) - print(jdiff_evaluate_response) parsed_extra, parsed_missing = parse_diff( jdiff_evaluate_response, @@ -198,6 +189,5 @@ def test_parse_diff(actual, intended, match_config, extra, missing): # pylint: intended, match_config, ) - print(parsed_extra, parsed_missing) assert parsed_extra == extra assert parsed_missing == missing diff --git a/towncrier_templates.j2 b/towncrier_templates.j2 new file mode 100644 index 0000000..46407b0 --- /dev/null +++ b/towncrier_templates.j2 @@ -0,0 +1,42 @@ + +# v{{ versiondata.version.split(".")[:2] | join(".") }} Release Notes + +This document describes all new features and changes in the release. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Release Overview + +- Major features or milestones +- Changes to compatibility with Nautobot and/or other apps, libraries etc. + +{% if render_title %} +## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/networktocode/{{ cookiecutter.project_slug }}/releases/tag/v{{ versiondata.version}}) + +{% endif %} +{% for section, _ in sections.items() %} +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section] %} +{% if sections[section][category]|length != 0 %} +### {{ definitions[category]['name'] }} + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category].items() %} +{% for item in text.split('\n') %} +{% if values %} +- {{ values|join(', ') }} - {{ item.strip() }} +{% else %} +- {{ item.strip() }} +{% endif %} +{% endfor %} +{% endfor %} + +{% else %} +- {{ sections[section][category]['']|join(', ') }} + +{% endif %} +{% endif %} +{% endfor %} +{% else %} +No significant changes. + +{% endif %} +{% endfor %} From cac4d5c23d04d5f8827fbba81dc53dc8d461f864 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Fri, 22 Aug 2025 12:34:30 -0600 Subject: [PATCH 07/11] JSON data reconstruction for remediation --- docs/user/usage.md | 117 ++++++++++++++++++++++++++++++++++++ jdiff/utils/diff_helpers.py | 59 +++++++++++------- tests/test_diff_helpers.py | 38 ++++++------ 3 files changed, 175 insertions(+), 39 deletions(-) diff --git a/docs/user/usage.md b/docs/user/usage.md index ce7692a..f96e6d0 100644 --- a/docs/user/usage.md +++ b/docs/user/usage.md @@ -610,3 +610,120 @@ Can you guess what would be the outcome for an `int`, `float` operator? ``` See `tests` folder in the repo for more examples. + +## Putting a Result Back Together + +Jdiff results are very helpful in determining what is wrong with the outputs. What if you want to reconstruct the results in order to fix the problem. The `parse_diff` helper does just that. Imagine you have a `jdiff` result such as: + +Examples of jdiff evaluated results: + +```python +ex1 = {'bar-2': 'missing', 'bar-1': 'new'} +ex2 = { + 'hostname': {'new_value': 'veos-actual', 'old_value': 'veos-intended'}, + 'domain-name': 'new' + } +ex3 = { + 'hostname': {'new_value': 'veos-0', 'old_value': 'veos'}, + "index_element['ip name']": 'missing', + 'domain-name': 'new' + } +ex4 = { + 'servers': + { + 'server': defaultdict(, + { + 'missing': [ + { + 'address': '1.us.pool.ntp.org', + 'config': {'address': '1.us.pool.ntp.org'}, + 'state': {'address': '1.us.pool.ntp.org'} + } + ] + } + ) + } + } +``` + +And you need to understand what is extra and what is missing from the result. (Think configuration compliance on a JSON/JSON-RPC system). + +Well running the `parse_diff` will give you what is extra (in the comparison data) and missing from the reference data, and also the reverse. What is missing (in the reference data) that is missing from the comparison data. + +An example will help visualize the results. + +```python +In [1]: from jdiff import extract_data_from_json + ...: from jdiff.check_types import CheckType + ...: from jdiff.utils.diff_helpers import parse_diff + +In [2]: reference_data = {"foo": {"bar-2": "baz2"}} + ...: comparison_data = {"foo": {"bar-1": "baz1"}} + ...: match_key = "foo" + +In [3]: extracted_comparison_data = extract_data_from_json(comparison_data, match_key) + +In [4]: extracted_comparison_data +Out[4]: {'bar-1': 'baz1'} + +In [5]: extracted_reference_data = extract_data_from_json(reference_data, match_key) + +In [6]: extracted_reference_data +Out[6]: {'bar-2': 'baz2'} + +In [7]: jdiff_exact_match = CheckType.create("exact_match") + ...: jdiff_evaluate_response, _ = jdiff_exact_match.evaluate(extracted_reference_data, extracted_comparison_data) + +In [8]: jdiff_evaluate_response +Out[8]: {'bar-2': 'missing', 'bar-1': 'new'} + +In [9]: parsed_extra, parsed_missing = parse_diff( + ...: jdiff_evaluate_response, + ...: comparison_data, + ...: reference_data, + ...: match_key, + ...: ) + ...: + +In [10]: parsed_extra +Out[10]: {'bar-1': 'baz1'} + +In [10]: parsed_missing +Out[10]: {'bar-2': 'baz2'} +``` + +What about one with a more true JSON data structure. Like this RESTCONF YANG response. + +```python +from jdiff import extract_data_from_json +from jdiff.check_types import CheckType +from jdiff.utils.diff_helpers import parse_diff + +reference_data = {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}} +comparison_data = {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}} +match_key = '"openconfig-system:config"' +extracted_comparison_data = extract_data_from_json(comparison_data, match_key) +extracted_reference_data = extract_data_from_json(reference_data, match_key) +jdiff_exact_match = CheckType.create("exact_match") +jdiff_evaluate_response, _ = jdiff_exact_match.evaluate(extracted_reference_data, extracted_comparison_data) + +parsed_extra, parsed_missing = parse_diff( + jdiff_evaluate_response, + comparison_data, + reference_data, + match_key, +) +``` +Which results in: + +```python +In [24]: parsed_extra +{'hostname': 'veos-0', 'domain-name': 'ntc.com'} + +In [25]: parsed_missing +Out[25]: {'hostname': 'veos', 'ip name': 'ntc.com'} +``` + +Now you can see how valuable this data can be to reconstruct, or remediate a out of compliant JSON object. + +For more detailed examples see the `test_diff_helpers.py` file. diff --git a/jdiff/utils/diff_helpers.py b/jdiff/utils/diff_helpers.py index 90d73cf..1609f25 100644 --- a/jdiff/utils/diff_helpers.py +++ b/jdiff/utils/diff_helpers.py @@ -122,43 +122,58 @@ def set_nested_value(data, keys, value): set_nested_value(data[keys[0]], keys[1:], value) -# {'foo': {'bar-1': 'missing', 'bar-2': 'new'}} def parse_diff(jdiff_evaluate_response, actual, intended, match_config): - """Parse jdiff evaluate result into missing and extra dictionaries.""" - extra = {} - missing = {} + """Parse jdiff evaluate result into missing and extra dictionaries. + + Dict value in jdiff_evaluate_response can be: + - 'missing' -> In the intended but missing from actual. + - 'new' -> In the actual missing from intended. + + Examples of jdiff_evaluate_response: + - {'bar-2': 'missing', 'bar-1': 'new'} + - {'hostname': {'new_value': 'veos-actual', 'old_value': 'veos-intended'}, 'domain-name': 'new'} + - {'hostname': {'new_value': 'veos-0', 'old_value': 'veos'}, "index_element['ip name']": 'missing', 'domain-name': 'new'} + - {'servers': {'server': defaultdict(, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}} + """ + # Remove surrounding double quotes if present from jmespath/config-to-match match with - in the string. + match_config = match_config.strip('"') + extra = {} # In the actual missing from intended. + missing = {} # In the intended but missing from actual. def process_diff(_map, extra_map, missing_map, previous_key=None): + """Process the diff recursively.""" for key, value in _map.items(): - print("value", value) - print("type(value)", type(value)) - if isinstance(value, dict) and "new_value" in value and "old_value" in value: - extra_map[key] = value["old_value"] - missing_map[key] = value["new_value"] + if isinstance(value, dict) and all(nested_key in value for nested_key in ("new_value", "old_value")): + extra_map[key] = value["new_value"] + missing_map[key] = value["old_value"] elif isinstance(value, str): - if "missing" in value: - print("missing", value) - extra_map[key] = actual.get(match_config, {}).get(key) - if "new" in value: - print("new", value) + if "missing" in value and "index_element" in key: key_chain, _ = _parse_index_element_string(key) - new_value = reduce(getitem, key_chain, intended) - set_nested_value(missing_map, key_chain[1::], new_value) + if len(key_chain) == 1: + missing_map[key_chain[0]] = intended.get(match_config, {}).get(key_chain[0]) + else: + new_value = reduce(getitem, key_chain, intended) + set_nested_value(extra_map, key_chain[1::], new_value) + elif "missing" in value: + missing_map[key] = intended.get(match_config, {}).get(key) + else: + if "new" in value: + extra_map[key] = actual.get(match_config, {}).get(key) elif isinstance(value, defaultdict): - if dict(value).get("new"): - missing[previous_key][key] = dict(value).get("new", {}) - if dict(value).get("missing"): - extra_map[previous_key][key] = dict(value).get("missing", {}) + value_dict = dict(value) + if "new" in value_dict: + extra_map[previous_key][key] = value_dict.get("new", {}) + if "missing" in value_dict: + missing_map[previous_key][key] = value_dict.get("missing", {}) elif isinstance(value, dict): extra_map[key] = {} missing_map[key] = {} - process_diff(value, extra_map[key], missing_map[key], previous_key=key) + process_diff(value, extra_map, missing_map, previous_key=key) return extra_map, missing_map extras, missing = process_diff(jdiff_evaluate_response, extra, missing) # Don't like this, but with less the performant way of doing it right now it works to clear out # Any empty dicts that are left over from the diff. - # This is a bit of a hack, but it works for now. final_extras = extras.copy() final_missing = missing.copy() for key, value in extras.items(): diff --git a/tests/test_diff_helpers.py b/tests/test_diff_helpers.py index bd9d27e..b4a563a 100644 --- a/tests/test_diff_helpers.py +++ b/tests/test_diff_helpers.py @@ -2,6 +2,7 @@ import pytest +from jdiff import extract_data_from_json from jdiff.check_types import CheckType from jdiff.utils.diff_helpers import ( _parse_index_element_string, @@ -85,30 +86,30 @@ def test__parse_index_element_string(index_element, result): {"foo": {"bar-1": "baz1"}}, # actual {"foo": {"bar-2": "baz2"}}, # intended "foo", # match_config - {"foo": {"bar-1": "baz1"}}, # extra - {"foo": {"bar-2": "baz2"}}, # missing + {"bar-1": "baz1"}, # extra + {"bar-2": "baz2"}, # missing ) parse_diff_case_1 = ( - {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, - {"openconfig-system:config": {"hostname": "veos"}}, - "openconfig-system:config", - {"hostname": "veos-0", "domain-name": "ntc.com"}, - {"hostname": "veos"}, + {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-actual"}}, # actual + {"openconfig-system:config": {"hostname": "veos-intended"}}, # intended + '"openconfig-system:config"', # match_config + {"hostname": "veos-actual", "domain-name": "ntc.com"}, # extra + {"hostname": "veos-intended"}, # missing ) parse_diff_case_2 = ( - {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, - {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, - "openconfig-system:config", - {"domain-name": "ntc.com", "hostname": "veos-0"}, - {"hostname": "veos", "ip name": "ntc.com"}, + {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, # actual + {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, # intended + '"openconfig-system:config"', # match_config + {"domain-name": "ntc.com", "hostname": "veos-0"}, # extra + {"hostname": "veos", "ip name": "ntc.com"}, # missing ) parse_diff_case_3 = ( {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, {"openconfig-system:config": {"ip name": "ntc.com"}}, - "openconfig-system:config", + '"openconfig-system:config"', {"domain-name": "ntc.com", "hostname": "veos-0"}, {"ip name": "ntc.com"}, ) @@ -116,7 +117,7 @@ def test__parse_index_element_string(index_element, result): parse_diff_case_4 = ( {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos"}}, {"openconfig-system:config": {"hostname": "veos"}}, - "openconfig-system:config", + '"openconfig-system:config"', {"domain-name": "ntc.com"}, {}, ) @@ -124,7 +125,7 @@ def test__parse_index_element_string(index_element, result): parse_diff_case_5 = ( {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, - "openconfig-system:config", + '"openconfig-system:config"', {"hostname": "veos-0", "domain-name": "ntc.com"}, {"ip name": "ntc.com", "hostname": "veos"}, ) @@ -144,7 +145,7 @@ def test__parse_index_element_string(index_element, result): } } }, - "openconfig-system:ntp", + '"openconfig-system:ntp"', {}, { "servers": { @@ -181,7 +182,10 @@ def test__parse_index_element_string(index_element, result): def test_parse_diff(actual, intended, match_config, extra, missing): # pylint: disable=too-many-arguments """Test that index_element can be unpacked.""" jdiff_param_match = CheckType.create("exact_match") - jdiff_evaluate_response, _ = jdiff_param_match.evaluate(actual, intended) + extracted_actual = extract_data_from_json(actual, match_config) + extracted_intended = extract_data_from_json(intended, match_config) + jdiff_evaluate_response, _ = jdiff_param_match.evaluate(extracted_intended, extracted_actual) + print("jdiff_evaluate_response", jdiff_evaluate_response) parsed_extra, parsed_missing = parse_diff( jdiff_evaluate_response, From 2c4cc2065ee88c65bf8add0374c972afc71552b0 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Fri, 22 Aug 2025 12:39:22 -0600 Subject: [PATCH 08/11] add change fragement --- changes/130.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/130.added diff --git a/changes/130.added b/changes/130.added new file mode 100644 index 0000000..b79276f --- /dev/null +++ b/changes/130.added @@ -0,0 +1 @@ +Add the ability to reconstruct JSON blobs to perform JSON data compliance. From 3dbda2d7b0b38ad75a8b5d8db2b9938b7c8cd41d Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Fri, 22 Aug 2025 14:46:27 -0600 Subject: [PATCH 09/11] add missign code references, add logo, other cleanups --- docs/code-reference/jdiff/__init__.md | 1 + docs/code-reference/jdiff/check_types.md | 1 + docs/code-reference/jdiff/evaluators.md | 1 + docs/code-reference/jdiff/extract_data.md | 1 + docs/code-reference/jdiff/operator.md | 1 + docs/code-reference/jdiff/utils/__init__.md | 1 + .../jdiff/utils/data_normalization.md | 1 + .../jdiff/utils/diff_helpers.md | 1 + .../jdiff/utils/jmespath_parsers.md | 1 + docs/generate_code_reference_pages.py | 2 +- docs/images/jdiff_logo.png | Bin 0 -> 16485 bytes jdiff/check_types.py | 4 +- jdiff/utils/diff_helpers.py | 12 ++--- jdiff/utils/jmespath_parsers.py | 6 +-- mkdocs.yml | 10 +++++ towncrier_templates.j2 | 42 ------------------ 16 files changed, 32 insertions(+), 53 deletions(-) create mode 100644 docs/code-reference/jdiff/__init__.md create mode 100644 docs/code-reference/jdiff/check_types.md create mode 100644 docs/code-reference/jdiff/evaluators.md create mode 100644 docs/code-reference/jdiff/extract_data.md create mode 100644 docs/code-reference/jdiff/operator.md create mode 100644 docs/code-reference/jdiff/utils/__init__.md create mode 100644 docs/code-reference/jdiff/utils/data_normalization.md create mode 100644 docs/code-reference/jdiff/utils/diff_helpers.md create mode 100644 docs/code-reference/jdiff/utils/jmespath_parsers.md create mode 100644 docs/images/jdiff_logo.png delete mode 100644 towncrier_templates.j2 diff --git a/docs/code-reference/jdiff/__init__.md b/docs/code-reference/jdiff/__init__.md new file mode 100644 index 0000000..b811229 --- /dev/null +++ b/docs/code-reference/jdiff/__init__.md @@ -0,0 +1 @@ +::: jdiff diff --git a/docs/code-reference/jdiff/check_types.md b/docs/code-reference/jdiff/check_types.md new file mode 100644 index 0000000..c516f6f --- /dev/null +++ b/docs/code-reference/jdiff/check_types.md @@ -0,0 +1 @@ +::: jdiff.check_types diff --git a/docs/code-reference/jdiff/evaluators.md b/docs/code-reference/jdiff/evaluators.md new file mode 100644 index 0000000..6fbf718 --- /dev/null +++ b/docs/code-reference/jdiff/evaluators.md @@ -0,0 +1 @@ +::: jdiff.evaluators diff --git a/docs/code-reference/jdiff/extract_data.md b/docs/code-reference/jdiff/extract_data.md new file mode 100644 index 0000000..76d8840 --- /dev/null +++ b/docs/code-reference/jdiff/extract_data.md @@ -0,0 +1 @@ +::: jdiff.extract_data diff --git a/docs/code-reference/jdiff/operator.md b/docs/code-reference/jdiff/operator.md new file mode 100644 index 0000000..ed20ee9 --- /dev/null +++ b/docs/code-reference/jdiff/operator.md @@ -0,0 +1 @@ +::: jdiff.operator diff --git a/docs/code-reference/jdiff/utils/__init__.md b/docs/code-reference/jdiff/utils/__init__.md new file mode 100644 index 0000000..9f026c8 --- /dev/null +++ b/docs/code-reference/jdiff/utils/__init__.md @@ -0,0 +1 @@ +::: jdiff.utils diff --git a/docs/code-reference/jdiff/utils/data_normalization.md b/docs/code-reference/jdiff/utils/data_normalization.md new file mode 100644 index 0000000..6f08a0b --- /dev/null +++ b/docs/code-reference/jdiff/utils/data_normalization.md @@ -0,0 +1 @@ +::: jdiff.utils.data_normalization diff --git a/docs/code-reference/jdiff/utils/diff_helpers.md b/docs/code-reference/jdiff/utils/diff_helpers.md new file mode 100644 index 0000000..9760b57 --- /dev/null +++ b/docs/code-reference/jdiff/utils/diff_helpers.md @@ -0,0 +1 @@ +::: jdiff.utils.diff_helpers diff --git a/docs/code-reference/jdiff/utils/jmespath_parsers.md b/docs/code-reference/jdiff/utils/jmespath_parsers.md new file mode 100644 index 0000000..aeee717 --- /dev/null +++ b/docs/code-reference/jdiff/utils/jmespath_parsers.md @@ -0,0 +1 @@ +::: jdiff.utils.jmespath_parsers diff --git a/docs/generate_code_reference_pages.py b/docs/generate_code_reference_pages.py index 0f1bed3..636ab53 100644 --- a/docs/generate_code_reference_pages.py +++ b/docs/generate_code_reference_pages.py @@ -4,7 +4,7 @@ import mkdocs_gen_files -for file_path in Path("pyntc").rglob("*.py"): +for file_path in Path("jdiff").rglob("*.py"): module_path = file_path.with_suffix("") doc_path = file_path.with_suffix(".md") full_doc_path = Path("code-reference", doc_path) diff --git a/docs/images/jdiff_logo.png b/docs/images/jdiff_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..032a750c798549f768a8d9976b19ef64eabd3a8c GIT binary patch literal 16485 zcmch<2UJu~(k{5E!!LAW8CofJ6n!Ip;Lw%qT%XU_{Bu0HOjz&L9Iw5Cl}R z5(Pw*C?FsR+k^VN|L56v_w1fM-*?W)+^O!auJEhszJ2SOa9vlOoa6!t4h|0aH4RmL z92^J&T)!kD1b?e?Kly<#Tt9tvI1YMGN{e3> z5|zQW{Fm%%fCd@?&A`^l7UBAj`LJpQIwJgBfsSVy|BExW#n0Z>!|^v=^tTD@yb%tL zh~Ff@P>f6bDwl{d=!*S_t2nsYIwNd7!4EH6Ak`Vi>%eS+e}=m{{HD2inwUuhc?KFv z_&F*gWRxYerLgAyH*SAJa=#AQ3U3C00s@klz%k%i_NY&WOMB z{FxR@FmMh1tIC<+Vj|7~t`3fVf4Kb~>F@3B;qU76??iiVFE0Rtf9q(AKzIlJdp9R< zk3ZV|nbij<{%?b@1cU#{q*$u4tAjs44%Pm+$AXM#!uY%h<)=(*rnH6kOVRxH@Aa)gIUq z8;5WZy#B5LQ_6oDJY5|eu$saVjzBqEyECmpXu^HGL4Gh4x?96^=K@kHFTf2Yp_={%jw0|4}q;&in zxW7q0w*Q5PGl=}}LHS?MRCfdk&mR#2-1CQ{^dCI^$pd2Iz+qTR2K`r5iT&;4^3RBp z`7_Y=Pb544i#2xm{|VYbpFhF!zZW07y8K5>{kK5=Bl&*=WCz#4e*o>D*mknD{|)-T zfYkT4^Y#bJ#NXdQm;V|6|0|UL3+~|JzsI?ogR7Gh;Qrr3_+R9I6Zk)Xjqncea&W|^ zODyt&TwMJf4Sas95d@Mwp#2l`2>U~~z@c)-s6$iTxwg1Hman|;4Yw3xgTsb%O;!13u+@70xhzh@y@t6DeQBoesW?|SmAOuF zZ#>wAQPEyiLgh_iHionZ-v|k?n_eTm6RmOmm6DojLfV5oI#g|R5qXTGK}6WomZ12l z0`v=X^VrN-`;na7x5eqOji$}=g2TP?u%Yei)VR>|e?Kl75G$2X!IG;=_^N+DDAb4w z8-)Guhg?$$j#ie^^-78+y7T7~lUMMU17PKH9YTd5Z}vr)HR}qCct}xsFu+9ypPYeUDZZg6NlgcpiQ06BH!&;obV5h=gilK3tdu9rpJZ) zsMy@Bl#P(ij9%ViSaPB=rNzpS1Fbx2yPrKse0T3eznq>5F_tzW4SW=YZ+2UYSo+Nj zezgEry8u;;z1+nD0`G1wDn!*iSFAG(Ac7q=m_*ED-IaU3Sie4Fg?Qr{eAyjO{gxJE zVPW@G7{WfEqU65Cjypi-C)$*&o`H4t!xJLyo<;d9as%;8n_z3< zle)^*fNypylgQ*6k8jhAt)Mo(Dw`0p^L1WAQ(b4AW?wAy{V)(>8tOt^aEVaTBmypR z21#THewkNoo72ogDjH%Lc`?Ab4)*qfg&HO(5ssEra4|a5&>Lc~)p(C$^X=^P161iI z`GX*KlpcVc2NfHCAIk}0wm)|HN*pYw z65sGB#$u01LLYeV0z^>x`k+WM^o+q5<%c7=VdqPzW#lh{IX8e8>2Vba=@5$9PG^Dw zNJ&yaoN&`j#CO>&T$b%+F%@08gaR zE8>CunK4Z871(WrwXU+mCQO)kystjoCZe$_Wr2{n$gM0_Va++pg*$*FGHX?#**CU) z(sv2k2SsMg6WFXXmuqNYyW|#Yy_2XUe%^(62qy)X&LeKwEg9PZ z1w%6Wz2q$9fMvpbb+{lp>cE1cEb&q^&2=_d)b!no?zN{vm`0B?v%f6^@GOaLDt5@B zy*)07jja?SIYHRUow7HH(L^|jCS&%1CmTeFxRn>SLRI0Bq*O!gLd=a*DkV!;ZjH^A z%+TPg@FA#aHVEM_SMHI%Qa;O|n2Wf?GC>}Wrr;M`claz{N^`NnR%j~jCMn&gMxorl zpHmvZ<^mlh_>hF{6g|?3n+-Pcrlq0|WLO6!CAbh?Uk)5UCnGw>V#KSBP6%j6j!WF+ zA?;2FbOGOr<=r^Kxc9=_t^QDPw9};K)r!qbnZwXMIBJu*!(2N z68}3;RA}y~bPS~Q(LBInutAC9j#3I3wjAtNclit%pfhC)8>=c?1<>hxbB$Rh;GBOR zg(|(DjACL(X@Lowri}zO0r(@a(hXbnQA7J!f%wPk&y6?%M2x}6uWqtScZrtnf?0+n zZw>(1J(B_>_v$&mtK)wME+COG8R94GS z>p?kZO<535|MKvgN!LK&tPLZ>k>I2Yk$@PERUuhJ8~&5ijOjpq(p*8xmU8lMg5;%? zam(~bE<)Ne-9w1NF&VdREDv{QHYMh$kUM{SU@^#=sos)t-1LHxDRYK-5ys`SN7gaH z@uop=7|HZ6lH;$?EK&ZML&x<2puw&xh^0p0xRn%7?ST>xa_2Sti2md82E|tM)Ocdn z?#pIf8{BrfA1(x%lV(^37jO)>QGT;Vy{UTlQMp&DgTL!*o1RVS_SV}B$NnG!`d6j~X2!*O`d z%W+GEZ=A%Gl}K^xzYO%3>;!7B@Yy9If!rB24@ z@c@H$z$Y)bGQT#Ot}k0Z|B<(}SyAlQE|9RJB}jC7aH&j2xzl}u{A`%-i|6`$dc52=gjBL!z1T~w~8H0uCawvrQezvK z_`mzDT%4p-@J`^?wJO9f&9gMmT;KRu(`&TZ6;aW3YH2ld&`lS8jpIpj#HxzPQL9Tt z>1x0Ea-@2B@$No1`|^MeA;#up*kh zD0zD6#f+f2z!K+rxq(gW)h=hbJt2l5>%&OJ$@AzL=aa(N{K*get5UjgkrACMtlxJ! z-Hqy&fP2!!A7EGTS!`P9uG!x1bGa8suSlYPQY>UqFr1(w89OjAP5W(Xdx&@|zx0K5 z7ta&hN4N`%f+RZ+WIK-xM;24?NcSOO^;M@tdQo;fIm67Up=OT?+ zLq%b!0mJe13eQ>}Q`g7%1yWe=J*3Ez6mKmIjiG*K)<>Wa_~qy{+GuzsYQf;SIs@`y z{*5@$BQa{yO7k zThGxz5vSh%whWmk;gL(b+%F$rPD0gdtj^{TxC(@X(rA69j#Z+9IZs@p6x2oV zN62)$FC6mTf1wuwK@PiP_%QzSX^%AuIIYH@L?5Qoqu+a`l^~(DzPT&!FwgZ$?QhiF zph8Hf%=vvfM-f|Y^mRE%59i!0{)C*V)J8_Y(*9AG)i=W*+n-l#hv&G=dl%I^pAMuq z*k=CPZ||_wt>sCH(dmC9A3;avk@^y?xxXB_!z%sd>7l3T?&^aZ%OiK?ezAD$mb&c3 z&t%s8s*2FpidQm&qn8F3F(Sh&1^(0SLi{r^7Oo35$Xw(?!Ensy$M?ePy5@171{#dEuf5i3HqujVE9Q_~@5}d{nnqI&LLG$#SJz%n z@K|5_Na1r#7i;{!r$6RFe%yLUdCRNU?t;X^EY`F8*D}5?)&_rT4i%QRc@k1#D(Lw7 zpm$+QF|)zm{m06&F}k}g%q5*$fOW1%<+G`HT>LDV@x&p0T<`<;IayDbvDv%a?vWzC zio9_aZ9zF+j-}?7?eh@~U+car35L7Bk_m$%ii;;ni~OQT~u8D}!N znEqTJU80sf@_y));95>&TpYM=;Dgy5F~6Bn7WvWuw(ZsQV`#LqbhB|#cS?Y@FrhJi zw#*`N{q+;uCyoQdHkb5%JcqNHb5;(XjxY5bRG-iEmfsEHTGDjSB{^n3h+Q#w6Kp^^ zfYxJ$1N*HSBX1N82kbQ^{@mtM?Mr8_`&QpxtXLBIB)BB9bvS{C3L!Jq_tQk}L5p?Z z2fumNuU&JERwf^4NsV|X%J@2SR(k3fDo%InXZ`jqe@HX`C<*>CD$4{x$6D+wW*Cb^ zznj&#Fps0ibRh4Xz`LZ`rO>nJALoOHaP)5)tHLHCwk$E13Wg2pLLdm~(ijU0sk%U| zMD`{0u@v)h?}x)IhtG_Wo6lt}NSi9fyfPKWkNBrJ-Nz@F@T;j1Tzcz66XkxIFUj}9 ztu!_wUkbdUK@@j?9Dwp`jC?-i6?y!T<3=osKRNP3>Ow7|W#b*!@!e~ET=>rxW{6dx zhz$KMefP^|AH8TB_ffNLfr3jyhf!NK6?+QpF&4iRZkUobZE(^syjyH~rr#Gp|2nEj zxT}rM>iL|yknAuOQB&Ie%^ant$=g)^h8wvt7U}bs%b1b8aP$ppT&pS`=GJTLF?oiL zCIQ66O=`zWWBPsR!EFiZNfSNZ7oavd3OqE38Os|sub2rkZlU-^>um1SBp!aDVSuhY z%0wlpXhon75=6%kgXXHgT7hH|gJ7}Pw=3*X2*y=Cw1 z`!$Yw(Ym%H`ehw(SMC0s`n5ipsY$ZC+)#wkS~ud=I^M4Dn!5qM8vaF(K@WlD#f+4Y z9F3s}8`b%mhs)D=x>-AAA5H_OxFPE#cfTM_;6uye6aMv2fV?bmivm*c+jUdx>7+9) ztsD(UAGTKTIF~N0-OK*wH3mmJz_v{Nbv@tS!&EUISA}0pt#3W2hM!+HB+!F9H>|{| zXrg`2{`7M-fz;c~6`7c5NgIORq~jtJQM)`owTH%~DXjHQ;)TXAsY+ID_wv?Uq1e21 z`I_&}={Z*A#3b9h&x6svT4Q-fcmQ{=0vP906naZ{)5pr%P6H~N-n*0v z;#TCD$inE0ce7(GF37I;sJX$>En&*Cvef>+s;2vrKl+t_=siXdzD-+6gd+76v6$J|=S!mhnz7L&H0c=R{;hyg zI2`qIAc;a?^n9J1E7t;@-ym{=P(vT)H z66~X%dGF2!JvvGEYn;+p+_t#etg)#Z>rH<@)HJ2J!Z?_|s*`=E06E)w>_OP-7Il%m zUe+qkVB^lD=?oRrhkZ$th2Ub@@ESVIzlHO=Sa{{kVfDB2z*V^kT8>+Rfjsm6HotrX z5<(}htz7jJI0!*AK=XRi{D%A^=AHf6!_INC(<)= zB6Yh$K=Skz(+O3

-&QX-5b(^ju`!_g@v2j1^%QB7HbO46n%J^Gwq@i5{9Yx4#wu`gp>#C&{=(;#g-Joa5$Tl0_b~VfYx;B4w~fg88byC%l*D`M@k{|-K`XD*t7trqA*b^O(oH|658@Dg!UI>eoG>h3X#H^AM#m?$2&f%jcs;q%w{?x z4%PDgXzPCRz_3TfQObe%-G&%_m}fir{Cj!K#zeQPv;6sWuW_qy9iq8iM*LUly~jvo zNS!8q>t>CsR=>v{1*L!3HTBWHtJ+Il(kUoRE0CjP*#JKjrasOUFb+AHP>35jWsKk{ z2tLQsP^9fPM0|7mhTcR8rkA2W3fChay?j_Um0zK6^C~`NVa6VxU-AGmMOG(gX`ov5-QrE)z^)06tT~{-`iQnLyK74=bdinpWxQH zEhwB!uWf5LYBju3ADFeW6M2$!&c-E1>qpIpQtFoC>`!@pu_5j2{abo1!8ad8954zv z!eS^sbhq4&z?wa_sv(>imsxRTKt1-wo#%}XNT`Z@ zp{zlb5kChNBKw}AFTeCri_N=Oz4mi@+A)rFgw(_X0yKlkobV0fmOYki8GE|20G8t* z{mY7>KbpVsY?Y5aNlxZlQt*y7ty!o_h`rCh<2zAD!sBce(|(r<@$QF8+cfo^t;Xh{ z$46uxi~^l2mc((X18f~We)wuMRAhyOYxzixp(q?i_w9vML64ht@S%;Y8IP5*wp#fpW1^eS>i0HDjgm$YgRynu zoAI{9FHdUMaZd71J3+c+fc=u=hXn8}X_5*%Ee`Yjow)%={r6&5a+)qI@e za5SYt(z4go5I2otUBx5VAiJ&lH0^ll_hHr(Mh`D3d;ZJBO%8V@uE;~kVN03=gCZS1 zU>Rqac$jaQL>wojy;;TQ{#*vaQLQ5q9#&=dswQP09l8Tf$7{4giV_~q>JDlebkic} zOhfZ$+%}3WPs!GzR63o8hm8wt3za7x3k#@WcY zG}Q)ya2Y0 z)czUCb5vyJl93a{ajcIhyIPG-?wv@Ergqy142V79FYZ%Lp(e}KIFXNFya$%m`Wj`~ zFj{J|UX91s$HccKSv;+nGL0oZbh8o5(BCYrmht5kpii?EOF`AVYHMJfY~lCTL(&RX z-Ar81XG8= zL{S6zp7W(j{4N8Tgy=_%cb3lKXq~&ikGWGpJMf}OSH#1a&AE!#d9)R~YRrT;&RyYZ zwTWPc?T@~zT&Qjn7IwRUl8zQ=ODr~hKH77Fr*%IRUWd}aRO|k>^zG=H?-rt+Ml2?`7H(+-=Gz=N%Be{xcN4X!d9K<-&L519KsX{* zqSVF&MBwnD?TSZVK9A5K+{xlH^`6P?yRAeU;%U?*&r)0odVtcp|9naxK3|*QS ze|CD(moXLs76B774cKW1ggU^U;wNbYmmnoFQ(O!}_-YuR;oc42&-DZfa{`~m z{PS(3bFpePhZv1eA&gG_pIXcC%(PW>P_*ktt%803O558|z3Qd_b+5?CL^GWq+D*(e zm*HsHR~Y9KyBFkGj3o8CR3y`U&uIxKOPit~A&#RIPF!+KGbO$F3nxN)%6-}TkvvnQ z%KUx9TMK$y2Lwr5emq#60(NMgg5$HthMpZ#a(G9@S&C?U(L|K{sqBH@iPn4UlAI=| zXqxJ&36t7?fWEz#6!cOc$c#09OHkyKKV;~2giXw_P#vcuG1JR)w^by|Tg#GIV>wRKPTq4Zy?1t_3s4^-pC~Jnti$fH zb?sfB2GJ^=J_qYhEQZKsFI=JD?Y_IH-rJ728P#f8`W!BiPgFm5D0-Jw6t z=f~xFd^t7gp`JN8Bs3PkD?i$B;mgVvu6H7aXff0FmM!Vs$`{;YCC2F}zk|MB@%#)_ z*mkf9|5B;yg$k)sT7>7mQg7S#mUw!KhmrPyf0Janqgf+wfZpijZ7YW~w^#5#lDQ zEW2B_VV=$<0j~8=sSti4KPP1p3>!cM*Mj@%531tznHi{~$K-^&m(a8Jk1J zYqK*@xnmR=T4?>SJ$sX1dAs!VSNlx!4N2=Y-utslDaz;fafLjdPDw|7i}k*)!FF6@ z=T!@%Xn5orV^KFX(mw2A0gyU_GJr`=1I@TVwLNtRbdZx>|iY?y*Ri zt(Nth7=pF$s{MkAD5e0ZybKZAmKb35{#3Mq9~DA=i!e6Lssz$Ie2CZd(BS8mMWa7o z8V(L!KDHJ>l<#dvolS2+uL$cwnZ7n=t+^;OBp^(5av_#MNb=?c+s8C3qaWd9o_UKQ z_*KfQDHa|MXu=|{B?UQJLQ0Z-#H#O>^09e3ExA@-tY5{`c&l1$`gmz^Uh9vxDxTry4vNnN5x~sCv7)c7lY9`*_6j< zVY;s9`zt==2~65+p8?84J*Tt!z%yp{a90YTbnKX^;9>Zv^#VJ;QI=g2`;u-OY0pfi zhkkzF;ILkW(S3OXA$6%X6lu7CdXwe(p#08`_jVTN^1dFnE+7otJ!M^-Ss}W3<34Kx zJeaA#55|V7#r7-j#aQn;$~cJ-O24(e>Gpc&TvExaAKu@&VVQZhnso1}5T^9zPHZ5JJ&$ zrlFge)MM`huRoKssd%D(Mfa)4!0#I+!@cH;jE8n_EXz`Ct;NbDD+ipc#k?L1bF|V~ zT?=3*Zo28qEt-*mYVnbh{?4>e@P>k9f63_$$tF`>JP0fO?Vs2Bx)%+-+j2GgXrFMU zaz5<7QR}e(D}psu7b5h6@19$m{H3-7>I1U+G<(dwl!4srb9%U4^6lSPEWFI;?dzh3 zvpCR_h*k}LMrUX4mn-70Ut3TibSsVQFAKsN_B4G?AH9>4{!Di;*8X@e`8_{CSRhYc zNED5G+y1-KvBRk^V{X;X{mWmTo<|^uA7fnRO?uy@)f)?&BnVd|D>h9xnth}=Mo&Bq zeB8t{|5JA%e|LeVBE7`D&C7Tv=;}4a3Bzv<)a83oXPsV2VnE6$rhINE+LNzZE8ALhvMhnmZ+-oCba8%ab#-Q z-+E5O!$3Qzs(3Me6vBXhq{!jNg0WF1HTt;!itKUG2r)SP+NMJIy2z~OK1GH7FfkFf zBh)C}vPoxo?*dLN-pbC?Q*!2lL~`Yx)(?C~*(;fFp{*(=W@iYx-z`1x^#+}#chns} zR?K`*dem2c6Qj~swK5Qfo=S*#3j}VFF~%pVBNYuB+T)V?#M%NWwOYmWrFIk*<%C3@F`djMjxLs z^3s(=#_zJ5g3PzDlP;fdk|{A(S%@TQQ?Sn=wa=Hq*VnCTWtq44`Xx{H=sbK~t3x!O z1exJ&RA%jxc)q`E()seIjEnIa@wxaSn$wS^Tjl&0wT12{5MORZeM`&;&$Q1PekQV!PUgzimG#MjFuV_W430U3)vi@!rgU4CH&Oy~576DLi24;*|BxX#taTFfbIbr6|^BVD}Ar-Y~#gu~C5 zM-5x)v5+$fGAT%scTiuQY(2m&^NVa$yb;RWCUcCK=$9`UzJ>g;_d&Vk>~@czDe|lQ z!Pg4r-!{rakzO%PH?@41X8It^EdwWRJ!w^l-639c_l<-)j>@AqxyvK6LQinw-dUMX zSPuSl2b-B19#>Ks^T~StE=9rdD|R*02*ox=^l{KD+-@)9NY=<|-(ArOMLaqO(E%Zb zo9|fU$m)+ZmyxGOG#PcA%9m=l!!VC>W0}6L7`H#w9Jf}$9j{c?r!~v@MsEE)^cC!_ z2te`Fi(oT{(`FAHDCT*2Oo z*N@&3Io565-19N_&LguXZld)_NI6&J52st+C=6*E3l9D==ySUGTde0hVA-3T*$>)@fP~KDtbWP6sL>8h)r$i7B9<7wdj{TX*5Cpy-yc4@oYwmUzcIT(8)1m5U4Wj5+SVTc-#1@LPvg+xSBP*;1r7jguQOS4uJSiRhR-{_3jr zW-7$zK08uO3^hrKl~8U%8#%}Heg2i(`MejmohzS>Yp%a4pZ6#&r+PGVEVp*pLUfVe zzsbf}+G)o{a;;$c{i01j{kxy)t9!@3l|%YBLg{p`Pw1>#+0-(Q@sl6{_ilt)VL)1AT^4t%+0WSKwseHL4p*k^J;}f3a$SGp;PSd&MzzfFT z7j~mM`&yoaUalz2Kge&;@`oBaWnFjNZSRyQbMqYvW$W%;P8skCZY2p%$4JOiS2yNjL&YHXv zT*>V1I}<8pfYYUnZz(emw?n^hGQDsxl_wkMETN3k#n+uvedMskwFo=Ale9%XdKH#i z0BTfrO}J|xfCo`cF0cL9YoD9h-mKo);HjKh-g+M&ds>o)s(INS3F(=kKlv)E8G9$D zaeK|(x=P;HL7CU-9UY^1`<=XTGwDJ984GT~!1XYWj}-|h(ho8Z889|coR;<2RG+J? z?Z|*zf@5eQlTt^p8MwST5Qd|2E?2yz^v%cmsxPlLr-O0He8+izFGHpzGUk1c@ks5w zQUgdTEEg{imM3jI)6OC+d#dm6K{GLO~bXpn3)R`2fJdE+M@KZYL{yh%LO?c(Y68uqEGmv z7kI!Du!2WZTSJL!FV4GTL}KHzAFiO;dvv9i9cC<*OLeucJVZ#_XX~;V``n|CG!R-) z7(mBFbo=M+Hpx0;Bsb1%o<*>{Gmo3MwovLIADcxBW!78SoybPXh6dknpR4Y1kSk%D zP^7x{t9DI1q&|Va^5+TWZw~X=7PtMJZF+c{JZbL=M~@MgncfK=L)n{;vdpA+>t#PC zxWrNl;fvtstsL$)UZqxOTnGiF1cMn@p$L^o3Zo?}9Z+@TOC(K=K&BQuR33R71sA2- za}H8rY&r*T!6gxIYQRo|#Bo;h#(O9!xuMR9Vaun*Lkr`&ST$x|pjwMl(`*LiYi84p zI*Sj8>)U|9zH&J5g`<^gSAZb z1G0gyAPkzc42YvPxi1cTb`N$77{g!pS!0)i>%W%*@IY&WemDOL$08`cdJfgN+82-> z=C;{fGIme7wunH)WOX(r&)BB&VJsB0osBONd{8#v*o4Ns?HKRIi4!z9!cUiSuohd} zJMkGVY1(}O!hL)MvUY-P}Bni4yV-lTSkMwc+o@%qMU zgj^5RR;h)t4MP{no3&<-sA#a`ICQ~}G+sBbZ#KI^7glE%G%J+7W1u(zFvzBPmH;+o zyZAESRku24G{dnLt|GcAxO6KpQX50dcqf^D-GOftm+Z$RzCtD>-f@e_l;!)Qg5fB! ztWqZf{MZ+C8TWN?$?8535I33KEA*b-kLwa6-P=gHnuzwuhK+#|y$gv-d+;G=IiF`; zW0F^KRDvQwsY7(>bSPM7x3~({*EBKK7ZsiiGGfl3dlqBCY^Hk@8O%q9Mhy?X!1Hiy zAN#|kjkxJ4k7e2UoIrl{p%o~!!xxE>XS_t(6y$mfCr(q_fDS=oLlhQ0@XG>@zTl?Y6O-=x;gkNs zgt_k1j2oa*F&^Jd5IYU*>q#-^Qi0IyrB;crts(1~X($66m9%rJa3SKho4rF@kF13m zn(a-WL^0n|M2OiE4C^cp`euD40&1`^78wzrPr5PkY%xX05!Es9Le$hZM}v> zGGJxQhy;hfJN03dOzv@f*mmqOqU5;Sw^4I&PbAqxf&CO19uojRuJYfOG*;8jDg*gW$c@?pC?N}gT$>qtgC9vPInD-y%X`p3D z6HC6dbV88#7$>f1H0&-osU{oMU9eJQ{JH!6W9aUj&6*;gp5JkX&&JU5Q8;)2At6+i zD~7EU<$!?4tOOz!M71kT5S5%GI^{lz2pkpWS!hgN#OnhCQAk}AzsM_GrjiP*&mE6u zBhIfer1k%#r^LLK4K3`~-3hcTJsxgohn#HBz4GeE*6bz|6$vjv6pASGLn<)ycI|vb zP0mSnPge05KKWFzEztl3Ii$U$LR|Y2kW8+%WS%phcW$BR<4U2IjpBiW|DjU+N2@qX zwC>ttG7$TtD5V_u#)5rpBz^jk5wE-sm2b*-&qOdf=RYOBl*pwV!$nKDO?gzh54 z;G69Iu1eYV>HS$1n-($Gp7t!_+j{ZCPO3HKR#;PBV@p8sr{CAdQPB)zI%tKI72(AB zM-wWv;56WXlzYxKoSzD@oxR7fy?@*G$nZ&S$Vo)w7PPX(dwniOA~P>)4ea1s_~*A1 z-mSkV0oq1TX|ONR;b(jgpM5r5EKy*L98Sb=5{MMl1SvP*hiQYT;tawC>^fje@y_B$gjtRLy@hZ=y2d;ikm!;F+dHJvkghAZro8q6V>7e zvebs6$BVjw!|o{kIuFWy(UFoW1$UHAKqH-KYv2jpQ(6!!ca$v82j76Bg?=dOVoR56 z@%3<-KKlolWt^2o=U~7IY~3tmP#6-z{gz6O&?94o%gqza$9`P}k_l1=>UZs|3waOJ zhw823zWW#iTcXcq4k=vp1&7S_ZzXbxF4H65b^O2#1`CW^kWRQ^J77eS(XTQ4oJ-7p z86i-YP>81T``y%N93ZeX``gtJ@T79GzYNHCFPhzWj~A;)G$};*{c1S}Yz0#80W$d; z4p3gNfmu*uwf$X6t`UpTmg^Z=xX1;|B++98#4qX5Gxv8uF}TSCkW;}&AGQU>4HNBY zF@_7#7KpI}J?=$syk5mUEAsXy_NPPlUh6APG8_k;`=Ku6LfrEzVi@%9K(#S`)=l2v8dgl0H(YVMIE=PogvKwK}wq8rk`L zI;WXGvqWdi3^a?ei%>Tv=z6zr?5wH^as13t!&k-O&FlYVDS(|q3I7G#{5#$F6|FwQ z4qX-STc7zSaGACj?{cP6_&{FTaNfvrvQDX{`=*Ey)QZ8 z*s6Zy70xZb{MoO|$7!h2u9-9~c2pRoyaSDYEV2J>^tXS>AuKKL7G~ICnS|O?phR6G zUP()YWs=_H^2OWLz^rVLCXOvhAEx|S+qQCT9z@eeKS2%r3wM;Xwgf8WDueMt1dY#t zLJGMbo!}ozyWkZEQ(g$k_ICNYT_ml_|GO_ z?J&>>f1aPpZ{vnLA&|M5$A*1F=SrKz<6u4*eQerK;E+ZhHqn)WD(rg%&w?OTs(FHH zaP(}$&65NraVT%AQyrC6v-pJe5xNe*i2=8TdOz~P#Pb>7YkjK=*z#*(NG#5s?r7St z1EL`uSe=H5v~g!iNIiI^z-efW{5@MR@e5)a{rk*jd%@)3T_BJ-*nLckX*kj|&fYc< z1aBLJ5YeO&7Fvf7UTF-WL#KnOE)ye%kwIJ1=mv-O8`2tmS05EA0Pb}h*PjiBo)@A1 zVl1nE`3zD;a17)EA&zlIY5s*ELgJ>zSEHbUpAEPJsfTIq);vEoT)}u|$BU3nyi<;O zhMwU4r?%jw9rDC#81VN+ Tuple[Dict, bool]: This method is the one that each CheckType has to implement. Args: - *args: arguments specific to child class implementation - **kwargs: named arguments + *args (tuple): arguments specific to child class implementation + **kwargs (dict): named arguments Returns: tuple: Dictionary representing check result, bool indicating if differences are found. diff --git a/jdiff/utils/diff_helpers.py b/jdiff/utils/diff_helpers.py index 1609f25..706961e 100644 --- a/jdiff/utils/diff_helpers.py +++ b/jdiff/utils/diff_helpers.py @@ -13,10 +13,10 @@ def get_diff_iterables_items(diff_result: Mapping) -> DefaultDict: """Helper function for diff_generator to postprocess changes reported by DeepDiff for iterables. DeepDiff iterable_items are returned when the source data is a list - and provided in the format: "root['Ethernet3'][1]" - or more generically: root['KEY']['KEY']['KEY']...[numeric_index] + and provided in the format: `"root['Ethernet3'][1]"` + or more generically: `root['KEY']['KEY']['KEY']...[numeric_index]` where the KEYs are dict keys within the original object - and the "[index]" is appended to indicate the position within the list. + and the `"[index]"` is appended to indicate the position within the list. Args: diff_result: iterable comparison result from DeepDiff @@ -52,10 +52,12 @@ def fix_deepdiff_key_names(obj: Mapping) -> Dict: Args: obj (Mapping): Mapping to be fixed. For example: + ``` { "root[3]['7.7.7.7']['is_enabled']": {'new_value': False, 'old_value': True}, "root[3]['7.7.7.7']['is_up']": {'new_value': False, 'old_value': True} } + ``` Returns: Dict: aggregated output, for example: {'7.7.7.7': {'is_enabled': {'new_value': False, 'old_value': True}, @@ -107,10 +109,10 @@ def set_nested_value(data, keys, value): Args: data (dict): The nested dictionary to modify. keys (list): A list of keys to access the target value. - value: The value to set. + value (str): The value to set. Returns: - None: The function modifies the dictionary in place. Returns None. + None (None): The function modifies the dictionary in place. Returns None. """ if not keys: return # Should not happen, but good to have. diff --git a/jdiff/utils/jmespath_parsers.py b/jdiff/utils/jmespath_parsers.py index e5fe681..d9c8644 100644 --- a/jdiff/utils/jmespath_parsers.py +++ b/jdiff/utils/jmespath_parsers.py @@ -135,10 +135,10 @@ def multi_reference_keys(jmspath: str, data): """Build a list of concatenated reference keys. Args: - jmspath: "$*$.peers.$*$.*.ipv4.[accepted_prefixes]" - data: tests/mock/napalm_get_bgp_neighbors/multi_vrf.json + jmspath (str): "$*$.peers.$*$.*.ipv4.[accepted_prefixes]" + data (dict): tests/mock/napalm_get_bgp_neighbors/multi_vrf.json - Returns: + Returns (str): ["global.10.1.0.0", "global.10.2.0.0", "global.10.64.207.255", "global.7.7.7.7", "vpn.10.1.0.0", "vpn.10.2.0.0"] """ ref_key_regex = re.compile(r"\$.*?\$") diff --git a/mkdocs.yml b/mkdocs.yml index 5d4200a..d2b5ab0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -139,3 +139,13 @@ nav: - Contributing to the Library: "dev/contributing.md" - Development Environment: "dev/dev_environment.md" - Architecture Decisions: "dev/arch_decision.md" + - Code Reference: + - Jdiff: "code-reference/jdiff/__init__.md" + - check_types: "code-reference/jdiff/check_types.md" + - evaluators: "code-reference/jdiff/evaluators.md" + - extract_data: "code-reference/jdiff/extract_data.md" + - operator: "code-reference/jdiff/operator.md" + - jdiff_utils: "code-reference/jdiff/utils/__init__.md" + - data_normalization: "code-reference/jdiff/utils/data_normalization.md" + - diff_helpers: "code-reference/jdiff/utils/diff_helpers.md" + - jmespath_parsers: "code-reference/jdiff/utils/jmespath_parsers.md" diff --git a/towncrier_templates.j2 b/towncrier_templates.j2 deleted file mode 100644 index 46407b0..0000000 --- a/towncrier_templates.j2 +++ /dev/null @@ -1,42 +0,0 @@ - -# v{{ versiondata.version.split(".")[:2] | join(".") }} Release Notes - -This document describes all new features and changes in the release. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## Release Overview - -- Major features or milestones -- Changes to compatibility with Nautobot and/or other apps, libraries etc. - -{% if render_title %} -## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/networktocode/{{ cookiecutter.project_slug }}/releases/tag/v{{ versiondata.version}}) - -{% endif %} -{% for section, _ in sections.items() %} -{% if sections[section] %} -{% for category, val in definitions.items() if category in sections[section] %} -{% if sections[section][category]|length != 0 %} -### {{ definitions[category]['name'] }} - -{% if definitions[category]['showcontent'] %} -{% for text, values in sections[section][category].items() %} -{% for item in text.split('\n') %} -{% if values %} -- {{ values|join(', ') }} - {{ item.strip() }} -{% else %} -- {{ item.strip() }} -{% endif %} -{% endfor %} -{% endfor %} - -{% else %} -- {{ sections[section][category]['']|join(', ') }} - -{% endif %} -{% endif %} -{% endfor %} -{% else %} -No significant changes. - -{% endif %} -{% endfor %} From 423433d44a3962bc846bb81ec81237cf55ad6735 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Fri, 22 Aug 2025 14:47:08 -0600 Subject: [PATCH 10/11] fix yammlint --- mkdocs.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index d2b5ab0..031c194 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -140,12 +140,12 @@ nav: - Development Environment: "dev/dev_environment.md" - Architecture Decisions: "dev/arch_decision.md" - Code Reference: - - Jdiff: "code-reference/jdiff/__init__.md" - - check_types: "code-reference/jdiff/check_types.md" - - evaluators: "code-reference/jdiff/evaluators.md" - - extract_data: "code-reference/jdiff/extract_data.md" - - operator: "code-reference/jdiff/operator.md" - - jdiff_utils: "code-reference/jdiff/utils/__init__.md" - - data_normalization: "code-reference/jdiff/utils/data_normalization.md" - - diff_helpers: "code-reference/jdiff/utils/diff_helpers.md" - - jmespath_parsers: "code-reference/jdiff/utils/jmespath_parsers.md" + - Jdiff: "code-reference/jdiff/__init__.md" + - check_types: "code-reference/jdiff/check_types.md" + - evaluators: "code-reference/jdiff/evaluators.md" + - extract_data: "code-reference/jdiff/extract_data.md" + - operator: "code-reference/jdiff/operator.md" + - jdiff_utils: "code-reference/jdiff/utils/__init__.md" + - data_normalization: "code-reference/jdiff/utils/data_normalization.md" + - diff_helpers: "code-reference/jdiff/utils/diff_helpers.md" + - jmespath_parsers: "code-reference/jdiff/utils/jmespath_parsers.md" From 74316983f5bb46af62908e98928bec205e3cc2cb Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Sun, 24 Aug 2025 08:06:31 -0600 Subject: [PATCH 11/11] implement code review input --- docs/user/usage.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/usage.md b/docs/user/usage.md index f96e6d0..7ebdfd3 100644 --- a/docs/user/usage.md +++ b/docs/user/usage.md @@ -615,8 +615,6 @@ See `tests` folder in the repo for more examples. Jdiff results are very helpful in determining what is wrong with the outputs. What if you want to reconstruct the results in order to fix the problem. The `parse_diff` helper does just that. Imagine you have a `jdiff` result such as: -Examples of jdiff evaluated results: - ```python ex1 = {'bar-2': 'missing', 'bar-1': 'new'} ex2 = {