Skip to content

Commit fbed747

Browse files
jeffkalajeffkala
committed
create helpers for json compliance in GC
1 parent be82cec commit fbed747

File tree

3 files changed

+223
-22
lines changed

3 files changed

+223
-22
lines changed

‎jdiff/extract_data.py‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,24 @@
11
"""Extract data from JSON. Based on custom JMSPath implementation."""
2+
23
import re
34
import warnings
4-
from typing import Mapping, List, Dict, Any, Union, Optional
5+
from typing import Any, Dict, List, Mapping, Optional, Union
6+
57
import jmespath
8+
69
from .utils.data_normalization import exclude_filter, flatten_list
710
from .utils.jmespath_parsers import (
8-
jmespath_value_parser,
9-
jmespath_refkey_parser,
1011
associate_key_of_my_value,
12+
jmespath_refkey_parser,
13+
jmespath_value_parser,
1114
keys_values_zipper,
1215
multi_reference_keys,
1316
)
1417

1518

16-
def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude: Optional[List] = None) -> Any:
19+
def extract_data_from_json(
20+
data: Union[Mapping, List], path: str = "*", exclude: Optional[List] = None
21+
) -> Any:
1722
"""Return wanted data from outpdevice data based on the check path. See unit test for complete example.
1823
1924
Get the wanted values to be evaluated if JMESPath expression is defined,
@@ -32,12 +37,16 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude:
3237
"""
3338
if exclude and isinstance(data, Dict):
3439
if not isinstance(exclude, list):
35-
raise ValueError(f"Exclude list must be defined as a list. You have {type(exclude)}")
40+
raise ValueError(
41+
f"Exclude list must be defined as a list. You have {type(exclude)}"
42+
)
3643
# exclude unwanted elements
3744
exclude_filter(data, exclude)
3845

3946
if not path:
40-
warnings.warn("JMSPath cannot be empty string or type 'None'. Path argument reverted to default value '*'")
47+
warnings.warn(
48+
"JMSPath cannot be empty string or type 'None'. Path argument reverted to default value '*'"
49+
)
4150
path = "*"
4251

4352
if path == "*":
@@ -48,7 +57,10 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude:
4857
if len(re.findall(r"\$.*?\$", path)) > 1:
4958
clean_path = path.replace("$", "")
5059
values = jmespath.search(f"{clean_path}{' | []' * (path.count('*') - 1)}", data)
51-
return keys_values_zipper(multi_reference_keys(path, data), associate_key_of_my_value(clean_path, values))
60+
return keys_values_zipper(
61+
multi_reference_keys(path, data),
62+
associate_key_of_my_value(clean_path, values),
63+
)
5264

5365
values = jmespath.search(jmespath_value_parser(path), data)
5466

@@ -73,19 +85,27 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude:
7385
# Based on the expression or data we might have different data types
7486
# therefore we need to normalize.
7587
if re.search(r"\$.*\$", path):
76-
paired_key_value = associate_key_of_my_value(jmespath_value_parser(path), values)
88+
paired_key_value = associate_key_of_my_value(
89+
jmespath_value_parser(path), values
90+
)
7791
wanted_reference_keys = jmespath.search(jmespath_refkey_parser(path), data)
7892

79-
if isinstance(wanted_reference_keys, dict): # when wanted_reference_keys is dict() type
93+
if isinstance(
94+
wanted_reference_keys, dict
95+
): # when wanted_reference_keys is dict() type
8096
list_of_reference_keys = list(wanted_reference_keys.keys())
8197
elif any(
8298
isinstance(element, list) for element in wanted_reference_keys
8399
): # when wanted_reference_keys is a nested list
84100
list_of_reference_keys = flatten_list(wanted_reference_keys)[0]
85-
elif isinstance(wanted_reference_keys, list): # when wanted_reference_keys is a list
101+
elif isinstance(
102+
wanted_reference_keys, list
103+
): # when wanted_reference_keys is a list
86104
list_of_reference_keys = wanted_reference_keys
87105
else:
88-
raise ValueError("Reference Key normalization failure. Please verify data type returned.")
106+
raise ValueError(
107+
"Reference Key normalization failure. Please verify data type returned."
108+
)
89109

90110
normalized = keys_values_zipper(list_of_reference_keys, paired_key_value)
91111
# Data between pre and post may come in different order, so it needs to be sorted.

‎jdiff/utils/diff_helpers.py‎

Lines changed: 100 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
"""Diff helpers."""
2+
23
import re
34
from collections import defaultdict
45
from functools import partial
5-
from typing import Mapping, Dict, List, DefaultDict
6+
from typing import DefaultDict, Dict, List, Mapping
67

78
REGEX_PATTERN_RELEVANT_KEYS = r"'([A-Za-z0-9_\./\\-]*)'"
89

@@ -62,8 +63,12 @@ def fix_deepdiff_key_names(obj: Mapping) -> Dict:
6263
result = {} # type: Dict
6364
for key, value in obj.items():
6465
key_parts = re.findall(REGEX_PATTERN_RELEVANT_KEYS, key)
65-
if not key_parts: # If key parts can't be find, keep original key so data is not lost.
66-
key_parts = [key.replace("root", "index_element")] # replace root from DeepDiff with more meaningful name.
66+
if (
67+
not key_parts
68+
): # If key parts can't be find, keep original key so data is not lost.
69+
key_parts = [
70+
key.replace("root", "index_element")
71+
] # replace root from DeepDiff with more meaningful name.
6772
partial_res = group_value(key_parts, value)
6873
dict_merger(result, partial_res)
6974
return result
@@ -79,9 +84,99 @@ def group_value(tree_list: List, value: Dict) -> Dict:
7984
def dict_merger(original_dict: Dict, dict_to_merge: Dict):
8085
"""Function to merge a dictionary (dict_to_merge) recursively into the original_dict."""
8186
for key in dict_to_merge.keys():
82-
if key in original_dict and isinstance(original_dict[key], dict) and isinstance(dict_to_merge[key], dict):
87+
if (
88+
key in original_dict
89+
and isinstance(original_dict[key], dict)
90+
and isinstance(dict_to_merge[key], dict)
91+
):
8392
dict_merger(original_dict[key], dict_to_merge[key])
8493
elif key in original_dict.keys():
85-
original_dict[key + "_dup!"] = dict_to_merge[key] # avoid overwriting existing keys.
94+
original_dict[key + "_dup!"] = dict_to_merge[
95+
key
96+
] # avoid overwriting existing keys.
8697
else:
8798
original_dict[key] = dict_to_merge[key]
99+
100+
101+
def _parse_index_element_string(index_element_string):
102+
"""Build out dictionary from the index element string."""
103+
result = {}
104+
pattern = r"\[\'(.*?)\'\]"
105+
match = re.findall(pattern, index_element_string)
106+
if match:
107+
for inner_key in match[1::]:
108+
result[inner_key] = ""
109+
return result
110+
111+
112+
def parse_diff(jdiff_evaluate_response, actual, intended, match_config):
113+
"""Parse jdiff evaluate result into missing and extra dictionaries."""
114+
extra = {}
115+
missing = {}
116+
117+
def process_diff(_map, extra_map, missing_map):
118+
for key, value in _map.items():
119+
if (
120+
isinstance(value, dict)
121+
and "new_value" in value
122+
and "old_value" in value
123+
):
124+
extra_map[key] = value["old_value"]
125+
missing_map[key] = value["new_value"]
126+
elif isinstance(value, str):
127+
if "missing" in value:
128+
extra_map[key] = actual.get(match_config, {}).get(key)
129+
if "new" in value:
130+
new_key = _parse_index_element_string(key)
131+
new_key[key] = intended.get(match_config, {}).get(key)
132+
missing_map.update(new_key)
133+
elif isinstance(value, dict):
134+
extra_map[key] = {}
135+
missing_map[key] = {}
136+
process_diff(value, extra_map[key], missing_map[key])
137+
return extra_map, missing_map
138+
139+
extras, missings = process_diff(jdiff_evaluate_response, extra, missing)
140+
return extras, missings
141+
142+
143+
# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing'}
144+
# result = {'domain-name': 'missing'}
145+
# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'}
146+
# result = {'domain-name': 'missing','hostname': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'}
147+
# result = {'servers': {'server': defaultdict(<class 'list'>, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}}
148+
149+
# '''
150+
# ```
151+
# from jdiff import CheckType
152+
153+
# a = {
154+
# "openconfig-system:ntp": {
155+
# "servers": {
156+
# "server": [
157+
# {
158+
# "address": "1.us.pool.ntp.org",
159+
# "config": {
160+
# "address": "1.us.pool.ntp.org"
161+
# },
162+
# "state": {
163+
# "address": "1.us.pool.ntp.org"
164+
# }
165+
# }
166+
# ]
167+
# }
168+
# }
169+
# }
170+
171+
# i = {
172+
# "openconfig-system:ntp": {
173+
# "servers": {
174+
# "server": []
175+
# }
176+
# }
177+
# }
178+
179+
# jdiff_param_match = CheckType.create("exact_match")
180+
# result, compliant = jdiff_param_match.evaluate(a, i)
181+
# ```
182+
# '''

‎tests/test_diff_helpers.py‎

Lines changed: 92 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
"""DIff helpers unit tests."""
2-
from jdiff.utils.diff_helpers import dict_merger, group_value, fix_deepdiff_key_names, get_diff_iterables_items
2+
3+
import pytest
4+
from jdiff.utils.diff_helpers import (
5+
_parse_index_element_string,
6+
dict_merger,
7+
fix_deepdiff_key_names,
8+
get_diff_iterables_items,
9+
group_value,
10+
parse_diff,
11+
)
312

413

514
def test_dict_merger():
@@ -21,12 +30,16 @@ def test_group_value():
2130
"""Tests that nested dict is recursively created."""
2231
tree_list = ["10.1.0.0", "is_enabled"]
2332
value = {"new_value": False, "old_value": True}
24-
assert group_value(tree_list, value) == {"10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}}}
33+
assert group_value(tree_list, value) == {
34+
"10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}}
35+
}
2536

2637

2738
def test_fix_deepdiff_key_names():
2839
"""Tests that deepdiff return is parsed properly."""
29-
deepdiff_object = {"root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True}}
40+
deepdiff_object = {
41+
"root[0]['10.1.0.0']['is_enabled']": {"new_value": False, "old_value": True}
42+
}
3043
assert fix_deepdiff_key_names(deepdiff_object) == {
3144
"10.1.0.0": {"is_enabled": {"new_value": False, "old_value": True}}
3245
}
@@ -35,12 +48,85 @@ def test_fix_deepdiff_key_names():
3548
def test_get_diff_iterables_items():
3649
"""Tests that deepdiff return is parsed properly."""
3750
diff_result = {
38-
"values_changed": {"root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"}},
51+
"values_changed": {
52+
"root['Ethernet1'][0]['port']": {"new_value": "518", "old_value": "519"}
53+
},
3954
"iterable_item_added": {
40-
"root['Ethernet3'][1]": {"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"},
55+
"root['Ethernet3'][1]": {
56+
"hostname": "ios-xrv-unittest",
57+
"port": "Gi0/0/0/0",
58+
},
4159
},
4260
}
4361
result = get_diff_iterables_items(diff_result)
4462

4563
assert list(dict(result).keys())[0] == "['Ethernet3']"
46-
assert list(list(dict(result).values())[0].values())[0] == [{"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"}]
64+
assert list(list(dict(result).values())[0].values())[0] == [
65+
{"hostname": "ios-xrv-unittest", "port": "Gi0/0/0/0"}
66+
]
67+
68+
69+
# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing'}
70+
# result = {'domain-name': 'missing'}
71+
# result = {'hostname': {'new_value': 'veos', 'old_value': 'veos-0'}, 'domain-name': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'}
72+
# result = {'domain-name': 'missing','hostname': 'missing', "index_element['openconfig-system:config']['ip name']": 'new'}
73+
# result = {'servers': {'server': defaultdict(<class 'list'>, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}}
74+
75+
76+
index_element_case_1 = (
77+
"index_element['foo']['ip name']",
78+
{"ip name": ""},
79+
)
80+
81+
index_element_case_2 = (
82+
"index_element['foo']['ip name']['ip domain']",
83+
{"ip name": "", "ip domain": ""},
84+
)
85+
86+
87+
index_element_tests = [index_element_case_1, index_element_case_2]
88+
89+
90+
@pytest.mark.parametrize("index_element, result", index_element_tests)
91+
def test__parse_index_element_string(index_element, result):
92+
"""Test that index_element can be unpacked."""
93+
parsed_result = _parse_index_element_string(index_element)
94+
assert parsed_result == result
95+
96+
97+
parse_diff_case_1 = (
98+
{
99+
"hostname": {"new_value": "veos", "old_value": "veos-0"},
100+
"domain-name": "missing",
101+
},
102+
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}},
103+
{"openconfig-system:config": {"hostname": "veos"}},
104+
"openconfig-system:config",
105+
{"hostname": "veos-0"},
106+
{"hostname": "veos", "domain-name": "ntc.com"},
107+
)
108+
109+
110+
parse_diff_tests = [parse_diff_case_1]
111+
112+
113+
@pytest.mark.parametrize(
114+
"jdiff_evaluate_response, actual, intended, match_config, extra, missing",
115+
parse_diff_tests,
116+
)
117+
def test_parse_diff(
118+
jdiff_evaluate_response, actual, intended, match_config, extra, missing
119+
):
120+
"""Test that index_element can be unpacked."""
121+
parsed_extra, parsed_missing = parse_diff(
122+
jdiff_evaluate_response,
123+
actual,
124+
intended,
125+
match_config,
126+
)
127+
assert (
128+
parsed_extra == extra
129+
) # AssertionError: assert {'hostname': 'veos-0', 'domain-name': 'ntc.com'} == {'hostname': 'veos-0'}
130+
assert (
131+
parsed_missing == missing
132+
) # AssertionError: assert {'hostname': 'veos'} == {'hostname': 'veos', 'domain-name': 'ntc.com'}

0 commit comments

Comments
 (0)