Include type info and change the "unknown" value for flat rows to

something that is friendly for Postgres enums

Author: seperman

deepdiff/delta.py

@@ -878,7 +878,7 @@ def to_dict(self):
         return dict(self.diff)
 
     @staticmethod
-    def _get_flat_row(action, info, _parse_path, keys_and_funcs):
+    def _get_flat_row(action, info, _parse_path, keys_and_funcs, report_type_changes=True):
         for path, details in info.items():
             row = {'path': _parse_path(path), 'action': action}
             for key, new_key, func in keys_and_funcs:
@@ -887,6 +887,11 @@ def _get_flat_row(action, info, _parse_path, keys_and_funcs):
                         row[new_key] = func(details[key])
                     else:
                         row[new_key] = details[key]
+            if report_type_changes:
+                if 'value' in row and 'type' not in row:
+                    row['type'] = type(row['value'])
+                if 'old_value' in row and 'old_type' not in row:
+                    row['old_type'] = type(row['old_value'])
             yield FlatDeltaRow(**row)
 
     @staticmethod
@@ -1060,6 +1065,9 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
             'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed',
         }
         for action, info in self.diff.items():
+            if action == '_iterable_opcodes':
+                result.extend(self._flatten_iterable_opcodes())
+                continue
             if action.startswith('_'):
                 continue
             if action in FLATTENING_NEW_ACTION_MAP:
@@ -1072,12 +1080,20 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
                             path2.append((index, 'GET'))
                         else:
                             path2.append(index)
-                        result.append(FlatDeltaRow(path=path2, value=value, action=new_action))
+                        if report_type_changes:
+                            row = FlatDeltaRow(path=path2, value=value, action=new_action, type=type(value))
+                        else:
+                            row = FlatDeltaRow(path=path2, value=value, action=new_action)
+                        result.append(row)
             elif action in {'set_item_added', 'set_item_removed'}:
                 for path, values in info.items():
                     path = _parse_path(path)
                     for value in values:
-                        result.append(FlatDeltaRow(path=path, value=value, action=action))
+                        if report_type_changes:
+                            row = FlatDeltaRow(path=path, value=value, action=action, type=type(value))
+                        else:
+                            row = FlatDeltaRow(path=path, value=value, action=action)
+                        result.append(row)
             elif action == 'dictionary_item_added':
                 for path, value in info.items():
                     path = _parse_path(path)
@@ -1092,14 +1108,22 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
                     elif isinstance(value, set) and len(value) == 1:
                         value = value.pop()
                         action = 'set_item_added'
-                    result.append(FlatDeltaRow(path=path, value=value, action=action))
+                    if report_type_changes:
+                        row = FlatDeltaRow(path=path, value=value, action=action, type=type(value))
+                    else:
+                        row = FlatDeltaRow(path=path, value=value, action=action)
+                    result.append(row)
             elif action in {
                 'dictionary_item_removed', 'iterable_item_added',
                 'iterable_item_removed', 'attribute_removed', 'attribute_added'
             }:
                 for path, value in info.items():
                     path = _parse_path(path)
-                    result.append(FlatDeltaRow(path=path, value=value, action=action))
+                    if report_type_changes:
+                        row = FlatDeltaRow(path=path, value=value, action=action, type=type(value))
+                    else:
+                        row = FlatDeltaRow(path=path, value=value, action=action)
+                    result.append(row)
             elif action == 'type_changes':
                 if not report_type_changes:
                     action = 'values_changed'
@@ -1109,16 +1133,16 @@ def to_flat_rows(self, include_action_in_path=False, report_type_changes=True) -
                     info=info,
                     _parse_path=_parse_path,
                     keys_and_funcs=keys_and_funcs,
+                    report_type_changes=report_type_changes,
                 ):
                     result.append(row)
-            elif action == '_iterable_opcodes':
-                result.extend(self._flatten_iterable_opcodes())
             else:
                 for row in self._get_flat_row(
                     action=action,
                     info=info,
                     _parse_path=_parse_path,
                     keys_and_funcs=keys_and_funcs,
+                    report_type_changes=report_type_changes,
                 ):
                     result.append(row)
         return result

deepdiff/helper.py

@@ -771,7 +771,7 @@ class FlatDataAction(str, enum.Enum):
     unordered_iterable_item_removed = 'unordered_iterable_item_removed'
 
 
-UnkownValueCode = '*-UNKNOWN-*'
+UnkownValueCode = 'unknown___'
 
 
 class FlatDeltaRow(NamedTuple):

docs/serialization.rst

@@ -181,7 +181,7 @@ Flat Row Specs:
         unordered_iterable_item_removed = 'unordered_iterable_item_removed'
 
 
-    UnkownValueCode = '*-UNKNOWN-*'
+    UnkownValueCode = 'unknown___'
 
 
     class FlatDeltaRow(NamedTuple):
@@ -205,7 +205,7 @@ Delta Serialize To Flat Dictionaries
 
 Sometimes, it is desired to serialize a :ref:`delta_label` object to a list of flat dictionaries. For example, to store them in relation databases. In that case, you can use the Delta.to_flat_dicts to achieve the desired outcome.
 
-Since None is a valid value, we use a special hard-coded string to signify "unkown": '*-UNKNOWN-*'
+Since None is a valid value, we use a special hard-coded string to signify "unkown": 'unknown___'
 
 .. note::
     Many new keys are added to the flat dicts in DeepDiff 7.0.0
@@ -226,25 +226,25 @@ For example:
     >>> pprint(flat_dicts, indent=2)
     [ { 'action': 'dictionary_item_added',
         'new_path': None,
-        'old_type': '*-UNKNOWN-*',
-        'old_value': '*-UNKNOWN-*',
+        'old_type': 'unknown___',
+        'old_value': 'unknown___',
         'path': ['field2', 'key2'],
         't1_from_index': None,
         't1_to_index': None,
         't2_from_index': None,
         't2_to_index': None,
-        'type': '*-UNKNOWN-*',
+        'type': 'unknown___',
         'value': 'value2'},
       { 'action': 'dictionary_item_removed',
         'new_path': None,
-        'old_type': '*-UNKNOWN-*',
-        'old_value': '*-UNKNOWN-*',
+        'old_type': 'unknown___',
+        'old_value': 'unknown___',
         'path': ['key1'],
         't1_from_index': None,
         't1_to_index': None,
         't2_from_index': None,
         't2_to_index': None,
-        'type': '*-UNKNOWN-*',
+        'type': 'unknown___',
         'value': 'value1'}]
 
 
@@ -261,25 +261,25 @@ Example 2:
     >>> pprint(flat_dicts, indent=2)
     [ { 'action': 'iterable_item_added',
         'new_path': None,
-        'old_type': '*-UNKNOWN-*',
-        'old_value': '*-UNKNOWN-*',
+        'old_type': 'unknown___',
+        'old_value': 'unknown___',
         'path': [2],
         't1_from_index': None,
         't1_to_index': None,
         't2_from_index': None,
         't2_to_index': None,
-        'type': '*-UNKNOWN-*',
+        'type': 'unknown___',
         'value': 'C'},
       { 'action': 'iterable_item_added',
         'new_path': None,
-        'old_type': '*-UNKNOWN-*',
-        'old_value': '*-UNKNOWN-*',
+        'old_type': 'unknown___',
+        'old_value': 'unknown___',
         'path': [3],
         't1_from_index': None,
         't1_to_index': None,
         't2_from_index': None,
         't2_to_index': None,
-        'type': '*-UNKNOWN-*',
+        'type': 'unknown___',
         'value': 'D'}]
 
 

tests/test_delta.py

@@ -73,8 +73,8 @@ def test_list_difference_add_delta(self):
 
         flat_result1 = delta.to_flat_rows()
         flat_expected1 = [
-            FlatDeltaRow(path=[3], value=5, action='iterable_item_added'),
-            FlatDeltaRow(path=[2], value=3, action='iterable_item_added'),
+            FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int),
+            FlatDeltaRow(path=[2], value=3, action='iterable_item_added', type=int),
         ]
 
         assert flat_expected1 == flat_result1
@@ -291,9 +291,9 @@ def test_list_difference3_delta(self):
 
         flat_result1 = delta.to_flat_rows()
         flat_expected1 = [
-            FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5),
-            FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2),
-            FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added'),
+            FlatDeltaRow(path=[4, 'b', 2], action='values_changed', value=2, old_value=5, type=int, old_type=int),
+            FlatDeltaRow(path=[4, 'b', 1], action='values_changed', value=3, old_value=2, type=int, old_type=int),
+            FlatDeltaRow(path=[4, 'b', 3], value=5, action='iterable_item_added', type=int),
         ]
 
         assert flat_expected1 == flat_result1
@@ -332,9 +332,9 @@ def test_list_difference_delta_raises_error_if_prev_value_does_not_match(self):
 
         flat_result2 = delta2.to_flat_rows()
         flat_expected2 = [
-            FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5),
-            FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2),
-            FlatDeltaRow(path=[3], value=5, action='iterable_item_added'),
+            FlatDeltaRow(path=[2], action='values_changed', value=2, old_value=5, type=int, old_type=int),
+            FlatDeltaRow(path=[1], action='values_changed', value=3, old_value=2, type=int, old_type=int),
+            FlatDeltaRow(path=[3], value=5, action='iterable_item_added', type=int),
         ]
 
         assert flat_expected2 == flat_result2
@@ -363,8 +363,8 @@ def test_list_difference_delta1(self):
 
         flat_result = delta.to_flat_rows()
         flat_expected = [
-            FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed'),
-            FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed'),
+            FlatDeltaRow(path=[4, 'b', 2], value='to_be_removed', action='iterable_item_removed', type=str),
+            FlatDeltaRow(path=[4, 'b', 3], value='to_be_removed2', action='iterable_item_removed', type=str),
         ]
 
         assert flat_expected == flat_result
@@ -567,7 +567,8 @@ def compare_func(item1, item2, level=None):
                                              'professionalDesignation': '',
                                              'suffix': 'SR',
                                              'nameIdentifier': '00003'},
-                                   action='unordered_iterable_item_added'),
+                                   action='unordered_iterable_item_added',
+                                   type=dict),
                                   FlatDeltaRow(path=['individualNames', 1],
                                    value={'firstName': 'John',
                                              'lastName': 'Doe',
@@ -577,7 +578,9 @@ def compare_func(item1, item2, level=None):
                                              'professionalDesignation': '',
                                              'suffix': 'SR',
                                              'nameIdentifier': '00002'},
-                                   action='unordered_iterable_item_removed')]
+                                   action='unordered_iterable_item_removed',
+                                   type=dict),
+                                  ]
 
         preserved_flat_dict_list = copy.deepcopy(flat_rows_list)  # Use this later for assert comparison
 
@@ -1405,13 +1408,13 @@ def test_list_ignore_order_various_deltas2(self):
 
         flat_result1 = delta1.to_flat_rows()
         flat_expected1 = [
-            {'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added'},
-            {'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added'},
-            {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'},
-            {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'},
-            {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'},
-            {'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed'},
-            {'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed'},
+            {'path': [0], 'value': 7, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [6], 'value': 8, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [6], 'value': 6, 'action': 'unordered_iterable_item_removed', 'type': int},
+            {'path': [0], 'value': 5, 'action': 'unordered_iterable_item_removed', 'type': int},
         ]
         flat_expected1 = [FlatDeltaRow(**i) for i in flat_expected1]
         assert flat_expected1 == flat_result1
@@ -1422,11 +1425,11 @@ def test_list_ignore_order_various_deltas2(self):
 
         flat_result2 = delta2.to_flat_rows()
         flat_expected2 = [
-            {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added'},
-            {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added'},
-            {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added'},
-            {'path': [6], 'action': 'values_changed', 'value': 7},
-            {'path': [0], 'action': 'values_changed', 'value': 8},
+            {'path': [1], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [2], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [5], 'value': 4, 'action': 'unordered_iterable_item_added', 'type': int},
+            {'path': [6], 'action': 'values_changed', 'value': 7, 'type': int},
+            {'path': [0], 'action': 'values_changed', 'value': 8, 'type': int},
         ]
         flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2]
         assert flat_expected2 == flat_result2
@@ -1565,7 +1568,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self):
         assert [] == t4
 
         flat_result2 = delta2.to_flat_rows()
-        flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5}]
+        flat_expected2 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'type': int}]
         flat_expected2 = [FlatDeltaRow(**i) for i in flat_expected2]
 
         assert flat_expected2 == flat_result2
@@ -1575,7 +1578,7 @@ def test_apply_delta_to_incompatible_object6_value_change(self):
 
         delta3 = Delta(diff, raise_errors=False, bidirectional=True)
         flat_result3 = delta3.to_flat_rows()
-        flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4}]
+        flat_expected3 = [{'path': [1, 2, 0], 'action': 'values_changed', 'value': 5, 'old_value': 4, 'type': int, 'old_type': int}]
         flat_expected3 = [FlatDeltaRow(**i) for i in flat_expected3]
 
         assert flat_expected3 == flat_result3
@@ -1685,7 +1688,7 @@ def test_delta_to_dict(self):
         assert expected == result
 
         flat_result = delta.to_flat_rows()
-        flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B'}]
+        flat_expected = [{'action': 'unordered_iterable_item_removed', 'path': [2], 'value': 'B', 'type': str}]
         flat_expected = [FlatDeltaRow(**i) for i in flat_expected]
 
         assert flat_expected == flat_result
@@ -1766,10 +1769,10 @@ def test_delta_set_in_objects(self):
         delta = Delta(DeepDiff(t1, t2))
         flat_result = delta.to_flat_rows()
         flat_expected = [
-            {'path': [0, 1], 'value': 10, 'action': 'set_item_added'},
-            {'path': [0, 0], 'action': 'values_changed', 'value': 2},
-            {'path': [0, 1], 'value': 'A', 'action': 'set_item_removed'},
-            {'path': [0, 1], 'value': 'C', 'action': 'set_item_added'},
+            {'path': [0, 1], 'value': 10, 'action': 'set_item_added', 'type': int},
+            {'path': [0, 0], 'action': 'values_changed', 'value': 2, 'type': int},
+            {'path': [0, 1], 'value': 'A', 'action': 'set_item_removed', 'type': str},
+            {'path': [0, 1], 'value': 'C', 'action': 'set_item_added', 'type': str},
         ]
         flat_expected = [FlatDeltaRow(**i) for i in flat_expected]
 
@@ -1885,11 +1888,11 @@ def test_compare_func_with_duplicates_removed(self):
 
         flat_result = delta.to_flat_rows()
         flat_expected = [
-            {'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'},
-            {'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed'},
-            {'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed'},
-        {'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2]},
-            {'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0]},
+            {'path': [2], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict},
+            {'path': [0], 'value': {'id': 1, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict},
+            {'path': [3], 'value': {'id': 3, 'val': 3}, 'action': 'iterable_item_removed', 'type': dict},
+            {'path': [0], 'action': 'iterable_item_moved', 'value': {'id': 1, 'val': 3}, 'new_path': [2], 'type': dict},
+            {'path': [3], 'action': 'iterable_item_moved', 'value': {'id': 3, 'val': 3}, 'new_path': [0], 'type': dict},
         ]
         flat_expected = [FlatDeltaRow(**i) for i in flat_expected]
 
@@ -2289,11 +2292,13 @@ def test_subtract_delta_made_from_flat_dicts1(self):
         expected_flat_dicts = [{
             'path': ['field_name1', 0],
             'value': 'xxx',
-            'action': 'iterable_item_removed'
+            'action': 'iterable_item_removed',
+            'type': str,
         }, {
             'path': ['field_name1', 1],
             'value': 'yyy',
-            'action': 'iterable_item_removed'
+            'action': 'iterable_item_removed',
+            'type': str,
         }]
         expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts]
 
@@ -2318,11 +2323,13 @@ def test_subtract_delta_made_from_flat_dicts2(self):
         expected_flat_dicts = [{
             'path': ['field_name1', 0],
             'value': 'xxx',
-            'action': 'iterable_item_added'
+            'action': 'iterable_item_added',
+            'type': str,
         }, {
             'path': ['field_name1', 1],
             'value': 'yyy',
-            'action': 'iterable_item_added'
+            'action': 'iterable_item_added',
+            'type': str,
         }]
         expected_flat_dicts = [FlatDeltaRow(**i) for i in expected_flat_dicts]