Merge pull request #347 from seperman/dev

DeepDiff 6.2.0 - Using difflib to come up with better diff results when order is

Author: seperman

README.md

@@ -12,12 +12,16 @@
 - DeepSearch: Search for objects within other objects.
 - DeepHash: Hash any object based on their content.
 
-Tested on Python 3.6+ and PyPy3.
+Tested on Python 3.7+ and PyPy3.
 
 - **[Documentation](https://zepworks.com/deepdiff/6.1.0/)**
 
 ## What is new?
 
+DeepDiff 6-2-0
+
+- Major improvement in the diff report for lists when items are all hashable and the order of items is important.
+
 DeepDiff 6-1-0
 
 - DeepDiff.affected_paths can be used to get the list of all paths where a change, addition, or deletion was reported for.

deepdiff/diff.py

@@ -114,11 +114,13 @@ class np_type:
 only_complex_number = (complex,) + numpy_complex_numbers
 only_numbers = (int, float, complex, Decimal) + numpy_numbers
 datetimes = (datetime.datetime, datetime.date, datetime.timedelta, datetime.time)
-uuids = (uuid.UUID)
+uuids = (uuid.UUID, )
 times = (datetime.datetime, datetime.time)
 numbers = only_numbers + datetimes
 booleans = (bool, np_bool_)
 
+basic_types = strings + numbers + uuids + booleans + (type(None), )
+
 IndexedHash = namedtuple('IndexedHash', 'indexes item')
 
 current_dir = os.path.dirname(os.path.abspath(__file__))

deepdiff/helper.py

@@ -96,6 +96,9 @@ def __getitem__(self, item):
             self[item] = PrettyOrderedSet()
         return self.get(item)
 
+    def __len__(self):
+        return sum([len(i) for i in self.values() if isinstance(i, PrettyOrderedSet)])
+
 
 class TextResult(ResultDict):
     ADD_QUOTES_TO_STRINGS = True

deepdiff/model.py

@@ -31,6 +31,11 @@ The DeepDiff library includes the following modules:
 What is New
 ***********
 
+DeepDiff 6-2-0
+--------------
+
+-  Major improvement in the diff report for lists when items are all hashable and the order of items is important.
+
 DeepDiff 6-1-0
 --------------
 

docs/index.rst

@@ -28,7 +28,7 @@ def get_reqs(filename):
 
 setup(name='deepdiff',
       version=version,
-      description='Deep Difference and Search of any Python object/data.',
+      description='Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other.',
       url='https://github.com/seperman/deepdiff',
       download_url='https://github.com/seperman/deepdiff/tarball/master',
       author='Seperman',
@@ -42,15 +42,14 @@ def get_reqs(filename):
       long_description=long_description,
       long_description_content_type='text/markdown',
       install_requires=reqs,
-      python_requires='>=3.6',
+      python_requires='>=3.7',
       extras_require={
           "cli": cli_reqs,
       },
       classifiers=[
           "Intended Audience :: Developers",
           "Operating System :: OS Independent",
           "Topic :: Software Development",
-          "Programming Language :: Python :: 3.6",
           "Programming Language :: Python :: 3.7",
           "Programming Language :: Python :: 3.8",
           "Programming Language :: Python :: 3.9",

setup.py

@@ -13,7 +13,9 @@ def parameterize_cases(argnames, cases):
 
     """
     argnames_list = [i.strip() for i in argnames.split(',')]
-    argvalues = [tuple(i[k] for k in argnames_list) for i in cases.values()]
+    if 'test_name' not in argnames_list:
+        argnames_list.append('test_name')
+    argvalues = [tuple(test_name if (k == 'test_name') else test_dict[k] for k in argnames_list) for test_name, test_dict in cases.items()]
     ids = list(cases.keys())
     return {'argnames': argnames, 'argvalues': argvalues, 'ids': ids}
 

tests/__init__.py

@@ -1,59 +1,40 @@
 {
-  "dictionary_item_added": [
-    "root['Cars'][3]['dealers']"
-  ],
-  "dictionary_item_removed": [
-    "root['Cars'][3]['production']"
-  ],
-  "values_changed": {
-    "root['Cars'][2]['dealers'][0]['quantity']": {
-      "new_value": 50,
-      "old_value": 20
-    },
-    "root['Cars'][1]['model_numbers'][2]": {
-      "new_value": 3,
-      "old_value": 4
-    },
-    "root['Cars'][3]['model']": {
-      "new_value": "Supra",
-      "old_value": "supra"
-    }
-  },
-  "iterable_item_added": {
-    "root['Cars'][2]['dealers'][1]": {
-      "id": 200,
-      "address": "200 Fake St",
-      "quantity": 10
+    "dictionary_item_added": [
+        "root['Cars'][3]['dealers']"
+    ],
+    "dictionary_item_removed": [
+        "root['Cars'][3]['production']"
+    ],
+    "values_changed": {
+        "root['Cars'][3]['model']": {
+            "new_value": "Supra",
+            "old_value": "supra"
+        }
     },
-    "root['Cars'][1]['model_numbers'][3]": 4,
-    "root['Cars'][0]": {
-      "id": "7",
-      "make": "Toyota",
-      "model": "8Runner"
-    }
-  },
-  "iterable_item_removed": {
-    "root['Cars'][2]['dealers'][0]": {
-      "id": 103,
-      "address": "103 Fake St",
-      "quantity": 50
+    "iterable_item_added": {
+        "root['Cars'][0]": {
+            "id": "7",
+            "make": "Toyota",
+            "model": "8Runner"
+        }
     },
-    "root['Cars'][1]": {
-      "id": "2",
-      "make": "Toyota",
-      "model": "Highlander",
-      "dealers": [
-        {
-          "id": 123,
-          "address": "123 Fake St",
-          "quantity": 50
-        },
-        {
-          "id": 125,
-          "address": "125 Fake St",
-          "quantity": 20
+    "iterable_item_removed": {
+        "root['Cars'][1]": {
+            "id": "2",
+            "make": "Toyota",
+            "model": "Highlander",
+            "dealers": [
+                {
+                    "id": 123,
+                    "address": "123 Fake St",
+                    "quantity": 50
+                },
+                {
+                    "id": 125,
+                    "address": "125 Fake St",
+                    "quantity": 20
+                }
+            ]
         }
-      ]
     }
-  }
 }

tests/fixtures/compare_func_result1.json

@@ -634,21 +634,28 @@ def test_delta_dict_items_added_retain_order(self):
         'to_delta_kwargs': {},
         'expected_delta_dict': {'iterable_item_removed': {'root[9]': 'a', 'root[10]': 'b', 'root[11]': 'c'}}
     },
+    'delta_case19_value_removed_from_the_middle_of_list': {
+        't1': [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c'],
+        't2': [0, 1, 2, 3, 5, 6, 7, 8, 'a', 'b', 'c'],
+        'deepdiff_kwargs': {},
+        'to_delta_kwargs': {'directed': True},
+        'expected_delta_dict': {'iterable_item_removed': {'root[4]': 4}}
+    },
 }
 
 
-DELTA_CASES_PARAMS = parameterize_cases('t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict', DELTA_CASES)
+DELTA_CASES_PARAMS = parameterize_cases('test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict', DELTA_CASES)
 
 
 class TestDelta:
 
     @pytest.mark.parametrize(**DELTA_CASES_PARAMS)
-    def test_delta_cases(self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict):
+    def test_delta_cases(self, test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict):
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
         delta_dict = diff._to_delta_dict(**to_delta_kwargs)
-        assert expected_delta_dict == delta_dict
+        assert expected_delta_dict == delta_dict, f"test_delta_cases {test_name} failed."
         delta = Delta(diff, verify_symmetry=False, raise_errors=True)
-        assert t1 + delta == t2
+        assert t1 + delta == t2, f"test_delta_cases {test_name} failed."
 
 
 DELTA_IGNORE_ORDER_CASES = {
@@ -931,15 +938,15 @@ def test_delta_cases(self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_de
 }
 
 DELTA_IGNORE_ORDER_CASES_PARAMS = parameterize_cases(
-    't1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta', DELTA_IGNORE_ORDER_CASES)
+    'test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta', DELTA_IGNORE_ORDER_CASES)
 
 
 class TestIgnoreOrderDelta:
 
     @pytest.mark.parametrize(**DELTA_IGNORE_ORDER_CASES_PARAMS)
     def test_ignore_order_delta_cases(
-            self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta, request):
-        test_name = request.node.callspec.id
+            self, test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta, request):
+        # test_name = request.node.callspec.id
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
         delta_dict = diff._to_delta_dict(**to_delta_kwargs)
         assert expected_delta_dict == delta_dict, f"test_ignore_order_delta_cases {test_name} failed"
@@ -1094,31 +1101,31 @@ def test_ignore_order_delta_cases(
 
 
 DELTA_NUMPY_TEST_PARAMS = parameterize_cases(
-    't1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result', DELTA_NUMPY_TEST_CASES)
+    'test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result', DELTA_NUMPY_TEST_CASES)
 
 
 class TestNumpyDelta:
 
     @pytest.mark.parametrize(**DELTA_NUMPY_TEST_PARAMS)
-    def test_numpy_delta_cases(self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result):
+    def test_numpy_delta_cases(self, test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result):
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
         delta_dict = diff._to_delta_dict(**to_delta_kwargs)
         if expected_delta_dict:
-            assert expected_delta_dict == delta_dict
+            assert expected_delta_dict == delta_dict, f"test_numpy_delta_cases {test_name} failed."
         delta = Delta(diff, verify_symmetry=False, raise_errors=True)
         if expected_result == 't2':
             result = delta + t1
-            assert np.array_equal(result, t2)
+            assert np.array_equal(result, t2), f"test_numpy_delta_cases {test_name} failed."
         elif expected_result == 't2_via_deepdiff':
             result = delta + t1
             diff = DeepDiff(result, t2, ignore_order=True, report_repetition=True)
-            assert not diff
+            assert not diff, f"test_numpy_delta_cases {test_name} failed."
         elif expected_result is DeltaNumpyOperatorOverrideError:
             with pytest.raises(DeltaNumpyOperatorOverrideError):
-                assert t1 + delta
+                t1 + delta
         else:
             result = delta + t1
-            assert np.array_equal(result, expected_result)
+            assert np.array_equal(result, expected_result), f"test_numpy_delta_cases {test_name} failed."
 
     def test_invalid_numpy_type(self):
 
@@ -1510,11 +1517,27 @@ def test_compare_func_with_duplicates_removed(self):
         t2 = [{'id': 3, 'val': 3}, {'id': 2, 'val': 2}, {'id': 1, 'val': 3}]
         ddiff = DeepDiff(t1, t2, iterable_compare_func=self.compare_func, verbose_level=2)
         expected = {
-            'values_changed': {"root[2]['val']": {'new_value': 3, 'old_value': 1}},
-            'iterable_item_removed': {'root[2]': {'id': 1, 'val': 3}},
-            'iterable_item_moved': {
-                'root[0]': {'new_path': 'root[2]', 'value': {'id': 1, 'val': 3}},
-                'root[3]': {'new_path': 'root[0]', 'value': {'id': 3, 'val': 3}}
+            "iterable_item_removed": {
+                "root[2]": {
+                    "id": 1,
+                    "val": 3
+                }
+            },
+            "iterable_item_moved": {
+                "root[0]": {
+                    "new_path": "root[2]",
+                    "value": {
+                        "id": 1,
+                        "val": 3
+                    }
+                },
+                "root[3]": {
+                    "new_path": "root[0]",
+                    "value": {
+                        "id": 3,
+                        "val": 3
+                    }
+                }
             }
         }
         assert expected == ddiff
@@ -1527,11 +1550,27 @@ def test_compare_func_with_duplicates_added(self):
         t2 = [{'id': 1, 'val': 1}, {'id': 2, 'val': 2}, {'id': 1, 'val': 3}, {'id': 3, 'val': 3}]
         ddiff = DeepDiff(t1, t2, iterable_compare_func=self.compare_func, verbose_level=2)
         expected = {
-            'values_changed': {"root[0]['val']": {'new_value': 1, 'old_value': 3}},
-            'iterable_item_added': {'root[2]': {'id': 1, 'val': 3}},
+            'iterable_item_added': {
+                'root[2]': {
+                    'id': 1,
+                    'val': 3
+                }
+            },
             'iterable_item_moved': {
-                'root[2]': {'new_path': 'root[0]', 'value': {'id': 1, 'val': 1}},
-                'root[0]': {'new_path': 'root[3]', 'value': {'id': 3, 'val': 3}}
+                'root[0]': {
+                    'new_path': 'root[3]',
+                    'value': {
+                        'id': 3,
+                        'val': 3
+                    }
+                },
+                'root[2]': {
+                    'new_path': 'root[0]',
+                    'value': {
+                        'id': 1,
+                        'val': 1
+                    }
+                }
             }
         }
         assert expected == ddiff

tests/test_delta.py

@@ -132,12 +132,12 @@
 }
 
 
-NUMPY_CASES_PARAMS = parameterize_cases('t1, t2, deepdiff_kwargs, expected_result', NUMPY_CASES)
+NUMPY_CASES_PARAMS = parameterize_cases('test_name, t1, t2, deepdiff_kwargs, expected_result', NUMPY_CASES)
 
 
 class TestNumpy:
 
     @pytest.mark.parametrize(**NUMPY_CASES_PARAMS)
-    def test_numpy(self, t1, t2, deepdiff_kwargs, expected_result):
+    def test_numpy(self, test_name, t1, t2, deepdiff_kwargs, expected_result):
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
-        assert expected_result == diff
+        assert expected_result == diff, f"test_numpy {test_name} failed."