fixing tests

Author: seperman

CHANGELOG.md

@@ -1,5 +1,6 @@
 DeepDiff Change log
 
+- v5-2-2: Fixed Delta serialization when None type is present.
 - v5-2-0: Removed Murmur3 as the preferred hashing method. Using SHA256 by default now. Added commandline for deepdiff. Added group_by. Added math_epsilon. Improved ignoring of NoneType.
 - v5-0-2: Bug Fix NoneType in ignore type groups https://github.com/seperman/deepdiff/issues/207
 - v5-0-1: Bug fix to not apply format to non numbers.

deepdiff/commands.py

@@ -48,6 +48,7 @@ def cli():
 @click.option('--significant-digits', required=False, default=None, type=int, show_default=True)
 @click.option('--truncate-datetime', required=False, type=click.Choice(['second', 'minute', 'hour', 'day'], case_sensitive=True), show_default=True, default=None)
 @click.option('--verbose-level', required=False, default=1, type=click.IntRange(0, 2), show_default=True)
+@click.option('--debug', is_flag=True, show_default=False)
 def diff(
     *args, **kwargs
 ):
@@ -59,6 +60,7 @@ def diff(
 
     T1 and T2 are the path to the files to be compared with each other.
     """
+    debug = kwargs.pop('debug')
     kwargs['ignore_private_variables'] = not kwargs.pop('include_private_variables')
     kwargs['progress_logger'] = logger.info if kwargs['progress_logger'] == 'info' else logger.error
     create_patch = kwargs.pop('create_patch')
@@ -71,7 +73,10 @@ def diff(
         try:
             kwargs[name] = load_path_content(t_path, file_type=t_extension)
         except Exception as e:  # pragma: no cover.
-            sys.exit(str(f"Error when loading {name}: {e}"))  # pragma: no cover.
+            if debug:  # pragma: no cover.
+                raise  # pragma: no cover.
+            else:  # pragma: no cover.
+                sys.exit(str(f"Error when loading {name}: {e}"))  # pragma: no cover.
 
     # if (t1_extension != t2_extension):
     if t1_extension in {'csv', 'tsv'}:
@@ -92,7 +97,10 @@ def diff(
         try:
             delta = Delta(diff)
         except Exception as e:  # pragma: no cover.
-            sys.exit(f"Error when loading the patch (aka delta): {e}")  # pragma: no cover.
+            if debug:  # pragma: no cover.
+                raise  # pragma: no cover.
+            else:  # pragma: no cover.
+                sys.exit(f"Error when loading the patch (aka delta): {e}")  # pragma: no cover.
 
         # printing into stdout
         sys.stdout.buffer.write(delta.dumps())
@@ -105,8 +113,9 @@ def diff(
 @click.argument('delta_path', type=click.Path(exists=True, resolve_path=True))
 @click.option('--backup', '-b', is_flag=True, show_default=True)
 @click.option('--raise-errors', is_flag=True, show_default=True)
+@click.option('--debug', is_flag=True, show_default=False)
 def patch(
-    path, delta_path, backup, raise_errors
+    path, delta_path, backup, raise_errors, debug
 ):
     """
     Deep Patch Commandline
@@ -123,7 +132,10 @@ def patch(
     try:
         delta = Delta(delta_path=delta_path, raise_errors=raise_errors)
     except Exception as e:  # pragma: no cover.
-        sys.exit(str(f"Error when loading the patch (aka delta) {delta_path}: {e}"))  # pragma: no cover.
+        if debug:  # pragma: no cover.
+            raise  # pragma: no cover.
+        else:  # pragma: no cover.
+            sys.exit(str(f"Error when loading the patch (aka delta) {delta_path}: {e}"))  # pragma: no cover.
 
     extension = path.split('.')[-1]
 
@@ -137,7 +149,10 @@ def patch(
     try:
         save_content_to_path(result, path, file_type=extension, keep_backup=backup)
     except Exception as e:  # pragma: no cover.
-        sys.exit(str(f"Error when saving {path}: {e}"))  # pragma: no cover.
+        if debug:  # pragma: no cover.
+            raise  # pragma: no cover.
+        else:  # pragma: no cover.
+            sys.exit(str(f"Error when saving {path}: {e}"))  # pragma: no cover.
 
 
 @cli.command()
@@ -148,7 +163,8 @@ def patch(
 @click.option('--exclude-paths', required=False, type=str, show_default=False, multiple=True)
 @click.option('--exclude-regex-paths', required=False, type=str, show_default=False, multiple=True)
 @click.option('--verbose-level', required=False, default=1, type=click.IntRange(0, 2), show_default=True)
-def grep(item, path, **kwargs):
+@click.option('--debug', is_flag=True, show_default=False)
+def grep(item, path, debug, **kwargs):
     """
     Deep Grep Commandline
 
@@ -162,19 +178,26 @@ def grep(item, path, **kwargs):
     try:
         content = load_path_content(path)
     except Exception as e:  # pragma: no cover.
-        sys.exit(str(f"Error when loading {path}: {e}"))  # pragma: no cover.
+        if debug:  # pragma: no cover.
+            raise  # pragma: no cover.
+        else:  # pragma: no cover.
+            sys.exit(str(f"Error when loading {path}: {e}"))  # pragma: no cover.
 
     try:
         result = DeepSearch(content, item, **kwargs)
     except Exception as e:  # pragma: no cover.
-        sys.exit(str(f"Error when running deep search on {path}: {e}"))  # pragma: no cover.
+        if debug:  # pragma: no cover.
+            raise  # pragma: no cover.
+        else:  # pragma: no cover.
+            sys.exit(str(f"Error when running deep search on {path}: {e}"))  # pragma: no cover.
     pprint(result, indent=2)
 
 
 @cli.command()
 @click.argument('path_inside', required=True, type=str)
 @click.argument('path', type=click.Path(exists=True, resolve_path=True))
-def extract(path_inside, path):
+@click.option('--debug', is_flag=True, show_default=False)
+def extract(path_inside, path, debug):
     """
     Deep Extract Commandline
 
@@ -185,10 +208,16 @@ def extract(path_inside, path):
     try:
         content = load_path_content(path)
     except Exception as e:  # pragma: no cover.
-        sys.exit(str(f"Error when loading {path}: {e}"))  # pragma: no cover.
+        if debug:  # pragma: no cover.
+            raise  # pragma: no cover.
+        else:  # pragma: no cover.
+            sys.exit(str(f"Error when loading {path}: {e}"))  # pragma: no cover.
 
     try:
         result = deep_extract(content, path_inside)
     except Exception as e:  # pragma: no cover.
-        sys.exit(str(f"Error when running deep search on {path}: {e}"))  # pragma: no cover.
+        if debug:  # pragma: no cover.
+            raise  # pragma: no cover.
+        else:  # pragma: no cover.
+            sys.exit(str(f"Error when running deep search on {path}: {e}"))  # pragma: no cover.
     pprint(result, indent=2)

deepdiff/delta.py

@@ -512,7 +512,16 @@ def dump(self, file):
         """
         Dump into file object
         """
-        file.write(self.dumps())
+        # Small optimization: Our internal pickle serializer can just take a file object
+        # and directly write to it. However if a user defined serializer is passed
+        # we want to make it compatible with the expectation that self.serializer(self.diff)
+        # will give the user the serialization and then it can be written to
+        # a file object when using the dump(file) function.
+        param_names_of_serializer = set(self.serializer.__code__.co_varnames)
+        if 'file_obj' in param_names_of_serializer:
+            self.serializer(self.diff, file_obj=file)
+        else:
+            file.write(self.dumps())
 
     def dumps(self):
         """

deepdiff/serialization.py

@@ -39,6 +39,8 @@ class UnsupportedFormatErr(TypeError):
     pass
 
 
+NONE_TYPE = type(None)
+
 CSV_HEADER_MAX_CHUNK_SIZE = 2048  # The chunk needs to be big enough that covers a couple of rows of data.
 
 
@@ -254,10 +256,40 @@ def find_class(self, module, name):
         # Forbid everything else.
         raise ForbiddenModule(FORBIDDEN_MODULE_MSG.format(module_dot_class)) from None
 
+    def persistent_load(self, persistent_id):
+        if persistent_id == "<<NoneType>>":
+            return type(None)
+
+
+class _RestrictedPickler(pickle.Pickler):
+    def persistent_id(self, obj):
+        if obj is NONE_TYPE:  # NOQA
+            return "<<NoneType>>"
+        return None
+
 
-def pickle_dump(obj):
+def pickle_dump(obj, file_obj=None):
+    """
+    **pickle_dump**
+    Dumps the obj into pickled content.
+
+    **Parameters**
+
+    obj : Any python object
+
+    file_obj : (Optional) A file object to dump the contents into
+
+    **Returns**
+
+    If file_obj is passed the return value will be None. It will write the object's pickle contents into the file.
+    However if no file_obj is passed, then it will return the pickle serialization of the obj in the form of bytes.
+    """
+    file_obj_passed = bool(file_obj)
+    file_obj = file_obj or io.BytesIO()
     # We expect at least python 3.5 so protocol 4 is good.
-    return pickle.dumps(obj, protocol=4, fix_imports=False)
+    _RestrictedPickler(file_obj, protocol=4, fix_imports=False).dump(obj)
+    if not file_obj_passed:
+        return file_obj.getvalue()
 
 
 def pickle_load(content, safe_to_import=None):
@@ -406,8 +438,7 @@ def _save_content(content, path, file_type, keep_backup=True):
             content = toml.dump(content, the_file)
     elif file_type == 'pickle':
         with open(path, 'wb') as the_file:
-            content = pickle_dump(content)
-            the_file.write(content)
+            content = pickle_dump(content, file_obj=the_file)
     elif file_type in {'csv', 'tsv'}:
         if clevercsv is None:  # pragma: no cover.
             raise ImportError('CleverCSV needs to be installed.')  # pragma: no cover.

docs/changelog.rst

@@ -5,6 +5,7 @@ Changelog
 
 DeepDiff Changelog
 
+- v5-2-2: Fixed Delta serialization when None type is present.
 - v5-2-0: Removed Murmur3 as the preferred hashing method. Using SHA256 by default now. Added commandline for deepdiff. Added group_by. Added math_epsilon. Improved ignoring of NoneType.
 - v5-0-2: Bug Fix NoneType in ignore type groups https://github.com/seperman/deepdiff/issues/207
 - v5-0-1: Bug fix to not apply format to non numbers.

docs/delta.rst

@@ -331,9 +331,21 @@ At the time of writing this document, this list consists of:
 
 If you want to pass any other argument to safe_to_import, you will need to put the full path to the type as it appears in the sys.modules
 
-For example let's say you have a package call mypackage and has a module called mymodule. If you check the sys.modules, the address to this module must be mypackage.mymodule. In order for Delta to be able to serialize this object, first of all it has to be `picklable <https://docs.python.org/3/library/pickle.html#object.__reduce__>`_. Then you can pass:
+For example let's say you have a package call mypackage and has a module called mymodule. If you check the sys.modules, the address to this module must be mypackage.mymodule. In order for Delta to be able to serialize this object via pickle, first of all it has to be `picklable <https://docs.python.org/3/library/pickle.html#object.__reduce__>`_. 
 
->>> delta = Delta(t1, t2, safe_to_import={'mypackage.mymodule'})
+>>> diff = DeepDiff(t1, t2)
+>>> delta = Delta(diff)
+>>> dump = delta.dumps()
+
+The dump at this point is serialized via Pickle and can be written to disc if needed.
+
+Later when you want to load this dump, by default Delta will block you from importing anything that is NOT in deepdiff.serialization.SAFE_TO_IMPORT . In fact it will show you this error message when trying to load this dump:
+
+    deepdiff.serialization.ForbiddenModule: Module 'builtins.type' is forbidden. You need to explicitly pass it by passing a safe_to_import parameter
+
+In order to let Delta know that this specific module is safe to import, you will need to pass it to Delta during loading of this dump:
+
+>>> delta = Delta(dump, safe_to_import={'mypackage.mymodule'})
 
 .. _delta_verify_symmetry_label:
 

tests/test_delta.py

@@ -1268,3 +1268,11 @@ def test_ignore_order_but_not_report_repetition(self):
             Delta(DeepDiff(t1, t2, ignore_order=True))
 
         assert DELTA_IGNORE_ORDER_NEEDS_REPETITION_REPORT == str(excinfo.value)
+
+    def test_none_in_delta_object(self):
+        t1 = {"a": None}
+        t2 = {"a": 1}
+
+        dump = Delta(DeepDiff(t1, t2)).dumps()
+        delta = Delta(dump)
+        assert t2 == delta + t1