adding BaseOperatorPlus

Author: seperman

README.md

@@ -25,6 +25,7 @@ Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.
 
 DeepDiff 8-4-0
 
+- Adding BaseOperatorPlus base class for custom operators
 - default_timezone can be passed now to set your default timezone to something other than UTC.
 - New summarization algorithm that produces valid json
 - Better type hint support

deepdiff/deephash.py

@@ -2,7 +2,7 @@
 import pytz
 import logging
 import datetime
-from typing import Union
+from typing import Union, Optional, Any, List
 from collections.abc import Iterable, MutableMapping
 from collections import defaultdict
 from hashlib import sha1, sha256
@@ -141,31 +141,32 @@ class DeepHash(Base):
     def __init__(self,
                  obj,
                  *,
-                 hashes=None,
-                 exclude_types=None,
+                 apply_hash=True,
+                 custom_operators: Optional[List[Any]] =None,
+                 default_timezone:Union[datetime.timezone, datetime.timezone, pytz.tzinfo.BaseTzInfo]=datetime.timezone.utc,
+                 encodings=None,
+                 exclude_obj_callback=None,
                  exclude_paths=None,
-                 include_paths=None,
                  exclude_regex_paths=None,
+                 exclude_types=None,
                  hasher=None,
+                 hashes=None,
+                 ignore_encoding_errors=False,
+                 ignore_iterable_order=True,
+                 ignore_numeric_type_changes=False,
+                 ignore_private_variables=True,
                  ignore_repetition=True,
-                 significant_digits=None,
-                 truncate_datetime=None,
-                 number_format_notation="f",
-                 apply_hash=True,
-                 ignore_type_in_groups=None,
+                 ignore_string_case=False,
                  ignore_string_type_changes=False,
-                 ignore_numeric_type_changes=False,
+                 ignore_type_in_groups=None,
                  ignore_type_subclasses=False,
-                 ignore_string_case=False,
-                 use_enum_value=False,
-                 exclude_obj_callback=None,
+                 include_paths=None,
+                 number_format_notation="f",
                  number_to_string_func=None,
-                 ignore_private_variables=True,
                  parent="root",
-                 encodings=None,
-                 ignore_encoding_errors=False,
-                 ignore_iterable_order=True,
-                 default_timezone:Union[datetime.timezone, datetime.timezone, pytz.tzinfo.BaseTzInfo]=datetime.timezone.utc,
+                 significant_digits=None,
+                 truncate_datetime=None,
+                 use_enum_value=False,
                  **kwargs):
         if kwargs:
             raise ValueError(
@@ -192,7 +193,6 @@ def __init__(self,
         self.hashes[UNPROCESSED_KEY] = []
         self.use_enum_value = use_enum_value
         self.default_timezone = default_timezone
-
         self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes)
         self.truncate_datetime = get_truncate_datetime(truncate_datetime)
         self.number_format_notation = number_format_notation
@@ -216,6 +216,7 @@ def __init__(self,
         self.encodings = encodings
         self.ignore_encoding_errors = ignore_encoding_errors
         self.ignore_iterable_order = ignore_iterable_order
+        self.custom_operators = custom_operators
 
         self._hash(obj, parent=parent, parents_ids=frozenset({get_id(obj)}))
 
@@ -505,6 +506,13 @@ def _prep_tuple(self, obj, parent, parents_ids):
     def _hash(self, obj, parent, parents_ids=EMPTY_FROZENSET):
         """The main hash method"""
         counts = 1
+        if self.custom_operators is not None:
+            for operator in self.custom_operators:
+                func = getattr(operator, 'normalize_value_for_hashing', None)
+                if func is None:
+                    raise NotImplementedError(f"{operator.__class__.__name__} needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.".format(operator))
+                else:
+                    obj = func(parent, obj)
 
         if isinstance(obj, booleanTypes):
             obj = self._prep_bool(obj)

deepdiff/diff.py

@@ -112,6 +112,7 @@ def _report_progress(_stats, progress_logger, duration):
     'encodings',
     'ignore_encoding_errors',
     'default_timezone',
+    'custom_operators',
 )
 
 
@@ -130,6 +131,7 @@ def __init__(self,
                  custom_operators: Optional[List[Any]] =None,
                  cutoff_distance_for_pairs: float=CUTOFF_DISTANCE_FOR_PAIRS_DEFAULT,
                  cutoff_intersection_for_pairs: float=CUTOFF_INTERSECTION_FOR_PAIRS_DEFAULT,
+                 default_timezone:Union[datetime.timezone, datetime.timezone, pytz.tzinfo.BaseTzInfo]=datetime.timezone.utc,
                  encodings: Optional[List[str]]=None,
                  exclude_obj_callback: Optional[Callable]=None,
                  exclude_obj_callback_strict: Optional[Callable]=None,
@@ -156,6 +158,8 @@ def __init__(self,
                  include_paths: Union[str, List[str], None]=None,
                  iterable_compare_func: Optional[Callable]=None,
                  log_frequency_in_sec: int=0,
+                 log_scale_similarity_threshold: float=0.1,
+                 log_stacktrace: bool=False,
                  math_epsilon: Optional[float]=None,
                  max_diffs: Optional[int]=None,
                  max_passes: int=10000000,
@@ -164,15 +168,13 @@ def __init__(self,
                  progress_logger: Callable=logger.info,
                  report_repetition: bool=False,
                  significant_digits: Optional[int]=None,
-                 use_log_scale: bool=False,
-                 log_scale_similarity_threshold: float=0.1,
                  threshold_to_diff_deeper: float = 0.33,
                  truncate_datetime: Optional[str]=None,
                  use_enum_value: bool=False,
+                 use_log_scale: bool=False,
                  verbose_level: int=1,
                  view: str=TEXT_VIEW,
                  zip_ordered_iterables: bool=False,
-                 default_timezone:Union[datetime.timezone, datetime.timezone, pytz.tzinfo.BaseTzInfo]=datetime.timezone.utc,
                  _parameters=None,
                  _shared_parameters=None,
                  **kwargs):
@@ -186,7 +188,7 @@ def __init__(self,
                 "ignore_private_variables, ignore_nan_inequality, number_to_string_func, verbose_level, "
                 "view, hasher, hashes, max_passes, max_diffs, zip_ordered_iterables, "
                 "cutoff_distance_for_pairs, cutoff_intersection_for_pairs, log_frequency_in_sec, cache_size, "
-                "cache_tuning_sample_size, get_deep_distance, group_by, group_by_sort_key, cache_purge_level, "
+                "cache_tuning_sample_size, get_deep_distance, group_by, group_by_sort_key, cache_purge_level, log_stacktrace,"
                 "math_epsilon, iterable_compare_func, use_enum_value, _original_type, threshold_to_diff_deeper, default_timezone "
                 "ignore_order_func, custom_operators, encodings, ignore_encoding_errors, use_log_scale, log_scale_similarity_threshold "
                 "_parameters and _shared_parameters.") % ', '.join(kwargs.keys()))
@@ -209,6 +211,7 @@ def __init__(self,
             self.log_scale_similarity_threshold = log_scale_similarity_threshold
             self.use_log_scale = use_log_scale
             self.default_timezone = default_timezone
+            self.log_stacktrace = log_stacktrace
             self.threshold_to_diff_deeper = threshold_to_diff_deeper
             self.ignore_string_type_changes = ignore_string_type_changes
             self.ignore_type_in_groups = self.get_ignore_types_in_groups(
@@ -276,6 +279,10 @@ def _group_by_sort_key(x):
             self.cache_size = cache_size
             _parameters = self.__dict__.copy()
             _parameters['group_by'] = None  # overwriting since these parameters will be passed on to other passes.
+            if log_stacktrace:
+                self.log_err = logger.exception
+            else:
+                self.log_err = logger.error
 
         # Non-Root
         if _shared_parameters:
@@ -736,7 +743,7 @@ def _compare_in_order(
         self, level,
         t1_from_index=None, t1_to_index=None,
         t2_from_index=None, t2_to_index=None
-    ):
+    ) -> List[Tuple[Tuple[int, int], Tuple[Any, Any]]]:
         """
         Default compare if `iterable_compare_func` is not provided.
         This will compare in sequence order.
@@ -756,7 +763,7 @@ def _get_matching_pairs(
         self, level,
         t1_from_index=None, t1_to_index=None,
         t2_from_index=None, t2_to_index=None
-    ):
+    ) -> List[Tuple[Tuple[int, int], Tuple[Any, Any]]]:
         """
         Given a level get matching pairs. This returns list of two tuples in the form:
         [
@@ -1088,44 +1095,48 @@ def _create_hashtable(self, level, t):
                 # It only includes the ones needed when comparing iterables.
                 # The self.hashes dictionary gets shared between different runs of DeepHash
                 # So that any object that is already calculated to have a hash is not re-calculated.
-                deep_hash = DeepHash(item,
-                                     hashes=self.hashes,
-                                     parent=parent,
-                                     apply_hash=True,
-                                     **self.deephash_parameters,
-                                     )
+                deep_hash = DeepHash(
+                    item,
+                    hashes=self.hashes,
+                    parent=parent,
+                    apply_hash=True,
+                    **self.deephash_parameters,
+                 )
             except UnicodeDecodeError as err:
                 err.reason = f"Can not produce a hash for {level.path()}: {err.reason}"
                 raise
-            except Exception as e:  # pragma: no cover
-                logger.error("Can not produce a hash for %s."
-                             "Not counting this object.\n %s" %
-                             (level.path(), e))
+            except NotImplementedError:
+                raise
+            # except Exception as e:  # pragma: no cover
+            #     logger.error("Can not produce a hash for %s."
+            #                  "Not counting this object.\n %s" %
+            #                  (level.path(), e))
             else:
                 try:
                     item_hash = deep_hash[item]
                 except KeyError:
                     pass
                 else:
                     if item_hash is unprocessed:  # pragma: no cover
-                        logger.warning("Item %s was not processed while hashing "
+                        self.log_err("Item %s was not processed while hashing "
                                        "thus not counting this object." %
                                        level.path())
                     else:
                         self._add_hash(hashes=local_hashes, item_hash=item_hash, item=item, i=i)
 
         # Also we hash the iterables themselves too so that we can later create cache keys from those hashes.
-        try:
-            DeepHash(
-                obj,
-                hashes=self.hashes,
-                parent=level.path(),
-                apply_hash=True,
-                **self.deephash_parameters,
-            )
-        except Exception as e:  # pragma: no cover
-            logger.error("Can not produce a hash for iterable %s. %s" %
-                         (level.path(), e))
+        DeepHash(
+            obj,
+            hashes=self.hashes,
+            parent=level.path(),
+            apply_hash=True,
+            **self.deephash_parameters,
+        )
+        # try:
+        # except Exception as e:  # pragma: no cover
+        #     import pytest; pytest.set_trace()
+        #     self.log_err("Can not produce a hash for iterable %s. %s" %
+        #                  (level.path(), e))
         return local_hashes
 
     @staticmethod

deepdiff/operator.py

@@ -1,10 +1,41 @@
 import re
+from typing import Any, Optional, List
+from abc import ABCMeta, abstractmethod
 from deepdiff.helper import convert_item_or_items_into_compiled_regexes_else_none
 
 
+
+class BaseOperatorPlus(metaclass=ABCMeta):
+
+    @abstractmethod
+    def match(self, level) -> bool:
+        """
+        Given a level which includes t1 and t2 in the tree view, is this operator a good match to compare t1 and t2?
+        If yes, we will run the give_up_diffing to compare t1 and t2 for this level.
+        """
+        pass
+
+    @abstractmethod
+    def give_up_diffing(self, level, diff_instance: float) -> bool:
+        """
+        Given a level which includes t1 and t2 in the tree view, and the "distance" between l1 and l2.
+        do we consider t1 and t2 to be equal or not. The distance is a number between zero to one and is calculated by DeepDiff to measure how similar objects are.
+        """
+
+    @abstractmethod
+    def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
+        """
+        You can use this function to normalize values for ignore_order=True
+
+        For example, you may want to turn all the words to be lowercase. Then you return obj.lower()
+        """
+        pass
+
+
+
 class BaseOperator:
 
-    def __init__(self, regex_paths=None, types=None):
+    def __init__(self, regex_paths:Optional[List[str]]=None, types:Optional[List[type]]=None):
         if regex_paths:
             self.regex_paths = convert_item_or_items_into_compiled_regexes_else_none(regex_paths)
         else: