Add optional callback for diff progress

Author: glennmatthews

README.md

@@ -22,9 +22,11 @@ A.sync_from(B)
 A.sync_to(B)
 ```
 
+You may wish to peruse the [`diffsync` GitHub topic](https://github.com/topics/diffsync) for examples of projects using this library.
+
 # Getting started
 
-To be able to properly compare different datasets, DiffSync relies on a shared datamodel that both systems must use.
+To be able to properly compare different datasets, DiffSync relies on a shared data model that both systems must use.
 Specifically, each system or dataset must provide a `DiffSync` "adapter" subclass, which in turn represents its dataset as instances of one or more `DiffSyncModel` data model classes.
 
 When comparing two systems, DiffSync detects the intersection between the two systems (which data models they have in common, and which attributes are shared between each pair of data models) and uses this intersection to compare and/or synchronize the data.
@@ -39,9 +41,9 @@ Each `DiffSyncModel` subclass supports the following class-level attributes:
 - `_attributes` - List of non-identifier instance field names for this object; used to identify the fields in common between data models for different systems (Optional)
 - `_children` - Dict of `{<model_name>: <field_name>}` indicating which fields store references to child data model instances. (Optional)
 
-> DiffSyncModel instances must be uniquely identified by their unique id, composed of all fields defined in `_identifiers`. The unique id must be globally meaningful (such as an unique instance name or slug), as it is used to identify object correspondence between differing systems or data sets. It **must not** be a value that is only locally meaningful, such as a database primary key integer value.
+> DiffSyncModel instances must be uniquely identified by their unique ID (or, in database terminology, [natural key](https://en.wikipedia.org/wiki/Natural_key)), which is composed of the union of all fields defined in `_identifiers`. The unique ID must be globally meaningful (such as an unique instance name or slug), as it is used to identify object correspondence between differing systems or data sets. It **must not** be a value that is only locally meaningful to a specific data set, such as a database primary key value.
 
-> Only fields listed in `_identifiers`, `_attributes`, or `_children` will be potentially included in comparison and synchronization between systems or data sets. Any other fields will be ignored; this allows for a model to additionally contain fields that are only locally relevant (such as database primary key values) and therefore are irrelevant to comparisons.
+> Only fields listed in `_identifiers`, `_attributes`, or `_children` will be potentially included in comparison and synchronization between systems or data sets. Any other fields will be ignored; this allows for a model to additionally contain fields that are only locally relevant (such as database primary key values) and therefore are irrelevant to comparison and synchronization.
 
 ```python
 from typing import List, Optional

diffsync/__init__.py

@@ -1,6 +1,6 @@
 """DiffSync front-end classes and logic.
 
-Copyright (c) 2020 Network To Code, LLC <[email protected]>
+Copyright (c) 2020-2021 Network To Code, LLC <[email protected]>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 """
 from collections import defaultdict
 from inspect import isclass
-from typing import ClassVar, Dict, List, Mapping, MutableMapping, Optional, Text, Tuple, Type, Union
+from typing import Callable, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Text, Tuple, Type, Union
 
 from pydantic import BaseModel, PrivateAttr
 import structlog  # type: ignore
@@ -359,7 +359,7 @@ def remove_child(self, child: "DiffSyncModel"):
 class DiffSync:
     """Class for storing a group of DiffSyncModel instances and diffing/synchronizing to another DiffSync instance."""
 
-    # Add mapping of names to specific model classes here:
+    # In any subclass, you would add mapping of names to specific model classes here:
     # modelname1 = MyModelClass1
     # modelname2 = MyModelClass2
 
@@ -418,6 +418,10 @@ def __str__(self):
     def __repr__(self):
         return f"<{str(self)}>"
 
+    def __len__(self):
+        """Total number of elements stored in self._data."""
+        return sum(len(entries) for entries in self._data.values())
+
     def load(self):
         """Load all desired data from whatever backend data source into this instance."""
         # No-op in this generic class
@@ -502,29 +506,43 @@ def sync_complete(
     # ------------------------------------------------------------------------------
 
     def diff_from(
-        self, source: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE
+        self,
+        source: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[int, int], None]] = None,
     ) -> Diff:
         """Generate a Diff describing the difference from the other DiffSync to this one.
 
         Args:
             source (DiffSync): Object to diff against.
             diff_class (class): Diff or subclass thereof to use for diff calculation and storage.
             flags (DiffSyncFlags): Flags influencing the behavior of this diff operation.
+            callback (function): Function with parameters (current, total), to be called at intervals as the
+                calculation of the diff proceeds.
         """
-        differ = DiffSyncDiffer(src_diffsync=source, dst_diffsync=self, flags=flags, diff_class=diff_class)
+        differ = DiffSyncDiffer(
+            src_diffsync=source, dst_diffsync=self, flags=flags, diff_class=diff_class, callback=callback
+        )
         return differ.calculate_diffs()
 
     def diff_to(
-        self, target: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE
+        self,
+        target: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[int, int], None]] = None,
     ) -> Diff:
         """Generate a Diff describing the difference from this DiffSync to another one.
 
         Args:
             target (DiffSync): Object to diff against.
             diff_class (class): Diff or subclass thereof to use for diff calculation and storage.
             flags (DiffSyncFlags): Flags influencing the behavior of this diff operation.
+            callback (function): Function with parameters (current, total), to be called at intervals as the
+                calculation of the diff proceeds.
         """
-        return target.diff_from(self, diff_class=diff_class, flags=flags)
+        return target.diff_from(self, diff_class=diff_class, flags=flags, callback=callback)
 
     # ------------------------------------------------------------------------------
     # Object Storage Management
@@ -567,21 +585,21 @@ def get(
             raise ObjectNotFound(f"{modelname} {uid} not present in {self.name}")
         return self._data[modelname][uid]
 
-    def get_all(self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]):
+    def get_all(self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]) -> List[DiffSyncModel]:
         """Get all objects of a given type.
 
         Args:
             obj: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve
 
         Returns:
-            ValuesList[DiffSyncModel]: List of Object
+            List[DiffSyncModel]: List of Object
         """
         if isinstance(obj, str):
             modelname = obj
         else:
             modelname = obj.get_type()
 
-        return self._data[modelname].values()
+        return list(self._data[modelname].values())
 
     def get_by_uids(
         self, uids: List[Text], obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]

diffsync/helpers.py

@@ -1,6 +1,6 @@
 """DiffSync helper classes for calculating and performing diff and sync operations.
 
-Copyright (c) 2020 Network To Code, LLC <[email protected]>
+Copyright (c) 2020-2021 Network To Code, LLC <[email protected]>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
 limitations under the License.
 """
 from collections.abc import Iterable as ABCIterable, Mapping as ABCMapping
-from typing import Iterable, List, Mapping, Optional, Tuple, Type, TYPE_CHECKING
+from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Type, TYPE_CHECKING
 
 import structlog  # type: ignore
 
@@ -29,14 +29,19 @@
     from . import DiffSync, DiffSyncModel  # pylint: disable=cyclic-import
 
 
-class DiffSyncDiffer:
+class DiffSyncDiffer:  # pylint: disable=too-many-instance-attributes
     """Helper class implementing diff calculation logic for DiffSync.
 
     Independent from Diff and DiffElement as those classes are purely data objects, while this stores some state.
     """
 
-    def __init__(
-        self, src_diffsync: "DiffSync", dst_diffsync: "DiffSync", flags: DiffSyncFlags, diff_class: Type[Diff] = Diff
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        src_diffsync: "DiffSync",
+        dst_diffsync: "DiffSync",
+        flags: DiffSyncFlags,
+        diff_class: Type[Diff] = Diff,
+        callback: Optional[Callable[[int, int], None]] = None,
     ):
         """Create a DiffSyncDiffer for calculating diffs between the provided DiffSync instances."""
         self.src_diffsync = src_diffsync
@@ -45,13 +50,27 @@ def __init__(
 
         self.logger = structlog.get_logger().new(src=src_diffsync, dst=dst_diffsync, flags=flags)
         self.diff_class = diff_class
+        self.callback = callback
         self.diff: Optional[Diff] = None
 
+        self.models_processed = 0
+        self.total_models = len(src_diffsync) + len(dst_diffsync)
+        self.logger.debug(f"Diff calculation between these two datasets will involve {self.total_models} models")
+
+    def incr_models_processed(self, delta: int = 1):
+        """Increment self.models_processed, then call self.callback if present."""
+        if delta:
+            self.models_processed += delta
+            if self.callback:
+                self.callback(self.models_processed, self.total_models)
+
     def calculate_diffs(self) -> Diff:
         """Calculate diffs between the src and dst DiffSync objects and return the resulting Diff."""
         if self.diff is not None:
             return self.diff
 
+        self.models_processed = 0
+
         self.logger.info("Beginning diff calculation")
         self.diff = self.diff_class()
         for obj_type in intersection(self.dst_diffsync.top_level, self.src_diffsync.top_level):
@@ -66,7 +85,7 @@ def calculate_diffs(self) -> Diff:
         self.diff.complete()
         return self.diff
 
-    def diff_object_list(self, src: Iterable["DiffSyncModel"], dst: Iterable["DiffSyncModel"]) -> List[DiffElement]:
+    def diff_object_list(self, src: List["DiffSyncModel"], dst: List["DiffSyncModel"]) -> List[DiffElement]:
         """Calculate diffs between two lists of like objects.
 
         Helper method to `calculate_diffs`, usually doesn't need to be called directly.
@@ -90,6 +109,9 @@ def diff_object_list(self, src: Iterable["DiffSyncModel"], dst: Iterable["DiffSy
             # In the future we might support set, etc...
             raise TypeError(f"Type combination {type(src)}/{type(dst)} is not supported... for now")
 
+        # Any non-intersection between src and dst can be counted as "processed" and done.
+        self.incr_models_processed(max(len(src) - len(combined_dict), 0) + max(len(dst) - len(combined_dict), 0))
+
         self.validate_objects_for_diff(combined_dict.values())
 
         for uid in combined_dict:
@@ -168,10 +190,15 @@ def diff_object_pair(
             diff_class=self.diff_class,
         )
 
+        delta = 0
         if src_obj:
             diff_element.add_attrs(source=src_obj.get_attrs(), dest=None)
+            delta += 1
         if dst_obj:
             diff_element.add_attrs(source=None, dest=dst_obj.get_attrs())
+            delta += 1
+
+        self.incr_models_processed(delta)
 
         # Recursively diff the children of src_obj and dst_obj and attach the resulting diffs to the diff_element
         self.diff_child_objects(diff_element, src_obj, dst_obj)