Add optional callback for sync progress

Author: glennmatthews

diffsync/__init__.py

@@ -455,29 +455,45 @@ def str(self, indent: int = 0) -> str:
     # Synchronization between DiffSync instances
     # ------------------------------------------------------------------------------
 
-    def sync_from(self, source: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE):
+    def sync_from(
+        self,
+        source: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[int, int], None]] = None,
+    ):
         """Synchronize data from the given source DiffSync object into the current DiffSync object.
 
         Args:
             source (DiffSync): object to sync data from into this one
             diff_class (class): Diff or subclass thereof to use to calculate the diffs to use for synchronization
             flags (DiffSyncFlags): Flags influencing the behavior of this sync.
+            callback (function): Function with parameters (current, total), to be called at intervals as the
+                calculation of the diff proceeds.
         """
         diff = self.diff_from(source, diff_class=diff_class, flags=flags)
-        syncer = DiffSyncSyncer(diff=diff, src_diffsync=source, dst_diffsync=self, flags=flags)
+        syncer = DiffSyncSyncer(diff=diff, src_diffsync=source, dst_diffsync=self, flags=flags, callback=callback)
         result = syncer.perform_sync()
         if result:
             self.sync_complete(source, diff, flags, syncer.base_logger)
 
-    def sync_to(self, target: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE):
+    def sync_to(
+        self,
+        target: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[int, int], None]] = None,
+    ):
         """Synchronize data from the current DiffSync object into the given target DiffSync object.
 
         Args:
             target (DiffSync): object to sync data into from this one.
             diff_class (class): Diff or subclass thereof to use to calculate the diffs to use for synchronization
             flags (DiffSyncFlags): Flags influencing the behavior of this sync.
+            callback (function): Function with parameters (current, total), to be called at intervals as the
+                calculation of the diff proceeds.
         """
-        target.sync_from(self, diff_class=diff_class, flags=flags)
+        target.sync_from(self, diff_class=diff_class, flags=flags, callback=callback)
 
     def sync_complete(
         self,

diffsync/diff.py

@@ -33,6 +33,13 @@ def __init__(self):
         `self.children[group][unique_id] == DiffElement(...)`
         """
 
+    def __len__(self):
+        """Total number of DiffElements stored herein."""
+        total = 0
+        for child in self.get_children():
+            total += len(child)
+        return total
+
     def complete(self):
         """Method to call when this Diff has been fully populated with data and is "complete".
 
@@ -205,6 +212,13 @@ def __str__(self):
             f"{self.source_name} → {self.dest_name} : {self.get_attrs_diffs()}"
         )
 
+    def __len__(self):
+        """Total number of DiffElements in this one, including itself."""
+        total = 1  # self
+        for child in self.get_children():
+            total += len(child)
+        return total
+
     @property
     def action(self) -> Optional[Text]:
         """Action, if any, that should be taken to remediate the diffs described by this element.

diffsync/helpers.py

@@ -245,19 +245,28 @@ def diff_child_objects(
         return diff_element
 
 
-class DiffSyncSyncer:
+class DiffSyncSyncer:  # pylint: disable=too-many-instance-attributes
     """Helper class implementing data synchronization logic for DiffSync.
 
     Independent from DiffSync and DiffSyncModel as those classes are purely data objects, while this stores some state.
     """
 
-    def __init__(
-        self, diff: Diff, src_diffsync: "DiffSync", dst_diffsync: "DiffSync", flags: DiffSyncFlags,
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        diff: Diff,
+        src_diffsync: "DiffSync",
+        dst_diffsync: "DiffSync",
+        flags: DiffSyncFlags,
+        callback: Optional[Callable[[int, int], None]] = None,
     ):
         """Create a DiffSyncSyncer instance, ready to call `perform_sync()` against."""
         self.diff = diff
         self.dst_diffsync = dst_diffsync
         self.flags = flags
+        self.callback = callback
+
+        self.elements_processed = 0
+        self.total_elements = len(diff)
 
         self.base_logger = structlog.get_logger().new(src=src_diffsync, dst=dst_diffsync, flags=flags)
 
@@ -266,6 +275,13 @@ def __init__(
         self.model_class: Type["DiffSyncModel"]
         self.action: Optional[str] = None
 
+    def incr_elements_processed(self, delta: int = 1):
+        """Increment self.elements_processed, then call self.callback if present."""
+        if delta:
+            self.elements_processed += delta
+            if self.callback:
+                self.callback(self.elements_processed, self.total_elements)
+
     def perform_sync(self) -> bool:
         """Perform data synchronization based on the provided diff.
 
@@ -327,6 +343,8 @@ def sync_diff_element(self, element: DiffElement, parent_model: "DiffSyncModel"
                 return changed
             self.dst_diffsync.remove(model)
 
+        self.incr_elements_processed()
+
         for child in element.get_children():
             changed |= self.sync_diff_element(child, parent_model=model)