Merge pull request #70 from FragmentedPacket/50-get-or-create

Add add_or_update to DiffSync class.

Author: dgarros

diffsync/__init__.py

@@ -329,7 +329,7 @@ def add_child(self, child: "DiffSyncModel"):
         attr_name = self._children[child_type]
         childs = getattr(self, attr_name)
         if child.get_unique_id() in childs:
-            raise ObjectAlreadyExists(f"Already storing a {child_type} with unique_id {child.get_unique_id()}")
+            raise ObjectAlreadyExists(f"Already storing a {child_type} with unique_id {child.get_unique_id()}", child)
         childs.append(child.get_unique_id())
 
     def remove_child(self, child: "DiffSyncModel"):
@@ -648,13 +648,17 @@ def add(self, obj: DiffSyncModel):
             obj (DiffSyncModel): Object to store
 
         Raises:
-            ObjectAlreadyExists: if an object with the same uid is already present
+            ObjectAlreadyExists: if a different object with the same uid is already present.
         """
         modelname = obj.get_type()
         uid = obj.get_unique_id()
 
-        if uid in self._data[modelname]:
-            raise ObjectAlreadyExists(f"Object {uid} already present")
+        existing_obj = self._data[modelname].get(uid)
+        if existing_obj:
+            if existing_obj is not obj:
+                raise ObjectAlreadyExists(f"Object {uid} already present", obj)
+            # Return so we don't have to change anything on the existing object and underlying data
+            return
 
         if not obj.diffsync:
             obj.diffsync = self
@@ -692,6 +696,59 @@ def remove(self, obj: DiffSyncModel, remove_children: bool = False):
                         # Since this is "cleanup" code, log an error and continue, instead of letting the exception raise
                         self._log.error(f"Unable to remove child {child_id} of {modelname} {uid} - not found!")
 
+    def get_or_instantiate(
+        self, model: Type[DiffSyncModel], ids: Dict, attrs: Dict = None
+    ) -> Tuple[DiffSyncModel, bool]:
+        """Attempt to get the object with provided identifiers or instantiate it with provided identifiers and attrs.
+
+        Args:
+            model (DiffSyncModel): The DiffSyncModel to get or create.
+            ids (Mapping): Identifiers for the DiffSyncModel to get or create with.
+            attrs (Mapping, optional): Attributes when creating an object if it doesn't exist. Defaults to None.
+
+        Returns:
+            Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not.
+        """
+        created = False
+        try:
+            obj = self.get(model, ids)
+        except ObjectNotFound:
+            if not attrs:
+                attrs = {}
+            obj = model(**ids, **attrs)
+            # Add the object to diffsync adapter
+            self.add(obj)
+            created = True
+
+        return obj, created
+
+    def update_or_instantiate(self, model: Type[DiffSyncModel], ids: Dict, attrs: Dict) -> Tuple[DiffSyncModel, bool]:
+        """Attempt to update an existing object with provided ids/attrs or instantiate it with provided identifiers and attrs.
+
+        Args:
+            model (DiffSyncModel): The DiffSyncModel to get or create.
+            ids (Dict): Identifiers for the DiffSyncModel to get or create with.
+            attrs (Dict): Attributes when creating/updating an object if it doesn't exist. Pass in empty dict, if no specific attrs.
+
+        Returns:
+            Tuple[DiffSyncModel, bool]: Provides the existing or new object and whether it was created or not.
+        """
+        created = False
+        try:
+            obj = self.get(model, ids)
+        except ObjectNotFound:
+            obj = model(**ids, **attrs)
+            # Add the object to diffsync adapter
+            self.add(obj)
+            created = True
+
+        # Update existing obj with attrs
+        for attr, value in attrs.items():
+            if getattr(obj, attr) != value:
+                setattr(obj, attr, value)
+
+        return obj, created
+
 
 # DiffSyncModel references DiffSync and DiffSync references DiffSyncModel. Break the typing loop:
 DiffSyncModel.update_forward_refs()

diffsync/diff.py

@@ -55,7 +55,7 @@ def add(self, element: "DiffElement"):
         """
         # Note that element.name is usually a DiffSyncModel.shortname() -- i.e., NOT guaranteed globally unique!!
         if element.name in self.children[element.type]:
-            raise ObjectAlreadyExists(f"Already storing a {element.type} named {element.name}")
+            raise ObjectAlreadyExists(f"Already storing a {element.type} named {element.name}", element)
 
         self.children[element.type][element.name] = element
 

diffsync/exceptions.py

@@ -39,6 +39,11 @@ class ObjectStoreException(Exception):
 class ObjectAlreadyExists(ObjectStoreException):
     """Exception raised when trying to store a DiffSyncModel or DiffElement that is already being stored."""
 
+    def __init__(self, message, existing_object, *args, **kwargs):
+        """Add existing_object to the exception to provide user with existing object."""
+        self.existing_object = existing_object
+        super().__init__(message, existing_object, *args, **kwargs)
+
 
 class ObjectNotFound(ObjectStoreException):
     """Exception raised when trying to access a DiffSyncModel that isn't in storage."""

diffsync/helpers.py

@@ -85,7 +85,8 @@ def calculate_diffs(self) -> Diff:
 
         for obj_type in intersection(self.dst_diffsync.top_level, self.src_diffsync.top_level):
             diff_elements = self.diff_object_list(
-                src=self.src_diffsync.get_all(obj_type), dst=self.dst_diffsync.get_all(obj_type),
+                src=self.src_diffsync.get_all(obj_type),
+                dst=self.dst_diffsync.get_all(obj_type),
             )
 
             for diff_element in diff_elements:
@@ -220,7 +221,10 @@ def diff_object_pair(
         return diff_element
 
     def diff_child_objects(
-        self, diff_element: DiffElement, src_obj: Optional["DiffSyncModel"], dst_obj: Optional["DiffSyncModel"],
+        self,
+        diff_element: DiffElement,
+        src_obj: Optional["DiffSyncModel"],
+        dst_obj: Optional["DiffSyncModel"],
     ):
         """For all children of the given DiffSyncModel pair, diff recursively, adding diffs to the given diff_element.
 

docs/source/examples/index.rst

@@ -6,4 +6,5 @@ For each example, the complete source code is `available in Github <https://gith
 
 .. mdinclude:: ../../../examples/01-multiple-data-sources/README.md
 .. mdinclude:: ../../../examples/02-callback-function/README.md
-.. mdinclude:: ../../../examples/03-remote-system/README.md
+.. mdinclude:: ../../../examples/03-remote-system/README.md
+.. mdinclude:: ../../../examples/04-get-update-instantiate/README.md

examples/04-get-update-instantiate/README.md

@@ -0,0 +1,72 @@
+# Example 4 - Using get or update helpers
+
+This example aims to expand on [Example 1](https://github.com/networktocode/diffsync/tree/main/examples/01-multiple-data-sources/README.md) that will take advantage of two new helper methods on the `DiffSync` class; `get_or_instantiate` and `update_or_instantiate`.
+
+Both methods act similar to Django's `get_or_create` function to return the object and then a boolean to identify whether the object was created or not. Let's dive into each of them.
+
+## get_or_instantiate
+
+The following arguments are supported: model (`DiffSyncModel`), ids (dictionary), and attrs (dictionary). The `model` and `ids` are used to find an existing object. If the object does not currently exist within the `DiffSync` adapter, it will then use `model`, `ids`, and `attrs` to add the object.
+
+It will then return a tuple that can be unpacked.
+
+```python
+obj, created = self.get_or_instantiate(Interface, {"device_name": "test100", "name": "eth0"}, {"description": "Test description"})
+```
+
+If the object already exists, `created` will be `False` or else it will return `True` if the object had to be created.
+
+## update_or_instantiate
+
+This helper is similar to `get_or_instantiate`, but it will update an existing object or add a new instance with the provided `ids` and `attrs`. The method does accept the same arguments, but requires `attrs`, whereas `get_or_instantiate` does not.
+
+```python
+obj, created = self.update_or_instantiate(Interface, {"device_name": "test100", "name": "eth0"}, {"description": "Test description"})
+```
+
+## Example Walkthrough
+
+We can take a look at the data we will be loading into each backend to understand why these helper methods are valuable.
+
+### Example Data
+
+```python
+BACKEND_DATA_A = [
+    {
+        "name": "nyc-spine1",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "nyc",
+    },
+    {
+        "name": "nyc-spine2",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "nyc",
+    },
+]
+```
+
+## Example Load
+
+```python
+    def load(self):
+        """Initialize the BackendA Object by loading some site, device and interfaces from DATA."""
+        for device_data in BACKEND_DATA_A:
+            device, instantiated = self.get_or_instantiate(
+                self.device, {"name": device_data["name"]}, {"role": device_data["role"]}
+            )
+
+            site, instantiated = self.get_or_instantiate(self.site, {"name": device_data["site"]})
+            if instantiated:
+                device.add_child(site)
+
+            for intf_name, desc in device_data["interfaces"].items():
+                intf, instantiated = self.update_or_instantiate(
+                    self.interface, {"name": intf_name, "device_name": device_data["name"]}, {"description": desc}
+                )
+                if instantiated:
+                    device.add_child(intf)
+```
+
+The new methods are helpful due to having devices that are part of the same site. As we iterate over the data and load it into the `DiffSync` adapter, we would have to account for `ObjectAlreadyExists` exceptions when we go to add each duplicate site we encounter within the data or possibly several other models depending how complex the synchronization of data is between backends.

examples/04-get-update-instantiate/backends.py

@@ -0,0 +1,134 @@
+"""Example of a DiffSync adapter implementation using new helper methods.
+
+Copyright (c) 2021 Network To Code, LLC <[email protected]>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from models import Site, Device, Interface
+from diffsync import DiffSync
+
+BACKEND_DATA_A = [
+    {
+        "name": "nyc-spine1",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "nyc",
+    },
+    {
+        "name": "nyc-spine2",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "nyc",
+    },
+    {
+        "name": "sfo-spine1",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "sfo",
+    },
+    {
+        "name": "sfo-spine2",
+        "role": "spine",
+        "interfaces": {"eth0": "TBD", "eth1": "ddd", "eth2": "Interface 2"},
+        "site": "sfo",
+    },
+]
+BACKEND_DATA_B = [
+    {
+        "name": "atl-spine1",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "atl",
+    },
+    {
+        "name": "atl-spine2",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "atl",
+    },
+    {
+        "name": "nyc-spine1",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0/0", "eth1": "Interface 1"},
+        "site": "nyc",
+    },
+    {
+        "name": "nyc-spine2",
+        "role": "spine",
+        "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"},
+        "site": "nyc",
+    },
+    {"name": "sfo-spine1", "role": "leaf", "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"}, "site": "sfo"},
+    {"name": "sfo-spine2", "role": "spine", "interfaces": {"eth0": "TBD", "eth1": "ddd"}, "site": "sfo"},
+]
+
+
+class BackendA(DiffSync):
+    """Example of a DiffSync adapter implementation."""
+
+    site = Site
+    device = Device
+    interface = Interface
+
+    top_level = ["device"]
+
+    type = "Backend A"
+
+    def load(self):
+        """Initialize the BackendA Object by loading some site, device and interfaces from DATA."""
+        for device_data in BACKEND_DATA_A:
+            device, instantiated = self.get_or_instantiate(
+                self.device, {"name": device_data["name"]}, {"role": device_data["role"]}
+            )
+
+            site, instantiated = self.get_or_instantiate(self.site, {"name": device_data["site"]})
+            if instantiated:
+                device.add_child(site)
+
+            for intf_name, desc in device_data["interfaces"].items():
+                intf, instantiated = self.update_or_instantiate(
+                    self.interface, {"name": intf_name, "device_name": device_data["name"]}, {"description": desc}
+                )
+                if instantiated:
+                    device.add_child(intf)
+
+
+class BackendB(DiffSync):
+    """Example of a DiffSync adapter implementation."""
+
+    site = Site
+    device = Device
+    interface = Interface
+
+    top_level = ["device"]
+
+    type = "Backend B"
+
+    def load(self):
+        """Initialize the BackendB Object by loading some site, device and interfaces from DATA."""
+        for device_data in BACKEND_DATA_B:
+            device, instantiated = self.get_or_instantiate(
+                self.device, {"name": device_data["name"]}, {"role": device_data["role"]}
+            )
+
+            site, instantiated = self.get_or_instantiate(self.site, {"name": device_data["site"]})
+            if instantiated:
+                device.add_child(site)
+
+            for intf_name, desc in device_data["interfaces"].items():
+                intf, instantiated = self.get_or_instantiate(
+                    self.interface, {"name": intf_name, "device_name": device_data["name"]}, {"description": desc}
+                )
+                if instantiated:
+                    device.add_child(intf)