demo onboarding fixes

datadog_sync/model/dashboards.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING, Optional, List, Dict, Tuple, cast
 
 from datadog_sync.utils.base_resource import BaseResource, ResourceConfig
+from datadog_sync.utils.resource_utils import CustomClientHTTPError, SkipResource
 
 if TYPE_CHECKING:
     from datadog_sync.utils.custom_client import CustomClient
@@ -44,9 +45,14 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di
         source_client = self.config.source_client
         import_id = _id or resource["id"]
 
-        resource = await source_client.get(self.resource_config.base_path + f"/{import_id}")
-        resource = cast(dict, resource)
+        try:
+            resource = await source_client.get(self.resource_config.base_path + f"/{import_id}")
+        except CustomClientHTTPError as err:
+            if err.status_code == 403:
+                raise SkipResource(import_id, self.resource_type, "No access to restricted dashboard")
+            raise err
 
+        resource = cast(dict, resource)
         return import_id, resource
 
     async def pre_resource_action_hook(self, _id, resource: Dict) -> None:

datadog_sync/model/host_tags.py

@@ -58,6 +58,3 @@ async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
     async def delete_resource(self, _id: str) -> None:
         destination_client = self.config.destination_client
         await destination_client.delete(self.resource_config.base_path + f"/{_id}")
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/logs_archives.py

@@ -62,6 +62,3 @@ async def delete_resource(self, _id: str) -> None:
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/logs_custom_pipelines.py

@@ -64,6 +64,3 @@ async def delete_resource(self, _id: str) -> None:
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/logs_indexes.py

@@ -92,9 +92,6 @@ async def delete_resource(self, _id: str) -> None:
             index_order["index_names"].append(index_name)
             await self.config.destination_client.put(self.logs_indexes_order_url, index_order)
 
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass
-
     async def get_destination_logs_indexes(self) -> Dict[str, Dict]:
         destination_global_variable_obj = {}
         destination_client = self.config.destination_client

datadog_sync/model/logs_metrics.py

@@ -61,6 +61,3 @@ async def delete_resource(self, _id: str) -> None:
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/logs_pipelines.py

@@ -151,9 +151,6 @@ async def delete_resource(self, _id: str) -> None:
                 self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
             )
 
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass
-
     async def get_destination_integration_pipelines(self):
         destination_integration_pipelines_obj = {}
         destination_client = self.config.destination_client

datadog_sync/model/metric_percentiles.py

@@ -52,6 +52,3 @@ async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
 
     async def delete_resource(self, _id: str) -> None:
         pass
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/metric_tag_configurations.py

@@ -72,9 +72,6 @@ async def delete_resource(self, _id: str) -> None:
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}/tags"
         )
 
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass
-
     async def get_destination_metric_tag_configuration(self) -> Dict[str, Dict]:
         destination_metric_tag_configurations = {}
         destination_client = self.config.destination_client

datadog_sync/model/metrics_metadata.py

@@ -44,16 +44,19 @@ async def pre_apply_hook(self) -> None:
         pass
 
     async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
+        if resource.get("type", None) == "distribution":
+            metric_name = _id or resource["id"]
+            raise SkipResource(metric_name, self.resource_type, "Metric metadata not supported for distributions.")
         return await self.update_resource(_id, resource)
 
     async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
+        if resource.get("type", None) == "distribution":
+            metric_name = _id or resource["id"]
+            raise SkipResource(metric_name, self.resource_type, "Metric metadata not supported for distributions.")
         destination_client = self.config.destination_client
         resp = await destination_client.put(self.resource_config.base_path + f"/{_id}", resource)
 
         return _id, resp
 
     async def delete_resource(self, _id: str) -> None:
         pass
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/monitors.py

@@ -23,6 +23,7 @@ class Monitors(BaseResource):
             "roles": ["restricted_roles"],
             "service_level_objectives": ["query"],
             "restriction_policies": ["restriction_policy"],
+            "rum_applications": ["query"],
         },
         base_path="/api/v1/monitor",
         excluded_attributes=[
@@ -102,8 +103,13 @@ def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optiona
         monitors = self.config.state.destination[resource_to_connect]
         synthetics_tests = self.config.state.destination["synthetics_tests"]
         slos = self.config.state.destination["service_level_objectives"]
+        rum_applications = self.config.state.destination["rum_applications"]
 
-        if r_obj.get("type") == "composite" and key == "query" and resource_to_connect != "service_level_objectives":
+        if (
+            r_obj.get("type") == "composite"
+            and key == "query"
+            and resource_to_connect not in ["service_level_objectives", "rum_applications"]
+        ):
             failed_connections = []
             ids = re.findall("[0-9]+", r_obj[key])
             for _id in ids:
@@ -131,6 +137,19 @@ def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optiona
                 else:
                     failed_connections.append(_id)
             return failed_connections
+        elif resource_to_connect == "rum_applications" and r_obj.get("type") == "rum alert" and key == "query":
+            failed_connections = []
+            regex = (
+                r"application.id"
+                + r"\:([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})"
+            )
+            if res := re.search(regex, r_obj[key]):
+                _id = res.group(1)
+                if _id in rum_applications:
+                    r_obj[key] = re.sub(_id, rum_applications[_id]["id"], r_obj[key])
+                else:
+                    failed_connections.append(_id)
+            return failed_connections
         elif key == "query":
             return None
         else:

datadog_sync/model/notebooks.py

@@ -8,6 +8,7 @@
 
 from datadog_sync.utils.base_resource import BaseResource, ResourceConfig
 from datadog_sync.utils.custom_client import PaginationConfig
+from datadog_sync.utils.resource_utils import CustomClientHTTPError, SkipResource
 
 if TYPE_CHECKING:
     from datadog_sync.utils.custom_client import CustomClient
@@ -50,7 +51,11 @@ async def get_resources(self, client: CustomClient) -> List[Dict]:
     async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]:
         if _id:
             source_client = self.config.source_client
-            resource = (await source_client.get(self.resource_config.base_path + f"/{_id}"))["data"]
+            try:
+                resource = (await source_client.get(self.resource_config.base_path + f"/{_id}"))["data"]
+            except CustomClientHTTPError as err:
+                if err.status_code == 403:
+                    raise SkipResource(_id, self.resource_type, "No access to restricted notebook")
 
         resource = cast(dict, resource)
         self.handle_special_case_attr(resource)
@@ -88,9 +93,6 @@ async def delete_resource(self, _id: str) -> None:
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
 
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass
-
     @staticmethod
     def handle_special_case_attr(resource):
         # Handle template_variables attribute

datadog_sync/model/roles.py

@@ -123,9 +123,6 @@ async def delete_resource(self, _id: str) -> None:
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
 
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass
-
     async def remap_permissions(self, resource):
         if not self.destination_permissions:
             try:

datadog_sync/model/rum_applications.py

@@ -29,6 +29,7 @@ class RUMApplications(BaseResource):
             "attributes.org_id",
             "attributes.updated_at",
             "attributes.updated_by_handle",
+            "attributes.product_analytics_preview_disabled",
             "attributes.product_scales.product_analytics_retention_scale.last_modified_at",
             "attributes.product_scales.product_analytics_retention_scale.state",
             "attributes.product_scales.rum_event_processing_scale.last_modified_at",
@@ -97,6 +98,3 @@ async def delete_resource(self, _id: str) -> None:
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/security_monitoring_rules.py

@@ -184,9 +184,6 @@ async def delete_resource(self, _id: str) -> None:
 
         await destination_client.delete(self.resource_config.base_path + f"/{destination_resource['id']}")
 
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass
-
     @staticmethod
     def limit_resource(resource):
         """Default and partner security rules have some fields that cannot be updated we need to remove them"""

datadog_sync/model/sensitive_data_scanner_groups.py

@@ -18,7 +18,7 @@ class SensitiveDataScannerGroups(BaseResource):
         base_path="/api/v2/sensitive-data-scanner/config",
         excluded_attributes=[
             "id",
-            "relationships",
+            "relationships.rules",
         ],
         concurrent=False,
     )
@@ -45,7 +45,13 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di
         return resource["id"], resource
 
     async def pre_resource_action_hook(self, _id, resource: Dict) -> None:
-        pass
+        """
+        The relationships.configuration.data.id is unique per org, so we need to pull
+        the config id from the destination and change the destination resource to have it
+        """
+        destination_client = self.config.destination_client
+        resp = await destination_client.get("/api/v2/sensitive-data-scanner/config")
+        resource["relationships"]["configuration"]["data"]["id"] = resp["data"]["id"]
 
     async def pre_apply_hook(self) -> None:
         pass
@@ -75,6 +81,3 @@ async def delete_resource(self, _id: str) -> None:
             self.resource_config.base_path + f"/groups/{self.config.state.destination[self.resource_type][_id]['id']}",
             body=payload,
         )
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/service_level_objectives.py

@@ -3,9 +3,11 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2019 Datadog, Inc.
 from __future__ import annotations
+from copy import deepcopy
 from typing import TYPE_CHECKING, Optional, List, Dict, Tuple, cast
 
 from datadog_sync.utils.base_resource import BaseResource, ResourceConfig, TaggingConfig
+from datadog_sync.utils.resource_utils import check_diff, SkipResource
 
 if TYPE_CHECKING:
     from datadog_sync.utils.custom_client import CustomClient
@@ -39,17 +41,44 @@ async def pre_resource_action_hook(self, _id, resource: Dict) -> None:
     async def pre_apply_hook(self) -> None:
         pass
 
+    async def _clean_resource_connections(self, _id: str, resource: Dict) -> Dict:
+        # SLO failed connections to monitors may need to be removed from the resource
+        cleaned_resource = deepcopy(resource)
+        if self.config.skip_failed_resource_connections:
+            if "monitor_ids" in cleaned_resource.keys():
+                failed_connections = self.connect_id("monitor_ids", cleaned_resource, "monitors")
+                for monitor_id in failed_connections:
+                    if "monitor_ids" in cleaned_resource.keys() and int(monitor_id) in cleaned_resource["monitor_ids"]:
+                        cleaned_resource["monitor_ids"].remove(int(monitor_id))
+            if _id in self.config.state.destination[self.resource_type]:
+                diff = check_diff(
+                    self.resource_config, self.config.state.destination[self.resource_type][_id], cleaned_resource
+                )
+                if not diff:
+                    raise SkipResource(_id, self.resource_type, "No differences after failed connections removed")
+        return cleaned_resource
+
     async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
-        destination_client = self.config.destination_client
-        resp = await destination_client.post(self.resource_config.base_path, resource)
+        cleaned_resource = await self._clean_resource_connections(
+            _id, self.config.state.source[self.resource_type][_id]
+        )
+        if resource.get("monitor_ids", None) and not cleaned_resource.get("monitor_ids", None):
+            raise SkipResource(_id, self.resource_type, "None of the monitor_ids exist at destination")
 
+        destination_client = self.config.destination_client
+        resp = await destination_client.post(self.resource_config.base_path, cleaned_resource)
         return _id, resp["data"][0]
 
     async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
+        cleaned_resource = await self._clean_resource_connections(
+            _id, self.config.state.source[self.resource_type][_id]
+        )
+        if resource.get("monitor_ids", None) and not cleaned_resource.get("monitor_ids", None):
+            raise SkipResource(_id, self.resource_type, "None of the monitor_ids exist at destination")
         destination_client = self.config.destination_client
         resp = await destination_client.put(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}",
-            resource,
+            cleaned_resource,
         )
 
         return _id, resp["data"][0]

datadog_sync/model/spans_metrics.py

@@ -61,6 +61,3 @@ async def delete_resource(self, _id: str) -> None:
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/synthetics_mobile_applications.py

@@ -16,9 +16,6 @@ class SyntheticsMobileApplications(BaseResource):
     resource_type = "synthetics_mobile_applications"
     resource_config = ResourceConfig(
         base_path="/api/unstable/synthetics/mobile/applications",
-        resource_connections={
-            "synthetics_mobile_applications_versions": ["versions.id"],
-        },
         excluded_attributes=[
             "id",
             "created_at",
@@ -75,6 +72,3 @@ async def delete_resource(self, _id: str) -> None:
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
-
-    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-        pass

datadog_sync/model/synthetics_mobile_applications_versions.py

@@ -191,7 +191,3 @@ async def delete_resource(self, _id: str) -> None:
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
-
-
-#    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
-#        pass