Michael.richey/allow partial permissions (#421)

michael-richey · web-flow · commit 5ee6821a6bca · 2025-12-01T15:33:44.000-05:00
* Allow roles to be created/updated without certain permissions

* Skip existing roles
diff --git a/datadog_sync/commands/shared/options.py b/datadog_sync/commands/shared/options.py
@@ -359,6 +359,14 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non
         "user determines monitors have enough telemetry to trigger appropriately.",
         cls=CustomOptionClass,
     ),
+    option(
+        "--allow-partial-permissions-roles",
+        required=False,
+        envvar=constants.DD_ALLOW_PARTIAL_PERMISSIONS_ROLES,
+        help="Comma separated list of permissions to allow partial sync for roles. "
+        "If a role has a permission that doesn't exist in the destination, it will be removed and retried.",
+        cls=CustomOptionClass,
+    ),
 ]
 
 
diff --git a/datadog_sync/constants.py b/datadog_sync/constants.py
@@ -23,6 +23,7 @@
 DD_VERIFY_DDR_STATUS = "DD_VERIFY_DDR_STATUS"
 DD_SHOW_PROGRESS_BAR = "DD_SHOW_PROGRESS_BAR"
 DD_VERIFY_SSL_CERTIFICATES = "DD_VERIFY_SSL_CERTIFICATES"
+DD_ALLOW_PARTIAL_PERMISSIONS_ROLES = "DD_ALLOW_PARTIAL_PERMISSIONS_ROLES"
 
 LOCAL_STORAGE_TYPE = "local"
 S3_STORAGE_TYPE = "s3"
diff --git a/datadog_sync/model/roles.py b/datadog_sync/model/roles.py
@@ -5,10 +5,12 @@
 
 from __future__ import annotations
 import copy
+import json
+import re
 from typing import TYPE_CHECKING, Optional, List, Dict, Tuple, cast
 
 from datadog_sync.utils.base_resource import BaseResource, ResourceConfig
-from datadog_sync.utils.resource_utils import CustomClientHTTPError, check_diff
+from datadog_sync.utils.resource_utils import CustomClientHTTPError, check_diff, SkipResource
 
 if TYPE_CHECKING:
     from datadog_sync.utils.custom_client import CustomClient
@@ -84,9 +86,52 @@ async def create_resource(self, _id, resource) -> Tuple[str, Dict]:
         # role does not exist at the destination, so create it
         if role_name not in self.destination_roles_mapping:
             destination_client = self.config.destination_client
-            payload = {"data": resource}
-            resp = await destination_client.post(self.resource_config.base_path, payload)
-            return _id, resp["data"]
+
+            # Retry loop to handle multiple invalid permissions
+            max_retries = 1 + len(self.config.allow_partial_permissions_roles)
+            retry_count = 0
+
+            while retry_count < max_retries:
+                payload = {"data": resource}
+                try:
+                    resp = await destination_client.post(self.resource_config.base_path, payload)
+                    return _id, resp["data"]
+                except CustomClientHTTPError as e:
+                    if e.status_code == 400 and self.config.allow_partial_permissions_roles:
+                        # Try to parse the invalid permission from the error
+                        invalid_permission = self._parse_invalid_permission_from_error(str(e))
+
+                        if invalid_permission and invalid_permission in self.config.allow_partial_permissions_roles:
+                            # Check if we removed the permission successfully
+                            if self._remove_permission_from_resource(resource, invalid_permission):
+                                self.config.logger.warning(
+                                    f"Trying again without '{invalid_permission}' permission for role '{role_name}'"
+                                )
+
+                                # Check if the modified resource now matches an existing destination role
+                                # This can happen if we already synced the role without this permission before
+                                if role_name in self.destination_roles_mapping:
+                                    matching_destination_role = self.destination_roles_mapping[role_name]
+                                    role_copy = copy.deepcopy(resource)
+                                    role_copy.update(matching_destination_role)
+
+                                    # If there's no diff, the role already exists without this permission
+                                    if not check_diff(self.resource_config, resource, role_copy):
+                                        raise SkipResource(
+                                            _id,
+                                            self.resource_type,
+                                            f"Role '{role_name}' already exists at destination "
+                                            "without '{invalid_permission}' permission",
+                                        )
+
+                                retry_count += 1
+                                continue  # Retry with the updated resource
+
+                    # If we couldn't handle the error, re-raise it
+                    raise
+
+            # If we exhausted retries, raise an error
+            raise Exception(f"Exceeded maximum retries ({max_retries}) while creating role '{role_name}'")
 
         # role already exists at the destination
         matching_destination_role = self.destination_roles_mapping[role_name]
@@ -108,21 +153,115 @@ async def create_resource(self, _id, resource) -> Tuple[str, Dict]:
 
     async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         destination_client = self.config.destination_client
+        role_name = resource["attributes"]["name"]
         resource["id"] = self.config.state.destination[self.resource_type][_id]["id"]
-        payload = {"data": resource}
-        resp = await destination_client.patch(
-            self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}",
-            payload,
-        )
 
-        return _id, resp["data"]
+        # Retry loop to handle multiple invalid permissions
+        max_retries = 1 + len(self.config.allow_partial_permissions_roles)
+        retry_count = 0
+
+        while retry_count < max_retries:
+            payload = {"data": resource}
+            try:
+                resp = await destination_client.patch(
+                    self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}",
+                    payload,
+                )
+                return _id, resp["data"]
+            except CustomClientHTTPError as e:
+                if e.status_code == 400 and self.config.allow_partial_permissions_roles:
+                    # Try to parse the invalid permission from the error
+                    invalid_permission = self._parse_invalid_permission_from_error(str(e))
+
+                    if invalid_permission and invalid_permission in self.config.allow_partial_permissions_roles:
+                        # Check if we removed the permission successfully
+                        if self._remove_permission_from_resource(resource, invalid_permission):
+                            self.config.logger.warning(
+                                f"Trying again without '{invalid_permission}' permission for role '{role_name}'"
+                            )
+
+                            # Check if the modified resource now matches the existing destination state
+                            # This can happen if we already synced the role without this permission before
+                            if _id in self.config.state.destination[self.resource_type]:
+                                destination_resource = self.config.state.destination[self.resource_type][_id]
+
+                                # If there's no diff, the role already exists without this permission
+                                if not check_diff(self.resource_config, resource, destination_resource):
+                                    raise SkipResource(
+                                        _id,
+                                        self.resource_type,
+                                        f"Role '{role_name}' already exists at destination "
+                                        "without '{invalid_permission}' permission",
+                                    )
+
+                            retry_count += 1
+                            continue  # Retry with the updated resource
+
+                # If we couldn't handle the error, re-raise it
+                raise
+
+        # If we exhausted retries, raise an error
+        raise Exception(f"Exceeded maximum retries ({max_retries}) while updating role '{role_name}'")
 
     async def delete_resource(self, _id: str) -> None:
         destination_client = self.config.destination_client
         await destination_client.delete(
             self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
         )
 
+    def _parse_invalid_permission_from_error(self, error_message: str) -> Optional[str]:
+        """Parse the invalid permission name from a 400 error message.
+
+        Example error: '400 Bad Request - {"title":"Generic Error","detail":"invalid UUID [assistant_access]"}'
+        Returns: "assistant_access"
+        """
+        try:
+            # Try to extract JSON from error message
+            match = re.search(r"\{.*\}", error_message)
+            if not match:
+                return None
+
+            error_json = json.loads(match.group(0))
+
+            # Look for "invalid UUID [permission_name]" pattern
+            detail = error_json.get("detail", "")
+            uuid_match = re.search(r"invalid UUID \[([^\]]+)\]", detail)
+            if uuid_match:
+                return uuid_match.group(1)
+
+            # Also check in errors array if present
+            errors = error_json.get("errors", [])
+            for error in errors:
+                if isinstance(error, str):
+                    uuid_match = re.search(r"invalid UUID \[([^\]]+)\]", error)
+                    if uuid_match:
+                        return uuid_match.group(1)
+                elif isinstance(error, dict):
+                    detail = error.get("detail", "")
+                    uuid_match = re.search(r"invalid UUID \[([^\]]+)\]", detail)
+                    if uuid_match:
+                        return uuid_match.group(1)
+        except (json.JSONDecodeError, KeyError, AttributeError):
+            pass
+
+        return None
+
+    def _remove_permission_from_resource(self, resource: Dict, permission_id: str) -> bool:
+        """Remove a permission from the resource's relationships.
+
+        Returns True if permission was found and removed, False otherwise.
+        """
+        if "relationships" not in resource or "permissions" not in resource["relationships"]:
+            return False
+
+        permissions_data = resource["relationships"]["permissions"].get("data", [])
+        original_length = len(permissions_data)
+
+        # Filter out the permission
+        resource["relationships"]["permissions"]["data"] = [p for p in permissions_data if p.get("id") != permission_id]
+
+        return len(resource["relationships"]["permissions"]["data"]) < original_length
+
     async def remap_permissions(self, resource):
         if not self.destination_permissions:
             try:
diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py
@@ -59,6 +59,7 @@ class Configuration(object):
     verify_ddr_status: bool
     backup_before_reset: bool
     show_progress_bar: bool
+    allow_partial_permissions_roles: List[str] = field(default_factory=list)
     resources: Dict[str, BaseResource] = field(default_factory=dict)
     resources_arg: List[str] = field(default_factory=list)
 
@@ -173,6 +174,11 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
     backup_before_reset = not kwargs.get("do_not_backup")
     show_progress_bar = kwargs.get("show_progress_bar")
 
+    # Parse allow_partial_permissions_roles
+    allow_partial_permissions_roles = []
+    if allow_partial_str := kwargs.get("allow_partial_permissions_roles"):
+        allow_partial_permissions_roles = [p.strip() for p in allow_partial_str.split(",")]
+
     cleanup = kwargs.get("cleanup")
     if cleanup:
         cleanup = {
@@ -251,6 +257,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
         verify_ddr_status=verify_ddr_status,
         backup_before_reset=backup_before_reset,
         show_progress_bar=show_progress_bar,
+        allow_partial_permissions_roles=allow_partial_permissions_roles,
     )
 
     # Initialize resource classes
