feat(integrations): Data Forwarding Endpoint to retrieve/create org-level configs (#101047)

Retrieve/Create functionality for org level configs. "GET" requires read
and gets all the configs for that org (including project overrides).
"POST" requires write and adds a new data forwarding config

---------

Co-authored-by: Irene Liu <[email protected]>

Author: liuirene256

src/sentry/api/urls.py

@@ -218,6 +218,7 @@
     TeamAlertsTriggeredTotalsEndpoint,
 )
 from sentry.insights.endpoints.starred_segments import InsightsStarredSegmentsEndpoint
+from sentry.integrations.api.endpoints.data_forwarding_index import DataForwardingIndexEndpoint
 from sentry.integrations.api.endpoints.doc_integration_avatar import DocIntegrationAvatarEndpoint
 from sentry.integrations.api.endpoints.doc_integration_details import DocIntegrationDetailsEndpoint
 from sentry.integrations.api.endpoints.doc_integrations_index import DocIntegrationsEndpoint
@@ -1403,6 +1404,12 @@ def create_group_urls(name_prefix: str) -> list[URLPattern | URLResolver]:
         OrganizationCodeMappingCodeOwnersEndpoint.as_view(),
         name="sentry-api-0-organization-code-mapping-codeowners",
     ),
+    # Data Forwarding
+    re_path(
+        r"^(?P<organization_id_or_slug>[^/]+)/forwarding/$",
+        DataForwardingIndexEndpoint.as_view(),
+        name="sentry-api-0-organization-forwarding",
+    ),
     re_path(
         r"^(?P<organization_id_or_slug>[^/]+)/codeowners-associations/$",
         OrganizationCodeOwnersAssociationsEndpoint.as_view(),

src/sentry/integrations/api/endpoints/data_forwarding_index.py

@@ -0,0 +1,85 @@
+from django.utils.decorators import method_decorator
+from django.views.decorators.cache import never_cache
+from drf_spectacular.utils import extend_schema
+from rest_framework import status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.api_owners import ApiOwner
+from sentry.api.api_publish_status import ApiPublishStatus
+from sentry.api.base import region_silo_endpoint
+from sentry.api.bases.organization import OrganizationEndpoint, OrganizationPermission
+from sentry.api.paginator import OffsetPaginator
+from sentry.api.serializers import serialize
+from sentry.apidocs.constants import RESPONSE_BAD_REQUEST, RESPONSE_CONFLICT, RESPONSE_FORBIDDEN
+from sentry.apidocs.parameters import GlobalParams
+from sentry.integrations.api.serializers.models.data_forwarder import (
+    DataForwarderSerializer as DataForwarderModelSerializer,
+)
+from sentry.integrations.api.serializers.rest_framework.data_forwarder import (
+    DataForwarderSerializer,
+)
+from sentry.integrations.models.data_forwarder import DataForwarder
+from sentry.web.decorators import set_referrer_policy
+
+
+class OrganizationDataForwardingDetailsPermission(OrganizationPermission):
+    scope_map = {
+        "GET": ["org:read"],
+        "POST": ["org:write"],
+    }
+
+
+@region_silo_endpoint
+@extend_schema(tags=["Integrations"])
+class DataForwardingIndexEndpoint(OrganizationEndpoint):
+    owner = ApiOwner.INTEGRATIONS
+    publish_status = {
+        "GET": ApiPublishStatus.EXPERIMENTAL,
+        "POST": ApiPublishStatus.EXPERIMENTAL,
+    }
+    permission_classes = (OrganizationDataForwardingDetailsPermission,)
+
+    @extend_schema(
+        operation_id="Retrieve Data Forwarding Configurations for an Organization",
+        parameters=[GlobalParams.ORG_ID_OR_SLUG],
+        responses={
+            200: DataForwarderModelSerializer,
+        },
+    )
+    @set_referrer_policy("strict-origin-when-cross-origin")
+    @method_decorator(never_cache)
+    def get(self, request: Request, organization) -> Response:
+        queryset = DataForwarder.objects.filter(organization_id=organization.id)
+
+        return self.paginate(
+            request=request,
+            queryset=queryset,
+            on_results=lambda x: serialize(x, request.user),
+            paginator_cls=OffsetPaginator,
+        )
+
+    @extend_schema(
+        operation_id="Create a Data Forwarding Configuration for an Organization",
+        parameters=[GlobalParams.ORG_ID_OR_SLUG],
+        request=DataForwarderSerializer,
+        responses={
+            201: DataForwarderModelSerializer,
+            400: RESPONSE_BAD_REQUEST,
+            403: RESPONSE_FORBIDDEN,
+            409: RESPONSE_CONFLICT,
+        },
+    )
+    @set_referrer_policy("strict-origin-when-cross-origin")
+    @method_decorator(never_cache)
+    def post(self, request: Request, organization) -> Response:
+        data = request.data
+        data["organization_id"] = organization.id
+
+        serializer = DataForwarderSerializer(data=data)
+        if not serializer.is_valid():
+            return self.respond(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        return self.respond(
+            serialize(serializer.save(), request.user), status=status.HTTP_201_CREATED
+        )

src/sentry/integrations/api/serializers/models/__init__.py

@@ -1,3 +1,4 @@
+from sentry.integrations.api.serializers.models.data_forwarder import *  # noqa: F401,F403
 from sentry.integrations.api.serializers.models.doc_integration import *  # noqa: F401,F403
 from sentry.integrations.api.serializers.models.doc_integration_avatar import *  # noqa: F401,F403
 from sentry.integrations.api.serializers.models.external_actor import *  # noqa: F401,F403

src/sentry/integrations/api/serializers/rest_framework/data_forwarder.py

@@ -1,5 +1,5 @@
 import re
-from collections.abc import MutableMapping
+from collections.abc import Mapping, MutableMapping
 from typing import Any, TypedDict
 
 from rest_framework import serializers
@@ -9,6 +9,7 @@
 from sentry.integrations.models.data_forwarder import DataForwarder
 from sentry.integrations.models.data_forwarder_project import DataForwarderProject
 from sentry.integrations.types import DataForwarderProviderSlug
+from sentry.models.project import Project
 from sentry_plugins.amazon_sqs.plugin import get_regions
 
 
@@ -124,7 +125,7 @@ def _validate_splunk_token_format(self, config: dict, errors: list[str]) -> None
             if not re.match(splunk_token_pattern, token):
                 errors.append("token must be a valid Splunk HEC token format")
 
-    def validate(self, attrs: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+    def validate(self, attrs: Mapping[str, Any]) -> Mapping[str, Any]:
         organization_id = attrs.get("organization_id")
         provider = attrs.get("provider")
 
@@ -178,6 +179,27 @@ def _validate_splunk_config(self, config) -> SplunkConfig:
 
         return config
 
+    def create(self, validated_data: Mapping[str, Any]) -> DataForwarder:
+        data_forwarder = DataForwarder.objects.create(**validated_data)
+
+        # Auto-enroll all existing projects in the organization
+        project_ids = Project.objects.filter(
+            organization_id=validated_data["organization_id"]
+        ).values_list("id", flat=True)
+
+        if project_ids:
+            project_configs = [
+                DataForwarderProject(
+                    data_forwarder=data_forwarder,
+                    project_id=project_id,
+                    is_enabled=True,
+                )
+                for project_id in project_ids
+            ]
+            DataForwarderProject.objects.bulk_create(project_configs, ignore_conflicts=True)
+
+        return data_forwarder
+
 
 class DataForwarderProjectSerializer(Serializer):
     data_forwarder_id = serializers.IntegerField()
@@ -226,3 +248,8 @@ def validate(self, attrs: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
             )
 
         return attrs
+
+    def create(self, validated_data: MutableMapping[str, Any]) -> DataForwarderProject:
+        project = validated_data.pop("project")
+        validated_data["project_id"] = project.id
+        return DataForwarderProject.objects.create(**validated_data)

src/sentry/testutils/factories.py

@@ -55,6 +55,7 @@
     IncidentType,
     TriggerStatus,
 )
+from sentry.integrations.models.data_forwarder import DataForwarder
 from sentry.integrations.models.doc_integration import DocIntegration
 from sentry.integrations.models.doc_integration_avatar import DocIntegrationAvatar
 from sentry.integrations.models.external_actor import ExternalActor
@@ -1193,6 +1194,13 @@ def create_group_activity(group, *args, **kwargs):
     def create_file(**kwargs):
         return File.objects.create(**kwargs)
 
+    @staticmethod
+    @assume_test_silo_mode(SiloMode.REGION)
+    def create_data_forwarder(organization, provider, config, **kwargs):
+        return DataForwarder.objects.create(
+            organization=organization, provider=provider, config=config, **kwargs
+        )
+
     @staticmethod
     @assume_test_silo_mode(SiloMode.REGION)
     def create_file_from_path(path, name=None, **kwargs):

src/sentry/testutils/fixtures.py

@@ -356,6 +356,11 @@ def create_n_groups_with_hashes(
     def create_file(self, **kwargs):
         return Factories.create_file(**kwargs)
 
+    def create_data_forwarder(self, organization=None, *args, **kwargs):
+        if organization is None:
+            organization = self.organization
+        return Factories.create_data_forwarder(organization, *args, **kwargs)
+
     def create_file_from_path(self, *args, **kwargs):
         return Factories.create_file_from_path(*args, **kwargs)