feat(ownership): Migrate issue owners cache invalidation to use timestamp versioning on ownership (#106108)

We've been seeing [some
timeouts](https://sentry.sentry.io/issues/6731345393/?project=1&referrer=Linear)
in the `code_owners_auto_sync` task since it performs expensive O(n)
cache invalidation. When CODEOWNERS or ownership rules changed, we would
previously:
1. Query all groups in the project with recent events
2. Delete the issue owners debounce cache entry for each group
individually (via batched `cache.delete_many`)

The solution implemented in this PR is to replace this expensive fan-out
cache invalidation with an O(1) timestamp-based versioning system on
ownership schemas.

**Before**:
- Issue owners debounce cache simply stored `True` for each group
- On ownership change: delete cache entries for ALL active groups in the
project

**After**:
- Issue owners debounce cache stores the timestamp of when owners were
last evaluated for the group
- On ownership change: set a cache value with a project-wide
`ownership_changed_at` timestamp
- On debounce check:
- If `issue_owners_debounce_time` is None --> no debounce, evaluate
ownership
- If `issue_owners_debounce_time` is True --> legacy cache value,
debounce (will disappear from cache 24hrs after this PR is rolled out,
and then this logic can be removed)
- If `issue_owners_debounce_time` is a timestamp --> compare timestamps
- If `ownership_changed_at` is None or `ownership_changed_at` <
`issue_owners_debounce_time` --> debounce
- If `ownership_changed_at` >= `issue_owners_debounce_time` -->
ownership rules changed in the interim, evaluate ownership

### Follow-up

Left TODOs for both of these in the code:
- Remove backwards compatibility logic for `True` cache values 24 hrs
after this PR is rolled out
- Investigate applying same O(1) cache invalidation pattern to
`invalidate_assignee_exists_cache`

Author: shashjar

src/sentry/models/groupassignee.py

@@ -224,7 +224,7 @@ def deassign(
             if not ownership:
                 ownership = ProjectOwnership(project_id=group.project.id)
             GroupOwner.invalidate_assignee_exists_cache(group.project.id, group.id)
-            GroupOwner.invalidate_debounce_issue_owners_evaluation_cache(group.project.id, group.id)
+            GroupOwner.invalidate_debounce_issue_owners_evaluation_cache(group.id)
 
             metrics.incr("group.assignee.change", instance="deassigned", skip_internal=True)
             # sync Sentry assignee to external issues

src/sentry/models/groupowner.py

@@ -28,6 +28,8 @@
 ASSIGNEE_EXISTS_KEY = lambda group_id: f"assignee_exists:1:{group_id}"
 ASSIGNEE_EXISTS_DURATION = 60 * 60 * 24
 ASSIGNEE_DOES_NOT_EXIST_DURATION = 60
+PROJECT_OWNERSHIP_VERSION_KEY = lambda project_id: f"ownership_version:1:{project_id}"
+PROJECT_OWNERSHIP_VERSION_DURATION = 60 * 60 * 24
 
 
 class GroupOwnerType(Enum):
@@ -190,32 +192,13 @@ def get_autoassigned_owner(cls, group_id, project_id, autoassignment_types):
         return issue_owner
 
     @classmethod
-    def invalidate_debounce_issue_owners_evaluation_cache(cls, project_id, group_id=None):
+    def invalidate_debounce_issue_owners_evaluation_cache(cls, group_id):
         """
-        If `group_id` is provided, clear the debounce issue owners cache for that group, else clear
-        the cache of all groups for a project that had an event within the
-        ISSUE_OWNERS_DEBOUNCE_DURATION window.
+        Clear the debounce issue owners cache for a specific group.
         """
-        if group_id:
-            cache.delete(ISSUE_OWNERS_DEBOUNCE_KEY(group_id))
-            return
-
-        # Get all the groups for a project that had an event within the ISSUE_OWNERS_DEBOUNCE_DURATION window.
-        # Any groups without events in that window would have expired their TTL in the cache.
-        queryset = Group.objects.filter(
-            project_id=project_id,
-            last_seen__gte=timezone.now() - timedelta(seconds=ISSUE_OWNERS_DEBOUNCE_DURATION),
-        ).values_list("id", flat=True)
-
-        # Run cache invalidation in batches
-        group_id_iter = queryset.iterator(chunk_size=1000)
-        while True:
-            group_ids = list(itertools.islice(group_id_iter, 1000))
-            if not group_ids:
-                break
-            cache_keys = [ISSUE_OWNERS_DEBOUNCE_KEY(group_id) for group_id in group_ids]
-            cache.delete_many(cache_keys)
+        cache.delete(ISSUE_OWNERS_DEBOUNCE_KEY(group_id))
 
+    # TODO(shashank): can make this O(1) cache invalidation by using the project ownership version timestamp (follow-up PR)
     @classmethod
     def invalidate_assignee_exists_cache(cls, project_id, group_id=None):
         """
@@ -243,6 +226,34 @@ def invalidate_assignee_exists_cache(cls, project_id, group_id=None):
             cache_keys = [ASSIGNEE_EXISTS_KEY(group_id) for group_id in group_ids]
             cache.delete_many(cache_keys)
 
+    @classmethod
+    def set_project_ownership_version(cls, project_id: int) -> float:
+        """
+        Set the project ownership version timestamp when ownership rules change.
+
+        When ownership rules (ProjectCodeOwners or ProjectOwnership) change, we set a
+        timestamp. During debounce checks, we compare the group's debounce timestamp
+        against this version timestamp to determine if re-evaluation is needed.
+
+        Returns the timestamp that was set.
+        """
+        version_time = timezone.now().timestamp()
+        cache.set(
+            PROJECT_OWNERSHIP_VERSION_KEY(project_id),
+            version_time,
+            PROJECT_OWNERSHIP_VERSION_DURATION,
+        )
+        return version_time
+
+    @classmethod
+    def get_project_ownership_version(cls, project_id: int) -> float | None:
+        """
+        Get the project ownership version timestamp.
+
+        Returns the timestamp when ownership rules were last changed, or None if not set.
+        """
+        return cache.get(PROJECT_OWNERSHIP_VERSION_KEY(project_id))
+
 
 def get_owner_details(group_list: Sequence[Group]) -> dict[int, list[OwnersSerialized]]:
     group_ids = [g.id for g in group_list]

src/sentry/models/projectcodeowners.py

@@ -151,18 +151,14 @@ def modify_date_updated(instance, **kwargs):
 
 def process_resource_change(instance, change, **kwargs):
     from sentry.models.groupowner import GroupOwner
-    from sentry.models.projectownership import ProjectOwnership
 
     cache.set(
         ProjectCodeOwners.get_cache_key(instance.project_id),
         None,
         READ_CACHE_DURATION,
     )
-    ownership = ProjectOwnership.get_ownership_cached(instance.project_id)
-    if not ownership:
-        ownership = ProjectOwnership(project_id=instance.project_id)
 
-    GroupOwner.invalidate_debounce_issue_owners_evaluation_cache(instance.project_id)
+    GroupOwner.set_project_ownership_version(instance.project_id)
 
 
 pre_save.connect(

src/sentry/models/projectownership.py

@@ -423,7 +423,7 @@ def process_resource_change(instance, change, **kwargs):
         READ_CACHE_DURATION,
     )
     GroupOwner.invalidate_assignee_exists_cache(instance.project.id)
-    GroupOwner.invalidate_debounce_issue_owners_evaluation_cache(instance.project_id)
+    GroupOwner.set_project_ownership_version(instance.project_id)
 
 
 # Signals update the cached reads used in post_processing

src/sentry/tasks/post_process.py

@@ -229,6 +229,7 @@ def handle_owner_assignment(job):
         ASSIGNEE_EXISTS_KEY,
         ISSUE_OWNERS_DEBOUNCE_DURATION,
         ISSUE_OWNERS_DEBOUNCE_KEY,
+        GroupOwner,
     )
     from sentry.models.projectownership import ProjectOwnership
 
@@ -250,11 +251,25 @@ def handle_owner_assignment(job):
         return
 
     issue_owners_key = ISSUE_OWNERS_DEBOUNCE_KEY(group.id)
-    debounce_issue_owners = cache.get(issue_owners_key)
+    issue_owners_debounce_time = cache.get(issue_owners_key)
+
+    # Debounce check with timestamp-based invalidation:
+    # - issue_owners_debounce_time is None: no debounce, process ownership
+    # - issue_owners_debounce_time is True: legacy format, debounce (backwards compatibility)
+    # - issue_owners_debounce_time is float: compare timestamps
+    #   - If ownership_changed_at is None or ownership_changed_at < issue_owners_debounce_time: debounce
+    #   - If ownership_changed_at >= issue_owners_debounce_time: ownership rules changed, re-evaluate
+    if issue_owners_debounce_time is not None:
+        # TODO(shashank): once rolled out for 24hrs, remove this legacy cache format check and the comment line above. prob will need to remove some tests in test_post_process.py as well.
+        if issue_owners_debounce_time is True:
+            # Backwards compatibility: old cache format, debounce
+            metrics.incr("sentry.tasks.post_process.handle_owner_assignment.debounce")
+            return
 
-    if debounce_issue_owners:
-        metrics.incr("sentry.tasks.post_process.handle_owner_assignment.debounce")
-        return
+        ownership_changed_at = GroupOwner.get_project_ownership_version(project.id)
+        if ownership_changed_at is None or ownership_changed_at < issue_owners_debounce_time:
+            metrics.incr("sentry.tasks.post_process.handle_owner_assignment.debounce")
+            return
 
     if should_issue_owners_ratelimit(
         project_id=project.id,
@@ -286,7 +301,7 @@ def handle_owner_assignment(job):
         issue_owners = ProjectOwnership.get_issue_owners(project.id, event.data)
         cache.set(
             issue_owners_key,
-            True,
+            timezone.now().timestamp(),
             ISSUE_OWNERS_DEBOUNCE_DURATION,
         )
 

tests/sentry/models/test_projectcodeowners.py

@@ -1,4 +1,5 @@
 from sentry.issues.ownership.grammar import Matcher, Owner, Rule, dump_schema
+from sentry.models.groupowner import PROJECT_OWNERSHIP_VERSION_KEY, GroupOwner
 from sentry.models.projectcodeowners import ProjectCodeOwners
 from sentry.testutils.cases import TestCase
 from sentry.utils.cache import cache
@@ -88,3 +89,33 @@ def test_merge_codeowners(self) -> None:
                 },
             ],
         }
+
+    def test_post_save_sets_ownership_version(self) -> None:
+        cache.delete(PROJECT_OWNERSHIP_VERSION_KEY(self.project.id))
+        assert GroupOwner.get_project_ownership_version(self.project.id) is None
+
+        self.create_codeowners(
+            self.project,
+            self.code_mapping,
+            raw=self.data["raw"],
+        )
+
+        version = GroupOwner.get_project_ownership_version(self.project.id)
+        assert version is not None
+        assert isinstance(version, float)
+
+    def test_post_delete_sets_ownership_version(self) -> None:
+        codeowners = self.create_codeowners(
+            self.project,
+            self.code_mapping,
+            raw=self.data["raw"],
+        )
+
+        cache.delete(PROJECT_OWNERSHIP_VERSION_KEY(self.project.id))
+        assert GroupOwner.get_project_ownership_version(self.project.id) is None
+
+        codeowners.delete()
+
+        version = GroupOwner.get_project_ownership_version(self.project.id)
+        assert version is not None
+        assert isinstance(version, float)

tests/sentry/models/test_projectownership.py

@@ -2,7 +2,12 @@
 
 from sentry.issues.ownership.grammar import Matcher, Owner, Rule, dump_schema, resolve_actors
 from sentry.models.groupassignee import GroupAssignee
-from sentry.models.groupowner import GroupOwner, GroupOwnerType, OwnerRuleType
+from sentry.models.groupowner import (
+    PROJECT_OWNERSHIP_VERSION_KEY,
+    GroupOwner,
+    GroupOwnerType,
+    OwnerRuleType,
+)
 from sentry.models.projectownership import ProjectOwnership
 from sentry.models.repository import Repository
 from sentry.testutils.cases import TestCase
@@ -12,6 +17,7 @@
 from sentry.types.actor import Actor, ActorType
 from sentry.users.models.user_avatar import UserAvatar
 from sentry.users.services.user.service import user_service
+from sentry.utils.cache import cache
 
 pytestmark = requires_snuba
 
@@ -772,6 +778,34 @@ def test_autoassignment_with_multiple_codeowners(self) -> None:
         assignee = GroupAssignee.objects.get(group=event.group)
         assert assignee.team_id == processing_team.id
 
+    def test_post_save_sets_ownership_version(self) -> None:
+        cache.delete(PROJECT_OWNERSHIP_VERSION_KEY(self.project.id))
+        assert GroupOwner.get_project_ownership_version(self.project.id) is None
+
+        ProjectOwnership.objects.create(
+            project_id=self.project.id,
+            schema=dump_schema([Rule(Matcher("path", "*.py"), [Owner("user", self.user.email)])]),
+        )
+
+        version = GroupOwner.get_project_ownership_version(self.project.id)
+        assert version is not None
+        assert isinstance(version, float)
+
+    def test_post_delete_sets_ownership_version(self) -> None:
+        ownership = ProjectOwnership.objects.create(
+            project_id=self.project.id,
+            schema=dump_schema([Rule(Matcher("path", "*.py"), [Owner("user", self.user.email)])]),
+        )
+
+        cache.delete(PROJECT_OWNERSHIP_VERSION_KEY(self.project.id))
+        assert GroupOwner.get_project_ownership_version(self.project.id) is None
+
+        ownership.delete()
+
+        version = GroupOwner.get_project_ownership_version(self.project.id)
+        assert version is not None
+        assert isinstance(version, float)
+
 
 class ResolveActorsTestCase(TestCase):
     def test_no_actors(self) -> None:

tests/sentry/tasks/test_post_process.py

@@ -1444,6 +1444,116 @@ def test_debounces_handle_owner_assignments(self, mock_incr: MagicMock) -> None:
         )
         mock_incr.assert_any_call("sentry.tasks.post_process.handle_owner_assignment.debounce")
 
+    @patch("sentry.utils.metrics.incr")
+    def test_debounces_with_timestamp_format(self, mock_incr: MagicMock) -> None:
+        self.make_ownership()
+        event = self.create_event(
+            data={
+                "message": "oh no",
+                "platform": "python",
+                "stacktrace": {"frames": [{"filename": "src/app.py"}]},
+            },
+            project_id=self.project.id,
+        )
+        debounce_time = cache.get(ISSUE_OWNERS_DEBOUNCE_KEY(event.group_id))
+        assert debounce_time is None
+
+        # First event: evaluates ownership and sets debounce timestamp
+        self.call_post_process_group(
+            is_new=False,
+            is_regression=False,
+            is_new_group_environment=False,
+            event=event,
+        )
+        debounce_time = cache.get(ISSUE_OWNERS_DEBOUNCE_KEY(event.group_id))
+        assert debounce_time is not None
+        assert isinstance(debounce_time, float)
+
+        # Second event: should debounce because no ownership change occurred
+        self.call_post_process_group(
+            is_new=False,
+            is_regression=False,
+            is_new_group_environment=False,
+            event=event,
+        )
+        mock_incr.assert_any_call("sentry.tasks.post_process.handle_owner_assignment.debounce")
+
+    @patch("sentry.utils.metrics.incr")
+    def test_timestamp_invalidation_when_ownership_changes(self, mock_incr: MagicMock) -> None:
+        self.make_ownership()
+        event = self.create_event(
+            data={
+                "message": "oh no",
+                "platform": "python",
+                "stacktrace": {"frames": [{"filename": "src/app.py"}]},
+            },
+            project_id=self.project.id,
+        )
+        debounce_time = cache.get(ISSUE_OWNERS_DEBOUNCE_KEY(event.group_id))
+        assert debounce_time is None
+
+        # First event: should evaluate ownership and set debounce timestamp
+        self.call_post_process_group(
+            is_new=False,
+            is_regression=False,
+            is_new_group_environment=False,
+            event=event,
+        )
+        debounce_time = cache.get(ISSUE_OWNERS_DEBOUNCE_KEY(event.group_id))
+        assert debounce_time is not None
+        assert isinstance(debounce_time, float)
+
+        # Simulate ownership rules changing
+        GroupOwner.set_project_ownership_version(self.project.id)
+
+        # Second event: should NOT debounce because ownership changed after debounce was set
+        self.call_post_process_group(
+            is_new=False,
+            is_regression=False,
+            is_new_group_environment=False,
+            event=event,
+        )
+        assert (
+            mock.call("sentry.tasks.post_process.handle_owner_assignment.debounce")
+            not in mock_incr.call_args_list
+        )
+
+    @patch("sentry.utils.metrics.incr")
+    def test_timestamp_debounce_when_ownership_older(self, mock_incr: MagicMock) -> None:
+        self.make_ownership()
+        event = self.create_event(
+            data={
+                "message": "oh no",
+                "platform": "python",
+                "stacktrace": {"frames": [{"filename": "src/app.py"}]},
+            },
+            project_id=self.project.id,
+        )
+        debounce_time = cache.get(ISSUE_OWNERS_DEBOUNCE_KEY(event.group_id))
+        assert debounce_time is None
+
+        # Ownership changed in the past (before any events)
+        GroupOwner.set_project_ownership_version(self.project.id)
+
+        # First event: evaluates ownership (ownership change is in the past, so no debounce yet)
+        self.call_post_process_group(
+            is_new=False,
+            is_regression=False,
+            is_new_group_environment=False,
+            event=event,
+        )
+        debounce_time = cache.get(ISSUE_OWNERS_DEBOUNCE_KEY(event.group_id))
+        assert debounce_time is not None
+
+        # Second event: should debounce because ownership change is older than the debounce timestamp
+        self.call_post_process_group(
+            is_new=False,
+            is_regression=False,
+            is_new_group_environment=False,
+            event=event,
+        )
+        mock_incr.assert_any_call("sentry.tasks.post_process.handle_owner_assignment.debounce")
+
     @patch("sentry.utils.metrics.incr")
     def test_issue_owners_should_ratelimit(self, mock_incr: MagicMock) -> None:
         cache.set(