ref(seer grouping): Add new sentry:similarity_backfill_completed project option to enablement check (#74600)

Currently, backfilling a project's issues into the Seer database and enabling Seer grouping are two separate steps - the backfill script has to be run and then the project has to be manually added to the list of projects using the feature. This has a few disadvantages:

- Any groups which are created between the time the backfill ends and the time the enablement change is deployed get missed by Seer entirely.
- Because we're doing the backfill in batches, we're also doing the enablement in batches, which causes large, abrupt increases in the load being placed on Seer. This means that if Seer reaches an overload point, we both won't have much warning and won't know exactly at what point the load was too much.
- Manually adding projects to the enablement list is prone to error and kind of a pain.

To solve this problem, this PR adds a new project option, `sentry:similarity_backfill_completed`, which the Seer similarity backfill script can set once all of a project's issues have been backfilled. It also updates `should_call_seer_for_grouping` to consider the presence of the option equivalent to having the feature flag(s) on. That way:

- There's no lag time between the backfill ending and grouping being turned on.
- Traffic to Seer can increase one project at a time, which lets us better understand how it handles the increases in load.
- There's no more annoying copying and pasting.

For now the project option will live alongside the flag (rather than replacing it), so that events from projects which already have the feature flag on but don't yet have the project option set won’t suddenly be deemed Seer-ineligible. We can decide later whether we want to run a migration to set the option for projects which already have the flag (and then delete the flag) or just live with the hybrid system under the theory that they're both temporary, and will disappear once Seer grouping is GAed.

Author: lobsterkatie

src/sentry/grouping/ingest/seer.py

@@ -30,11 +30,7 @@ def should_call_seer_for_grouping(event: Event, primary_hashes: CalculatedHashes
 
     project = event.project
 
-    has_either_seer_grouping_feature = features.has(
-        "projects:similarity-embeddings-metadata", project
-    ) or features.has("projects:similarity-embeddings-grouping", project)
-
-    if not has_either_seer_grouping_feature:
+    if not _project_has_similarity_grouping_enabled(project):
         return False
 
     if _has_customized_fingerprint(event, primary_hashes):
@@ -60,6 +56,20 @@ def should_call_seer_for_grouping(event: Event, primary_hashes: CalculatedHashes
     return True
 
 
+def _project_has_similarity_grouping_enabled(project: Project) -> bool:
+    has_either_seer_grouping_feature = features.has(
+        "projects:similarity-embeddings-metadata", project
+    ) or features.has("projects:similarity-embeddings-grouping", project)
+
+    # TODO: This is a hack to get ingest to turn on for projects as soon as they're backfilled. When
+    # the backfill script completes, we turn on this option, enabling ingest immediately rather than
+    # forcing the project to wait until it's been manually added to a feature handler. Once all
+    # projects have been backfilled, the option (and this check) can go away.
+    has_been_backfilled = project.get_option("sentry:similarity_backfill_completed")
+
+    return has_either_seer_grouping_feature or has_been_backfilled
+
+
 # TODO: Here we're including events with hybrid fingerprints (ones which are `{{ default }}`
 # combined with some other value). To the extent to which we're then using this function to decide
 # whether or not to call Seer, this means that the calculations giving rise to the default part of

src/sentry/models/options/project_option.py

@@ -53,6 +53,7 @@
         "sentry:secondary_grouping_config",
         "sentry:secondary_grouping_expiry",
         "sentry:grouping_auto_update",
+        "sentry:similarity_backfill_completed",
         "sentry:fingerprinting_rules",
         "sentry:relay_pii_config",
         "sentry:metrics_extraction_rules",

src/sentry/projectoptions/defaults.py

@@ -44,6 +44,12 @@
 # is auto upgrading enabled?
 register(key="sentry:grouping_auto_update", default=True)
 
+# Has this project had its issues backfilled into the Seer database, and if so, when did the
+# backfill complete? (This is a temporary way to flag projects as we roll out Seer grouping, because
+# it can be flipped on in the backfill script, unlike inclusion in a getsentry feature handler.)
+register(key="sentry:similarity_backfill_completed", default=None)
+
+
 # The JavaScript loader version that is the project default.  This option
 # is expected to be never set but the epoch defaults are used if no
 # version is set on a project's DSN.

tests/sentry/grouping/ingest/test_seer.py

@@ -45,23 +45,33 @@ def setUp(self):
         )
         self.primary_hashes = self.event.get_hashes()
 
-    def test_obeys_seer_similarity_flags(self):
-        for metadata_flag, grouping_flag, expected_result in [
-            (False, False, False),
-            (True, False, True),
-            (False, True, True),
-            (True, True, True),
+    def test_obeys_feature_enablement_check(self):
+        for metadata_flag, grouping_flag, backfill_completed_option, expected_result in [
+            # TODO: This manual cartesian product business is gross, but thankfully it's temporary -
+            # the metadata flag is about to go away and the backfill completed option will go away
+            # once all projects are backfilled.
+            (False, False, None, False),
+            (True, False, None, True),
+            (False, True, None, True),
+            (True, True, None, True),
+            (False, False, 11211231, True),
+            (True, False, 11211231, True),
+            (False, True, 11211231, True),
+            (True, True, 11211231, True),
         ]:
             with Feature(
                 {
                     "projects:similarity-embeddings-metadata": metadata_flag,
                     "projects:similarity-embeddings-grouping": grouping_flag,
                 }
             ):
+                self.project.update_option(
+                    "sentry:similarity_backfill_completed", backfill_completed_option
+                )
                 assert (
                     should_call_seer_for_grouping(self.event, self.primary_hashes)
                     is expected_result
-                ), f"Case ({metadata_flag}, {grouping_flag}) failed."
+                ), f"Case (metadata {metadata_flag}, grouping {grouping_flag}, backfill completed {backfill_completed_option}) failed."
 
     @with_feature("projects:similarity-embeddings-grouping")
     def test_obeys_content_filter(self):