feat: make it possible to disable using offset on API endpoint per team (#42534)

Author: orian

posthog/api/event.py

@@ -203,16 +203,18 @@ def list(self, request: request.Request, *args: Any, **kwargs: Any) -> response.
             except ValueError:
                 offset = 0
 
-            if settings.PATCH_EVENT_LIST_MAX_OFFSET > 0:
+            team = self.team
+
+            deprecate_offset = (
+                settings.PATCH_EVENT_LIST_MAX_OFFSET > 1 or team.id in settings.PATCH_EVENT_LIST_MAX_OFFSET_PER_TEAM
+            )
+            if settings.PATCH_EVENT_LIST_MAX_OFFSET > 0 or deprecate_offset:
                 if offset > 0:
                     time.sleep(1)
-                if offset > 50000 and (
-                    settings.PATCH_EVENT_LIST_MAX_OFFSET > 1 or random.random() < 0.01
-                ):  # 1% of queries fail
-                    raise serializers.ValidationError("Max supported offset value is 50000")
+                if offset > 50000 and (deprecate_offset or random.random() < 0.01):  # 1% of queries fail
+                    raise serializers.ValidationError("Offset is deprecated. Max supported offset value is 50000")
 
-            team = self.team
-            filter = Filter(request=request, team=self.team)
+            filter = Filter(request=request, team=team)
             order_by: list[str] = (
                 list(json.loads(request.GET["orderBy"])) if request.GET.get("orderBy") else ["-timestamp"]
             )
@@ -251,7 +253,14 @@ def list(self, request: request.Request, *args: Any, **kwargs: Any) -> response.
                 next_url = self._build_next_url(request, query_result[limit - 1]["timestamp"], order_by)
             headers = None
             if settings.PATCH_EVENT_LIST_MAX_OFFSET > 0:
-                headers = {"X-PostHog-Notif": "https://posthog.com/docs/events_list-upcoming-changes"}
+                headers = {"X-PostHog-Warn": "https://posthog.com/docs/events_list-upcoming-changes"}
+            elif deprecate_offset and offset:
+                headers = {
+                    "X-PostHog-Warn": (
+                        "offset is deprecated. "
+                        "Use: https://posthog.com/docs/api/queries#5-use-timestamp-based-pagination-instead-of-offset"
+                    )
+                }
             return response.Response({"next": next_url, "results": result}, headers=headers)
 
         except Exception as ex:

posthog/settings/data_stores.py

@@ -10,6 +10,7 @@
 
 from posthog.settings.base_variables import DEBUG, IN_EVAL_TESTING, IS_COLLECT_STATIC, TEST
 from posthog.settings.utils import get_from_env, get_list, str_to_bool
+from posthog.utils import str_to_int_set
 
 # See https://docs.djangoproject.com/en/3.2/ref/settings/#std:setting-DATABASE-DISABLE_SERVER_SIDE_CURSORS
 DISABLE_SERVER_SIDE_CURSORS: bool = get_from_env("USING_PGBOUNCER", False, type_cast=str_to_bool)
@@ -482,3 +483,6 @@ def _parse_kafka_hosts(hosts_string: str) -> list[str]:
 
 # Limiting event_list API, saving ClickHouse, 0 - disabled, 1 - migration period, 2 - enabled.
 PATCH_EVENT_LIST_MAX_OFFSET: int = get_from_env("PATCH_EVENT_LIST_MAX_OFFSET", 0, type_cast=int)
+PATCH_EVENT_LIST_MAX_OFFSET_PER_TEAM: set[int] = get_from_env(
+    "PATCH_EVENT_LIST_MAX_OFFSET_PER_TEAM", default=set[int]([]), type_cast=str_to_int_set
+)

posthog/test/test_utils.py

@@ -13,6 +13,7 @@
 from django.test import TestCase
 from django.test.client import RequestFactory
 
+from parameterized import parameterized
 from rest_framework.request import Request
 
 from posthog.exceptions import RequestParsingError, UnspecifiedCompressionFallbackParsingError
@@ -31,6 +32,7 @@
     load_data_from_request,
     refresh_requested_by_client,
     relative_date_parse,
+    str_to_int_set,
 )
 
 
@@ -588,3 +590,20 @@ def create_group_type_mapping_without_created_at(**kwargs) -> "GroupTypeMapping"
     GroupTypeMapping.objects.filter(id=instance.id).update(created_at=None)
     instance.refresh_from_db()
     return instance
+
+
+class TestStrToIntSet(TestCase):
+    @parameterized.expand(
+        [
+            (None, set()),
+            ("", set()),
+            ("[]", set()),
+            ("[1, 2, 3]", {1, 2, 3}),
+            ("[1, 1, 2]", {1, 2}),
+            ('["1", "2"]', {1, 2}),
+            ("invalid", set()),
+            ("123", set()),
+        ]
+    )
+    def test_str_to_int_set(self, value, expected):
+        assert str_to_int_set(value) == expected

posthog/utils.py

@@ -14,7 +14,7 @@
 import datetime as dt
 import dataclasses
 from collections.abc import Callable, Generator, Mapping, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from enum import Enum
 from functools import lru_cache, wraps
 from operator import itemgetter
@@ -1287,6 +1287,16 @@ def _request_has_key_set(key: str, request: Request, allowed_values: Optional[li
     return False
 
 
+def str_to_int_set(value: Any) -> set[int]:
+    """Return a set of integers"""
+    if not value:
+        return set[int]([])
+    with suppress(Exception):
+        as_json = json.loads(str(value))
+        return {int(v) for v in as_json}
+    return set[int]([])
+
+
 def str_to_bool(value: Any) -> bool:
     """Return whether the provided string (or any value really) represents true. Otherwise, false.
     Just like plugin server stringToBoolean.