feat(llm-analytics): add early adopters feature flag (#42101)

Co-authored-by: Claude <[email protected]>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

Author: 3 people

frontend/src/lib/constants.tsx

@@ -197,6 +197,7 @@ export const FEATURE_FLAGS = {
     QUERY_CACHE_USE_S3: 'query-cache-use-s3', // owner: @aspicer #team-product-analytics
     DASHBOARD_THREADS: 'dashboard-threads', // owner: @aspicer #team-product-analytics
     BATCH_EXPORTS_POSTHOG_HTTP: 'posthog-http-batch-exports',
+    LLM_ANALYTICS_EARLY_ADOPTERS: 'llm-analytics-early-adopters', // owner: #team-llm-analytics
     LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD: 'llm-analytics-customizable-dashboard', // owner: #team-llm-analytics
     LLM_ANALYTICS_DISCUSSIONS: 'llm-analytics-discussions', // owner: #team-llm-analytics
     BATCH_EXPORTS_DATABRICKS: 'databricks-batch-exports', // owner: @rossgray #team-batch-exports

products/llm_analytics/backend/api/summarization.py

@@ -29,7 +29,10 @@
 from posthog.models import User
 from posthog.rate_limit import ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle
 
-from products.llm_analytics.backend.summarization.constants import SUMMARIZATION_FEATURE_FLAG
+from products.llm_analytics.backend.summarization.constants import (
+    EARLY_ADOPTERS_FEATURE_FLAG,
+    SUMMARIZATION_FEATURE_FLAG,
+)
 from products.llm_analytics.backend.summarization.llm import summarize
 from products.llm_analytics.backend.text_repr.formatters import (
     FormatterOptions,
@@ -115,9 +118,10 @@ def _validate_feature_access(self, request: Request) -> None:
             return
 
         # Check feature flag using user's distinct_id to match against person-based cohorts
-        if not posthoganalytics.feature_enabled(
-            SUMMARIZATION_FEATURE_FLAG,
-            str(request.user.distinct_id),
+        distinct_id = str(request.user.distinct_id)
+        if not (
+            posthoganalytics.feature_enabled(SUMMARIZATION_FEATURE_FLAG, distinct_id)
+            or posthoganalytics.feature_enabled(EARLY_ADOPTERS_FEATURE_FLAG, distinct_id)
         ):
             raise exceptions.PermissionDenied("LLM trace summarization is not enabled for this user")
 

products/llm_analytics/backend/summarization/constants.py

@@ -4,5 +4,6 @@
 SUMMARIZATION_MODEL = "gpt-4.1-mini"
 SUMMARIZATION_TIMEOUT = 120  # 2 minutes
 
-# Feature flag
+# Feature flags
 SUMMARIZATION_FEATURE_FLAG = "llm-analytics-summarization"
+EARLY_ADOPTERS_FEATURE_FLAG = "llm-analytics-early-adopters"

products/llm_analytics/frontend/LLMAnalyticsReloadAction.tsx

@@ -23,7 +23,9 @@ export function LLMAnalyticsReloadAction(): JSX.Element {
     const { activeTab, selectedDashboardId, isRefreshing: oldTilesRefreshing } = useValues(llmAnalyticsLogic)
     const { refreshAllDashboardItems } = useActions(llmAnalyticsLogic)
 
-    const useCustomizableDashboard = featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD]
+    const useCustomizableDashboard =
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD] ||
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
 
     const shouldUseDashboardLogic = selectedDashboardId && useCustomizableDashboard && activeTab === 'dashboard'
     const dashboardLogicInstance = dashboardLogic({

products/llm_analytics/frontend/LLMAnalyticsScene.tsx

@@ -91,7 +91,9 @@ const Filters = ({ hidePropertyFilters = false }: { hidePropertyFilters?: boolea
     const dateFrom = activeTab === 'dashboard' ? dashboardDateFilter.dateFrom : dateFilter.dateFrom
     const dateTo = activeTab === 'dashboard' ? dashboardDateFilter.dateTo : dateFilter.dateTo
 
-    const useCustomizableDashboard = featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD]
+    const useCustomizableDashboard =
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD] ||
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
 
     return (
         <div className="flex gap-x-4 gap-y-2 items-center flex-wrap py-4 -mt-4 mb-4 border-b">
@@ -152,7 +154,9 @@ function LLMAnalyticsDashboard(): JSX.Element {
     const { selectedDashboardId, availableDashboardsLoading, dashboardDateFilter, propertyFilters } =
         useValues(llmAnalyticsLogic)
 
-    const useCustomizableDashboard = featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD]
+    const useCustomizableDashboard =
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD] ||
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
     const dashboardLogicInstance = React.useMemo(
         () =>
             selectedDashboardId
@@ -656,7 +660,10 @@ export function LLMAnalyticsScene(): JSX.Element {
         },
     ]
 
-    if (featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_ERRORS_TAB]) {
+    if (
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_ERRORS_TAB] ||
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
+    ) {
         tabs.push({
             key: 'errors',
             label: (
@@ -677,7 +684,10 @@ export function LLMAnalyticsScene(): JSX.Element {
         })
     }
 
-    if (featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SESSIONS_VIEW]) {
+    if (
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SESSIONS_VIEW] ||
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
+    ) {
         tabs.push({
             key: 'sessions',
             label: (

products/llm_analytics/frontend/LLMAnalyticsTraceScene.tsx

@@ -123,7 +123,8 @@ function TraceSceneWrapper(): JSX.Element {
                         />
                         <div className="flex flex-wrap justify-end items-center gap-x-2 gap-y-1">
                             <DisplayOptionsSelect />
-                            {featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_DISCUSSIONS] && (
+                            {(featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_DISCUSSIONS] ||
+                                featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]) && (
                                 <LemonButton
                                     type="secondary"
                                     size="xsmall"
@@ -212,7 +213,10 @@ function TraceMetadata({
     const { featureFlags } = useValues(featureFlagLogic)
 
     const getSessionUrl = (sessionId: string): string => {
-        if (featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SESSIONS_VIEW]) {
+        if (
+            featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SESSIONS_VIEW] ||
+            featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
+        ) {
             return urls.llmAnalyticsSession(sessionId)
         }
         // Fallback to filtering traces by session when feature flag is off
@@ -239,7 +243,8 @@ function TraceMetadata({
             {trace.aiSessionId && (
                 <Chip
                     title={
-                        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SESSIONS_VIEW]
+                        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SESSIONS_VIEW] ||
+                        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
                             ? 'AI Session ID - Click to view session details'
                             : 'AI Session ID - Click to filter traces by this session'
                     }
@@ -685,7 +690,9 @@ const EventContent = React.memo(
 
         const showEvalsTab = isGenerationEvent && featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EVALUATIONS]
 
-        const showSummaryTab = featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SUMMARIZATION]
+        const showSummaryTab =
+            featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SUMMARIZATION] ||
+            featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
 
         // Load AI data for the current event
         const { input: loadedInput, output: loadedOutput } = useAIData(event)
@@ -811,7 +818,8 @@ const EventContent = React.memo(
                                     content: (
                                         <>
                                             {displayOption === DisplayOption.TextView &&
-                                            featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_TEXT_VIEW] ? (
+                                            (featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_TEXT_VIEW] ||
+                                                featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]) ? (
                                                 isLLMEvent(event) &&
                                                 (event.event === '$ai_generation' ||
                                                     event.event === '$ai_span' ||
@@ -1024,7 +1032,8 @@ function DisplayOptionsSelect(): JSX.Element {
             label: 'Collapse except output and last input',
             tooltip: 'Focus on the most recent input and final output',
         },
-        ...(featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_TEXT_VIEW]
+        ...(featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_TEXT_VIEW] ||
+        featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
             ? [
                   {
                       value: DisplayOption.TextView,

products/llm_analytics/frontend/llmAnalyticsLogic.tsx

@@ -1382,7 +1382,10 @@ export const llmAnalyticsLogic = kea<llmAnalyticsLogicType>([
     afterMount(({ actions, values }) => {
         actions.loadAIEventDefinition()
 
-        if (values.featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD]) {
+        if (
+            values.featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_CUSTOMIZABLE_DASHBOARD] ||
+            values.featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
+        ) {
             actions.loadLLMDashboards()
         }
     }),

products/llm_analytics/frontend/llmAnalyticsTraceLogic.ts

@@ -177,7 +177,13 @@ export const llmAnalyticsTraceLogic = kea<llmAnalyticsTraceLogicType>([
             0,
             {
                 loadCommentCount: async (_, breakpoint) => {
-                    if (!values.traceId || !values.featureFlags?.[FEATURE_FLAGS.LLM_ANALYTICS_DISCUSSIONS]) {
+                    if (
+                        !values.traceId ||
+                        !(
+                            values.featureFlags?.[FEATURE_FLAGS.LLM_ANALYTICS_DISCUSSIONS] ||
+                            values.featureFlags?.[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
+                        )
+                    ) {
                         return 0
                     }
 
@@ -262,7 +268,10 @@ export const llmAnalyticsTraceLogic = kea<llmAnalyticsTraceLogicType>([
                 return {
                     activity_scope: ActivityScope.LLM_TRACE,
                     activity_item_id: traceId || '',
-                    discussions_disabled: !featureFlags?.[FEATURE_FLAGS.LLM_ANALYTICS_DISCUSSIONS],
+                    discussions_disabled: !(
+                        featureFlags?.[FEATURE_FLAGS.LLM_ANALYTICS_DISCUSSIONS] ||
+                        featureFlags?.[FEATURE_FLAGS.LLM_ANALYTICS_EARLY_ADOPTERS]
+                    ),
                     activity_item_context: { trace_id: traceId || '' },
                 }
             },