feat(ai): Full session replay mode

Author: Twixes

ee/hogai/core/agent_modes/presets/session_replay.py

@@ -2,7 +2,7 @@
 
 from posthog.schema import AgentMode
 
-from ee.hogai.tools.replay.summarize_sessions import SummarizeSessionsTool
+from ee.hogai.tools.replay.summarize_sessions import FilterSessionRecordingsTool, SummarizeSessionsTool
 
 from ..factory import AgentModeDefinition
 from ..toolkit import AgentToolkit
@@ -14,12 +14,12 @@
 class SessionReplayAgentToolkit(AgentToolkit):
     @property
     def tools(self) -> list[type["MaxTool"]]:
-        tools: list[type[MaxTool]] = [SummarizeSessionsTool]
+        tools: list[type[MaxTool]] = [FilterSessionRecordingsTool, SummarizeSessionsTool]
         return tools
 
 
 session_replay_agent = AgentModeDefinition(
     mode=AgentMode.SESSION_REPLAY,
-    mode_description="Specialized mode for analyzing session recordings and user behavior. This mode allows you to get summaries of session recordings and insights about them in natural language.",
+    mode_description="Specialized mode for analyzing session recordings and user behavior. This mode allows you to filter session recordings, and summarize entire sessions or a set of them.",
     toolkit_class=SessionReplayAgentToolkit,
 )

ee/hogai/tools/replay/summarize_sessions.py

@@ -1881,6 +1881,7 @@
                 "execute_sql",
                 "switch_mode",
                 "summarize_sessions",
+                "filter_session_recordings",
                 "create_insight"
             ],
             "type": "string"

frontend/src/queries/schema.json

@@ -294,9 +294,11 @@ export type AssistantTool =
     | 'filter_web_analytics'
     | 'create_feature_flag'
     | 'create_experiment'
+    // Below are modes-only
     | 'execute_sql'
     | 'switch_mode'
     | 'summarize_sessions'
+    | 'filter_session_recordings'
     | 'create_insight'
 
 export enum AgentMode {

frontend/src/queries/schema/schema-assistant-messages.ts

@@ -702,6 +702,7 @@ function AssistantActionComponent({
     icon,
     animate = true,
     showCompletionIcon = true,
+    widget = null,
 }: {
     id: string
     content: string
@@ -710,6 +711,7 @@ function AssistantActionComponent({
     icon?: React.ReactNode
     animate?: boolean
     showCompletionIcon?: boolean
+    widget?: JSX.Element | null
 }): JSX.Element {
     const isPending = state === 'pending'
     const isCompleted = state === 'completed'
@@ -801,6 +803,7 @@ function AssistantActionComponent({
                     })}
                 </div>
             )}
+            {widget}
         </div>
     )
 }
@@ -867,9 +870,18 @@ function ToolCallsAnswer({ toolCalls, registeredToolMap }: ToolCallsAnswerProps)
                         const updates = toolCall.updates ?? []
                         const definition = getToolDefinition(toolCall.name)
                         let description = `Executing ${toolCall.name}`
+                        let widget: JSX.Element | null = null
                         if (definition) {
                             if (definition.displayFormatter) {
-                                description = definition.displayFormatter(toolCall, { registeredToolMap })
+                                const displayFormatterResult = definition.displayFormatter(toolCall, {
+                                    registeredToolMap,
+                                })
+                                if (typeof displayFormatterResult === 'string') {
+                                    description = displayFormatterResult
+                                } else {
+                                    description = displayFormatterResult[0]
+                                    widget = displayFormatterResult[1]
+                                }
                             }
                             if (commentary) {
                                 description = commentary
@@ -884,6 +896,7 @@ function ToolCallsAnswer({ toolCalls, registeredToolMap }: ToolCallsAnswerProps)
                                 state={toolCall.status}
                                 icon={definition?.icon || <IconWrench />}
                                 showCompletionIcon={true}
+                                widget={widget}
                             />
                         )
                     })}

frontend/src/scenes/max/Thread.tsx

@@ -5,9 +5,11 @@ import { Scene } from 'scenes/sceneTypes'
 
 import { iconForType } from '~/layout/panel-layout/ProjectTree/defaultTree'
 import { AgentMode, AssistantTool } from '~/queries/schema/schema-assistant-messages'
+import { RecordingUniversalFilters } from '~/types'
 
 import { EnhancedToolCall } from './Thread'
 import { isAgentMode } from './maxTypes'
+import { RecordingsWidget } from './messages/UIPayloadAnswer'
 
 /** Static tool definition for display purposes. */
 export interface ToolDefinition<N extends string = string> {
@@ -29,7 +31,7 @@ export interface ToolDefinition<N extends string = string> {
     displayFormatter?: (
         toolCall: EnhancedToolCall,
         { registeredToolMap }: { registeredToolMap: Record<string, ToolRegistration> }
-    ) => string
+    ) => string | [text: string, widget: JSX.Element | null]
     /**
      * If only available in a specific product, specify it here.
      * We're using Scene instead of ProductKey, because that's more flexible (specifically for SQL editor there
@@ -212,6 +214,25 @@ export const TOOL_DEFINITIONS: Record<Exclude<AssistantTool, 'todo_write'>, Tool
             return 'Searching recordings...'
         },
     },
+    filter_session_recordings: {
+        name: 'Filter recordings',
+        description: 'Filter recordings to find the most relevant ones',
+        product: Scene.Replay,
+        icon: iconForType('session_replay'),
+        displayFormatter: (toolCall) => {
+            const widget = toolCall.args?.recordings_filters ? (
+                <RecordingsWidget
+                    toolCallId={toolCall.id}
+                    filters={toolCall.args.recordings_filters as RecordingUniversalFilters}
+                />
+            ) : null
+
+            if (toolCall.status === 'completed') {
+                return ['Filtered recordings', widget]
+            }
+            return ['Filtering recordings...', widget]
+        },
+    },
     generate_hogql_query: {
         name: 'Write and tweak SQL',
         description: 'Write and tweak SQL right there',

frontend/src/scenes/max/max-constants.tsx

@@ -35,7 +35,7 @@ export function UIPayloadAnswer({
     return null
 }
 
-function RecordingsWidget({
+export function RecordingsWidget({
     toolCallId,
     filters,
 }: {

frontend/src/scenes/max/messages/UIPayloadAnswer.tsx

@@ -295,6 +295,7 @@ class AssistantTool(StrEnum):
     EXECUTE_SQL = "execute_sql"
     SWITCH_MODE = "switch_mode"
     SUMMARIZE_SESSIONS = "summarize_sessions"
+    FILTER_SESSION_RECORDINGS = "filter_session_recordings"
     CREATE_INSIGHT = "create_insight"