Chat various improvements (#3942)

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: yujonglee

Cargo.lock

@@ -66,6 +66,7 @@ hypr-host = { path = "crates/host", package = "host" }
 hypr-http = { path = "crates/http", package = "hypr-http-utils" }
 hypr-importer-core = { path = "crates/importer-core", package = "importer-core" }
 hypr-intercept = { path = "crates/intercept", package = "intercept" }
+hypr-jina = { path = "crates/jina", package = "jina" }
 hypr-kyutai = { path = "crates/kyutai", package = "kyutai" }
 hypr-language = { path = "crates/language", package = "language" }
 hypr-llama = { path = "crates/llama", package = "llama" }
@@ -98,6 +99,7 @@ hypr-tcc = { path = "crates/tcc", package = "tcc" }
 hypr-template-app = { path = "crates/template-app", package = "template-app" }
 hypr-template-app-legacy = { path = "crates/template-app-legacy", package = "template-app-legacy" }
 hypr-template-eval = { path = "crates/template-eval", package = "template-eval" }
+hypr-template-support = { path = "crates/template-support", package = "template-support" }
 hypr-tiptap = { path = "crates/tiptap", package = "tiptap" }
 hypr-transcribe-aws = { path = "crates/transcribe-aws", package = "transcribe-aws" }
 hypr-transcribe-azure = { path = "crates/transcribe-azure", package = "transcribe-azure" }

Cargo.toml

@@ -9,6 +9,7 @@ hypr-api-auth = { workspace = true }
 hypr-api-calendar = { workspace = true }
 hypr-api-env = { workspace = true }
 hypr-api-nango = { workspace = true }
+hypr-api-research = { workspace = true }
 hypr-api-subscription = { workspace = true }
 hypr-api-support = { workspace = true }
 hypr-llm-proxy = { workspace = true }

apps/api/Cargo.toml

@@ -466,6 +466,18 @@
           "arch": {
             "type": "string"
           },
+          "buildHash": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "locale": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
           "osVersion": {
             "type": "string"
           },

apps/api/openapi.gen.json

@@ -27,6 +27,9 @@ pub struct Env {
     #[serde(flatten)]
     pub support_database: hypr_api_support::SupportDatabaseEnv,
 
+    pub exa_api_key: String,
+    pub jina_api_key: String,
+
     #[serde(flatten)]
     pub llm: hypr_llm_proxy::Env,
     #[serde(flatten)]

apps/api/src/env.rs

@@ -54,6 +54,10 @@ async fn app() -> Router {
         &env.supabase,
         auth_state_support.clone(),
     );
+    let research_config = hypr_api_research::ResearchConfig {
+        exa_api_key: env.exa_api_key.clone(),
+        jina_api_key: env.jina_api_key.clone(),
+    };
 
     let webhook_routes = Router::new().nest(
         "/nango",
@@ -63,6 +67,7 @@ async fn app() -> Router {
     let pro_routes = Router::new()
         .merge(hypr_transcribe_proxy::listen_router(stt_config.clone()))
         .merge(hypr_llm_proxy::chat_completions_router(llm_config.clone()))
+        .merge(hypr_api_research::router(research_config))
         .nest("/stt", hypr_transcribe_proxy::router(stt_config))
         .nest("/llm", hypr_llm_proxy::router(llm_config))
         .nest("/calendar", hypr_api_calendar::router(calendar_config))

apps/api/src/main.rs

@@ -1,12 +1,112 @@
-export type ContextItem = {
-  key: string;
-  label: string;
-  tooltip: string;
+import type { AccountInfo } from "@hypr/plugin-auth";
+import type { DeviceInfo } from "@hypr/plugin-misc";
+import type { ChatContext } from "@hypr/plugin-template";
+
+import type { HyprUIMessage } from "./types";
+import { isRecord } from "./utils";
+
+export type ContextEntity =
+  | {
+      kind: "session";
+      key: string;
+      chatContext: ChatContext;
+      wordCount?: number;
+      rawNotePreview?: string;
+      participantCount?: number;
+      eventTitle?: string;
+      removable?: boolean;
+    }
+  | ({ kind: "account"; key: string } & Partial<AccountInfo>)
+  | ({
+      kind: "device";
+      key: string;
+    } & Partial<DeviceInfo>);
+
+export type ContextEntityKind = ContextEntity["kind"];
+
+type ToolOutputAvailablePart = {
+  type: string;
+  state: "output-available";
+  output?: unknown;
+};
+
+function isToolOutputAvailablePart(
+  value: unknown,
+): value is ToolOutputAvailablePart {
+  return (
+    isRecord(value) &&
+    typeof value.type === "string" &&
+    value.state === "output-available"
+  );
+}
+
+function parseSearchSessionsOutput(output: unknown): ContextEntity[] {
+  if (!isRecord(output) || !Array.isArray(output.results)) {
+    return [];
+  }
+
+  return output.results.flatMap((item): ContextEntity[] => {
+    if (!isRecord(item)) {
+      return [];
+    }
+
+    if (typeof item.id !== "string" && typeof item.id !== "number") {
+      return [];
+    }
+
+    const title = typeof item.title === "string" ? item.title : null;
+    const content = typeof item.content === "string" ? item.content : null;
+
+    return [
+      {
+        kind: "session",
+        key: `session:search:${item.id}`,
+        chatContext: {
+          title,
+          date: null,
+          rawContent: content,
+          enhancedContent: null,
+          transcript: null,
+        },
+        rawNotePreview: content?.slice(0, 120) ?? undefined,
+        removable: true,
+      },
+    ];
+  });
+}
+
+const toolEntityExtractors: Record<
+  string,
+  (output: unknown) => ContextEntity[]
+> = {
+  search_sessions: parseSearchSessionsOutput,
 };
 
-export type ContextSource =
-  | { type: "account"; email?: string; userId?: string }
-  | { type: "device" }
-  | { type: "session"; title?: string; date?: string }
-  | { type: "transcript"; wordCount?: number }
-  | { type: "note"; preview?: string };
+export function extractToolContextEntities(
+  messages: Array<Pick<HyprUIMessage, "parts">>,
+): ContextEntity[] {
+  const seen = new Set<string>();
+  const entities: ContextEntity[] = [];
+
+  for (const message of messages) {
+    if (!Array.isArray(message.parts)) continue;
+    for (const part of message.parts) {
+      if (!isToolOutputAvailablePart(part) || !part.type.startsWith("tool-")) {
+        continue;
+      }
+
+      const toolName = part.type.slice(5);
+      const extractor = toolEntityExtractors[toolName];
+      if (!extractor) continue;
+
+      for (const entity of extractor(part.output)) {
+        if (!seen.has(entity.key)) {
+          seen.add(entity.key);
+          entities.push(entity);
+        }
+      }
+    }
+  }
+
+  return entities;
+}

apps/desktop/src/chat/context-item.ts

@@ -0,0 +1,18 @@
+import type { ContextEntity } from "../context-item";
+
+export function composeContextEntities(
+  groups: ContextEntity[][],
+): ContextEntity[] {
+  const seen = new Set<string>();
+  const merged: ContextEntity[] = [];
+
+  for (const group of groups) {
+    for (const entity of group) {
+      if (seen.has(entity.key)) continue;
+      seen.add(entity.key);
+      merged.push(entity);
+    }
+  }
+
+  return merged;
+}

apps/desktop/src/chat/context/composer.ts

@@ -0,0 +1,28 @@
+import { commands as miscCommands } from "@hypr/plugin-misc";
+
+import type { ContextEntity } from "../context-item";
+
+export async function collectDeviceEntity(): Promise<
+  Extract<ContextEntity, { kind: "device" }>
+> {
+  let deviceContext: Extract<ContextEntity, { kind: "device" }> = {
+    kind: "device",
+    key: "support:device",
+  };
+
+  try {
+    const deviceContextResult = await miscCommands.getDeviceInfo(
+      navigator.language || "en",
+    );
+    if (deviceContextResult.status === "ok") {
+      deviceContext = {
+        ...deviceContext,
+        ...deviceContextResult.data,
+      };
+    }
+  } catch (error) {
+    console.error("Failed to collect device context:", error);
+  }
+
+  return deviceContext;
+}