feat: optimize queries, add missing indeces

geclos · geclos · commit 1e956813dcb7 · 2026-02-16T13:04:09.000+01:00
diff --git a/apps/web/src/app/(admin)/backoffice/usage-overview/_components/UsageCell/index.tsx b/apps/web/src/app/(admin)/backoffice/usage-overview/_components/UsageCell/index.tsx
@@ -6,8 +6,8 @@ export function UsageCell({
 }: {
   usageOverview: GetUsageOverviewRow
 }) {
-  const lastMonth = usageOverview.lastMonthTraces ?? 0
-  const prevMonth = usageOverview.lastTwoMonthsTraces ?? 0
+  const lastMonth = usageOverview.lastMonthRuns ?? 0
+  const prevMonth = usageOverview.lastTwoMonthsRuns ?? 0
 
   return (
     <div className='flex flex-col gap-1'>
diff --git a/apps/web/src/app/(admin)/backoffice/usage-overview/buildUsageInformation.ts b/apps/web/src/app/(admin)/backoffice/usage-overview/buildUsageInformation.ts
@@ -54,8 +54,8 @@ function getEmailsList(emails: string | null) {
 export function buildUsageInformation(usage: GetUsageOverviewRow) {
   const plan = getPlanFromSubscriptionSlug(usage.subscriptionPlan)
   const trend = getTrendIndicator({
-    tracesTwoMonthsAgo: usage.lastTwoMonthsTraces,
-    tracesLast30Days: usage.lastMonthTraces,
+    tracesTwoMonthsAgo: usage.lastTwoMonthsRuns,
+    tracesLast30Days: usage.lastMonthRuns,
     plan,
   })
 
diff --git a/apps/web/src/app/(admin)/backoffice/usage-overview/page.tsx b/apps/web/src/app/(admin)/backoffice/usage-overview/page.tsx
@@ -88,8 +88,8 @@ export default async function AdminPage({
                         </ServerSideTableCell>
                         <ServerSideTableCell className='w-40'>
                           <Text.H6 color='foregroundMuted'>
-                            {usage.latestTraceAt
-                              ? formatDistanceToNow(usage.latestTraceAt, {
+                            {usage.latestRunAt
+                              ? formatDistanceToNow(usage.latestRunAt, {
                                   addSuffix: true,
                                 })
                               : '-'}
diff --git a/packages/core/clickhouse/migrations/clustered/0004_add_spans_secondary_indexes.down.sql b/packages/core/clickhouse/migrations/clustered/0004_add_spans_secondary_indexes.down.sql
@@ -0,0 +1,3 @@
+ALTER TABLE spans ON CLUSTER default DROP INDEX IF EXISTS idx_test_deployment_id;
+ALTER TABLE spans ON CLUSTER default DROP INDEX IF EXISTS idx_parent_id;
+ALTER TABLE spans ON CLUSTER default DROP INDEX IF EXISTS idx_experiment_uuid;
diff --git a/packages/core/clickhouse/migrations/clustered/0004_add_spans_secondary_indexes.up.sql b/packages/core/clickhouse/migrations/clustered/0004_add_spans_secondary_indexes.up.sql
@@ -0,0 +1,8 @@
+ALTER TABLE spans ON CLUSTER default
+  ADD INDEX IF NOT EXISTS idx_experiment_uuid experiment_uuid TYPE bloom_filter(0.01) GRANULARITY 1,
+  ADD INDEX IF NOT EXISTS idx_parent_id parent_id TYPE bloom_filter(0.01) GRANULARITY 1,
+  ADD INDEX IF NOT EXISTS idx_test_deployment_id test_deployment_id TYPE bloom_filter(0.01) GRANULARITY 1;
+
+ALTER TABLE spans ON CLUSTER default MATERIALIZE INDEX idx_experiment_uuid;
+ALTER TABLE spans ON CLUSTER default MATERIALIZE INDEX idx_parent_id;
+ALTER TABLE spans ON CLUSTER default MATERIALIZE INDEX idx_test_deployment_id;
diff --git a/packages/core/clickhouse/migrations/unclustered/0004_add_spans_secondary_indexes.down.sql b/packages/core/clickhouse/migrations/unclustered/0004_add_spans_secondary_indexes.down.sql
@@ -0,0 +1,3 @@
+ALTER TABLE spans DROP INDEX IF EXISTS idx_test_deployment_id;
+ALTER TABLE spans DROP INDEX IF EXISTS idx_parent_id;
+ALTER TABLE spans DROP INDEX IF EXISTS idx_experiment_uuid;
diff --git a/packages/core/clickhouse/migrations/unclustered/0004_add_spans_secondary_indexes.up.sql b/packages/core/clickhouse/migrations/unclustered/0004_add_spans_secondary_indexes.up.sql
@@ -0,0 +1,8 @@
+ALTER TABLE spans
+  ADD INDEX IF NOT EXISTS idx_experiment_uuid experiment_uuid TYPE bloom_filter(0.01) GRANULARITY 1,
+  ADD INDEX IF NOT EXISTS idx_parent_id parent_id TYPE bloom_filter(0.01) GRANULARITY 1,
+  ADD INDEX IF NOT EXISTS idx_test_deployment_id test_deployment_id TYPE bloom_filter(0.01) GRANULARITY 1;
+
+ALTER TABLE spans MATERIALIZE INDEX idx_experiment_uuid;
+ALTER TABLE spans MATERIALIZE INDEX idx_parent_id;
+ALTER TABLE spans MATERIALIZE INDEX idx_test_deployment_id;
diff --git a/packages/core/src/queries/clickhouse/spans/computeDocumentTracesAggregations.ts b/packages/core/src/queries/clickhouse/spans/computeDocumentTracesAggregations.ts
@@ -14,74 +14,73 @@ export async function computeDocumentTracesAggregations({
   const params: Record<string, unknown> = {
     workspaceId,
     documentUuid,
+    completionType: 'completion',
   }
 
-  const countConditions = [
+  const conditions = [
     `workspace_id = {workspaceId: UInt64}`,
-    `document_uuid = {documentUuid: String}`,
-  ]
-
-  const aggregationConditions = [
-    `workspace_id = {workspaceId: UInt64}`,
-    `document_uuid = {documentUuid: String}`,
-    `type = 'completion'`,
+    `document_uuid = {documentUuid: UUID}`,
   ]
 
   if (commitUuid) {
     params.commitUuid = commitUuid
-    countConditions.push(`commit_uuid = {commitUuid: String}`)
-    aggregationConditions.push(`commit_uuid = {commitUuid: String}`)
+    conditions.push(`commit_uuid = {commitUuid: UUID}`)
   }
 
-  // Query for total count of distinct traces
-  const countResult = await clickhouseClient().query({
-    query: `
-      SELECT count(DISTINCT trace_id) AS total_count
-      FROM ${SPANS_TABLE}
-      WHERE ${countConditions.join(' AND ')}
-    `,
-    format: 'JSONEachRow',
-    query_params: params,
-  })
-
-  const countRows = await countResult.json<{ total_count: string }>()
-  const totalCount = Number(countRows[0]?.total_count ?? 0)
-
-  // Query for aggregations with median calculations using quantile(0.5)
+  // Query all metrics in a single pass
   const aggregationResult = await clickhouseClient().query({
     query: `
       SELECT
+        countDistinct(trace_id) AS total_count,
         coalesce(
-          sum(
+          sumIf(
             coalesce(tokens_prompt, 0) +
             coalesce(tokens_cached, 0) +
             coalesce(tokens_reasoning, 0) +
-            coalesce(tokens_completion, 0)
+            coalesce(tokens_completion, 0),
+            type = {completionType: String}
           ),
           0
         ) AS total_tokens,
-        coalesce(sum(cost), 0) AS total_cost_in_millicents,
         coalesce(
-          avg(
+          sumIf(cost, type = {completionType: String}),
+          0
+        ) AS total_cost_in_millicents,
+        coalesce(
+          avgIf(
             coalesce(tokens_prompt, 0) +
             coalesce(tokens_cached, 0) +
             coalesce(tokens_reasoning, 0) +
-            coalesce(tokens_completion, 0)
+            coalesce(tokens_completion, 0),
+            type = {completionType: String}
           ),
           0
         ) AS average_tokens,
-        coalesce(avg(cost), 0) AS average_cost_in_millicents,
-        coalesce(quantile(0.5)(cost), 0) AS median_cost_in_millicents,
-        coalesce(avg(duration_ms), 0) AS average_duration,
-        coalesce(quantile(0.5)(duration_ms), 0) AS median_duration
+        coalesce(
+          avgIf(cost, type = {completionType: String}),
+          0
+        ) AS average_cost_in_millicents,
+        coalesce(
+          quantileIf(0.5)(cost, type = {completionType: String}),
+          0
+        ) AS median_cost_in_millicents,
+        coalesce(
+          avgIf(duration_ms, type = {completionType: String}),
+          0
+        ) AS average_duration,
+        coalesce(
+          quantileIf(0.5)(duration_ms, type = {completionType: String}),
+          0
+        ) AS median_duration
       FROM ${SPANS_TABLE}
-      WHERE ${aggregationConditions.join(' AND ')}
+      WHERE ${conditions.join(' AND ')}
     `,
     format: 'JSONEachRow',
     query_params: params,
   })
 
   const aggRows = await aggregationResult.json<{
+    total_count: string
     total_tokens: string
     total_cost_in_millicents: string
     average_tokens: string
@@ -92,6 +91,7 @@ export async function computeDocumentTracesAggregations({
   }>()
 
   const row = aggRows[0] ?? {
+    total_count: '0',
     total_tokens: '0',
     total_cost_in_millicents: '0',
     average_tokens: '0',
@@ -102,7 +102,7 @@ export async function computeDocumentTracesAggregations({
   }
 
   return {
-    totalCount,
+    totalCount: Number(row.total_count),
     totalTokens: Number(row.total_tokens),
     totalCostInMillicents: Number(row.total_cost_in_millicents),
     averageTokens: Number(row.average_tokens),
diff --git a/packages/core/src/queries/clickhouse/spans/computeDocumentTracesDailyCount.ts b/packages/core/src/queries/clickhouse/spans/computeDocumentTracesDailyCount.ts
@@ -30,13 +30,13 @@ export async function computeDocumentTracesDailyCount({
 
   const conditions = [
     `workspace_id = {workspaceId: UInt64}`,
-    `document_uuid = {documentUuid: String}`,
+    `document_uuid = {documentUuid: UUID}`,
     `started_at >= {startDate: DateTime64(6, 'UTC')}`,
   ]
 
   if (commitUuid) {
     params.commitUuid = commitUuid
-    conditions.push(`commit_uuid = {commitUuid: String}`)
+    conditions.push(`commit_uuid = {commitUuid: UUID}`)
   }
 
   const result = await clickhouseClient().query({
diff --git a/packages/core/src/queries/clickhouse/spans/countByDocument.ts b/packages/core/src/queries/clickhouse/spans/countByDocument.ts
@@ -35,8 +35,8 @@ export async function countDistinctTracesByDocument({
       SELECT count(DISTINCT trace_id) AS cnt
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_uuid = {documentUuid: String}
-        AND commit_uuid IN ({commitUuids: Array(String)})
+        AND document_uuid = {documentUuid: UUID}
+        AND commit_uuid IN ({commitUuids: Array(UUID)})
         ${sourceFilter}
     `,
     format: 'JSONEachRow',
diff --git a/packages/core/src/queries/clickhouse/spans/fetchConversation.ts b/packages/core/src/queries/clickhouse/spans/fetchConversation.ts
@@ -40,13 +40,13 @@ export async function fetchConversation({
 
   const conditions = [
     `workspace_id = {workspaceId: UInt64}`,
-    `document_log_uuid = {documentLogUuid: String}`,
+    `document_log_uuid = {documentLogUuid: UUID}`,
     `type IN ({mainSpanTypes: Array(String)})`,
   ]
 
   if (documentUuid) {
     params.documentUuid = documentUuid
-    conditions.push(`document_uuid = {documentUuid: String}`)
+    conditions.push(`document_uuid = {documentUuid: UUID}`)
   }
 
   const result = await clickhouseClient().query({
diff --git a/packages/core/src/queries/clickhouse/spans/fetchConversations.ts b/packages/core/src/queries/clickhouse/spans/fetchConversations.ts
@@ -64,15 +64,15 @@ export async function fetchConversations({
 
   const conditions = [
     `workspace_id = {workspaceId: UInt64}`,
-    `document_uuid = {documentUuid: String}`,
-    `commit_uuid IN ({commitUuids: Array(String)})`,
+    `document_uuid = {documentUuid: UUID}`,
+    `commit_uuid IN ({commitUuids: Array(UUID)})`,
     `type IN ({mainSpanTypes: Array(String)})`,
     `document_log_uuid IS NOT NULL`,
   ]
 
   if (filters.experimentUuids && filters.experimentUuids.length > 0) {
     params.experimentUuids = filters.experimentUuids
-    conditions.push(`experiment_uuid IN ({experimentUuids: Array(String)})`)
+    conditions.push(`experiment_uuid IN ({experimentUuids: Array(UUID)})`)
   }
 
   if (filters.testDeploymentIds && filters.testDeploymentIds.length > 0) {
@@ -93,10 +93,10 @@ export async function fetchConversations({
   }
 
   if (from) {
-    params.cursorStartedAt = from.startedAt
+    params.cursorStartedAt = toClickHouseDateTime(new Date(from.startedAt))
     params.cursorDocumentLogUuid = from.documentLogUuid
     conditions.push(
-      `(started_at, document_log_uuid) < ({cursorStartedAt: DateTime64(6, 'UTC')}, {cursorDocumentLogUuid: String})`,
+      `(started_at, document_log_uuid) < ({cursorStartedAt: DateTime64(6, 'UTC')}, {cursorDocumentLogUuid: UUID})`,
     )
   }
 
diff --git a/packages/core/src/queries/clickhouse/spans/findByDocumentLogUuid.ts b/packages/core/src/queries/clickhouse/spans/findByDocumentLogUuid.ts
@@ -15,7 +15,7 @@ export async function getLastTraceByLogUuid({
       SELECT trace_id
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid = {logUuid: String}
+        AND document_log_uuid = {logUuid: UUID}
       ORDER BY started_at DESC
       LIMIT 1
     `,
@@ -39,7 +39,7 @@ export async function listTraceIdsByLogUuid({
       SELECT DISTINCT trace_id
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid = {logUuid: String}
+        AND document_log_uuid = {logUuid: UUID}
       ORDER BY trace_id ASC
     `,
     format: 'JSONEachRow',
@@ -64,7 +64,7 @@ export async function findByDocumentLogUuids({
       SELECT *
       FROM ${SPANS_TABLE} FINAL
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid IN ({documentLogUuids: Array(String)})
+        AND document_log_uuid IN ({documentLogUuids: Array(UUID)})
     `,
     format: 'JSONEachRow',
     query_params: { workspaceId, documentLogUuids },
@@ -86,7 +86,7 @@ export async function findByDocumentLogUuid({
       SELECT *
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid = {documentLogUuid: String}
+        AND document_log_uuid = {documentLogUuid: UUID}
       ORDER BY ingested_at DESC
       LIMIT 1
     `,
@@ -111,7 +111,7 @@ export async function listByDocumentLogUuid({
       SELECT *
       FROM ${SPANS_TABLE} FINAL
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid = {documentLogUuid: String}
+        AND document_log_uuid = {documentLogUuid: UUID}
       ORDER BY started_at ASC, span_id ASC
     `,
     format: 'JSONEachRow',
@@ -136,7 +136,7 @@ export async function findLastMainSpanByDocumentLogUuid({
       SELECT *
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid = {documentLogUuid: String}
+        AND document_log_uuid = {documentLogUuid: UUID}
         AND type IN ({mainTypes: Array(String)})
       ORDER BY started_at DESC
       LIMIT 1
@@ -164,7 +164,7 @@ export async function findFirstMainSpanByDocumentLogUuid({
       SELECT *
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid = {documentLogUuid: String}
+        AND document_log_uuid = {documentLogUuid: UUID}
         AND type IN ({mainTypes: Array(String)})
       ORDER BY started_at ASC
       LIMIT 1
@@ -192,7 +192,7 @@ export async function getSpanIdentifiersByDocumentLogUuids({
       SELECT trace_id, span_id
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_log_uuid IN ({documentLogUuids: Array(String)})
+        AND document_log_uuid IN ({documentLogUuids: Array(UUID)})
     `,
     format: 'JSONEachRow',
     query_params: { workspaceId, documentLogUuids },
diff --git a/packages/core/src/queries/clickhouse/spans/findLimited.ts b/packages/core/src/queries/clickhouse/spans/findLimited.ts
@@ -43,7 +43,7 @@ function buildWhereClause(
   }
 
   if (filters.experimentUuids && filters.experimentUuids.length > 0) {
-    conditions.push(`experiment_uuid IN ({experimentUuids: Array(String)})`)
+    conditions.push(`experiment_uuid IN ({experimentUuids: Array(UUID)})`)
     params.experimentUuids = filters.experimentUuids
   }
 
@@ -200,12 +200,12 @@ export async function findByDocumentAndCommitLimited({
 }) {
   const baseConditions = [
     `workspace_id = {workspaceId: UInt64}`,
-    `document_uuid = {documentUuid: String}`,
+    `document_uuid = {documentUuid: UUID}`,
   ]
   const baseParams: Record<string, unknown> = { workspaceId, documentUuid }
 
   if (commitUuids && commitUuids.length > 0) {
-    baseConditions.push(`commit_uuid IN ({commitUuids: Array(String)})`)
+    baseConditions.push(`commit_uuid IN ({commitUuids: Array(UUID)})`)
     baseParams.commitUuids = commitUuids
   }
 
diff --git a/packages/core/src/queries/clickhouse/spans/getAnnotationsProgress.ts b/packages/core/src/queries/clickhouse/spans/getAnnotationsProgress.ts
@@ -21,7 +21,7 @@ export async function getSpansCountForAnnotationsProgress({
       SELECT count() AS total_count
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND commit_uuid IN ({commitUuids: Array(String)})
+        AND commit_uuid IN ({commitUuids: Array(UUID)})
         AND (source IN ({logSources: Array(String)}) OR source IS NULL)
         AND started_at >= {fromDate: DateTime64(6, 'UTC')}
     `,
diff --git a/packages/core/src/queries/clickhouse/spans/getByDocument.ts b/packages/core/src/queries/clickhouse/spans/getByDocument.ts
@@ -43,10 +43,10 @@ export async function getSpansByDocument({
 
   const conditions = [
     `workspace_id = {workspaceId: UInt64}`,
-    `document_uuid = {documentUuid: String}`,
+    `document_uuid = {documentUuid: UUID}`,
     `type IN ({spanTypes: Array(String)})`,
     `status = {okStatus: String}`,
-    `commit_uuid IN ({commitUuids: Array(String)})`,
+    `commit_uuid IN ({commitUuids: Array(UUID)})`,
     `source != {optimizationSource: String}`,
   ]
 
@@ -58,7 +58,7 @@ export async function getSpansByDocument({
   if (optimizationExperimentUuids.length > 0) {
     params.optimizationExperimentUuids = optimizationExperimentUuids
     experimentConditions.push(
-      `experiment_uuid NOT IN ({optimizationExperimentUuids: Array(String)})`,
+      `experiment_uuid NOT IN ({optimizationExperimentUuids: Array(UUID)})`,
     )
   }
 
diff --git a/packages/core/src/queries/clickhouse/spans/getExperimentPromptSpansBefore.ts b/packages/core/src/queries/clickhouse/spans/getExperimentPromptSpansBefore.ts
@@ -19,7 +19,7 @@ export async function getExperimentPromptSpansBefore({
       SELECT span_id, trace_id
       FROM ${SPANS_TABLE}
       WHERE workspace_id = {workspaceId: UInt64}
-        AND document_uuid = {documentUuid: String}
+        AND document_uuid = {documentUuid: UUID}
         AND started_at < {before: DateTime64(6, 'UTC')}
         AND experiment_uuid IS NULL
         AND type = {promptType: String}
diff --git a/packages/core/src/queries/clickhouse/spans/getExperimentRunMetadata.ts b/packages/core/src/queries/clickhouse/spans/getExperimentRunMetadata.ts
diff --git a/packages/core/src/queries/clickhouse/spans/getSpansWithoutIssues.ts b/packages/core/src/queries/clickhouse/spans/getSpansWithoutIssues.ts
diff --git a/packages/core/src/queries/clickhouse/spans/getUsageOverviewCounts.ts b/packages/core/src/queries/clickhouse/spans/getUsageOverviewCounts.ts

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+ALTER TABLE spans ON CLUSTER default DROP INDEX IF EXISTS idx_test_deployment_id;`
	`2`	`+ALTER TABLE spans ON CLUSTER default DROP INDEX IF EXISTS idx_parent_id;`
	`3`	`+ALTER TABLE spans ON CLUSTER default DROP INDEX IF EXISTS idx_experiment_uuid;`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+ALTER TABLE spans DROP INDEX IF EXISTS idx_test_deployment_id;`
	`2`	`+ALTER TABLE spans DROP INDEX IF EXISTS idx_parent_id;`
	`3`	`+ALTER TABLE spans DROP INDEX IF EXISTS idx_experiment_uuid;`
Original file line number	Diff line number	Diff line change
`@@ -30,13 +30,13 @@ export async function computeDocumentTracesDailyCount({`
`30`	`30`
`31`	`31`	`const conditions = [`
`32`	`32`	`workspace_id = {workspaceId: UInt64}`,
`33`		- `document_uuid = {documentUuid: String}`,
	`33`	+ `document_uuid = {documentUuid: UUID}`,
`34`	`34`	`started_at >= {startDate: DateTime64(6, 'UTC')}`,
`35`	`35`	`]`
`36`	`36`
`37`	`37`	`if (commitUuid) {`
`38`	`38`	`params.commitUuid = commitUuid`
`39`		- conditions.push(`commit_uuid = {commitUuid: String}`)
	`39`	+ conditions.push(`commit_uuid = {commitUuid: UUID}`)
`40`	`40`	`}`
`41`	`41`
`42`	`42`	`const result = await clickhouseClient().query({`