Add ClickHouse query layer for spans with feature flag support

.tmuxinator.yml

@@ -4,6 +4,6 @@ windows:
   - apps:
       layout: main-horizontal
       panes:
-        - pnpm catchup && pnpm dev --filter='./apps/*'
+        - pnpm catchup && NODE_OPTIONS=--max_old_space_size=4096 pnpm dev --filter='./apps/*'
   - docker: docker compose up db redis weaviate mailpit --menu=false
   - studio: cd packages/core && pnpm db:studio

apps/gateway/src/routes/api/v3/projects/versions/documents/logs/create.handler.test.ts

@@ -19,6 +19,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const mocks = vi.hoisted(() => ({
   diskPut: vi.fn(),
+  diskPutBuffer: vi.fn(),
   cacheDel: vi.fn(),
 }))
 
@@ -28,6 +29,7 @@ vi.mock('@latitude-data/core/lib/disk', async (importOriginal) => {
     ...original,
     diskFactory: () => ({
       put: mocks.diskPut,
+      putBuffer: mocks.diskPutBuffer,
     }),
   }
 })
@@ -95,6 +97,7 @@ describe('POST /projects/:projectId/versions/:versionUuid/documents/logs', () =>
 
       // Mock disk operations to return success
       mocks.diskPut.mockResolvedValue(Result.nil())
+      mocks.diskPutBuffer.mockResolvedValue(Result.nil())
       mocks.cacheDel.mockResolvedValue(undefined)
 
       const { workspace, user, project, providers } = await createProject({
@@ -206,11 +209,11 @@ describe('POST /projects/:projectId/versions/:versionUuid/documents/logs', () =>
       expect(completionSpan!.commitUuid).toBe(commitUuid)
 
       // Verify metadata was saved (mocked)
-      expect(mocks.diskPut).toHaveBeenCalledTimes(2)
+      expect(mocks.diskPutBuffer).toHaveBeenCalledTimes(2)
       expect(mocks.cacheDel).toHaveBeenCalledTimes(3)
 
       // Verify the metadata keys contain the correct span IDs
-      const diskPutCalls = mocks.diskPut.mock.calls
+      const diskPutCalls = mocks.diskPutBuffer.mock.calls
       expect(
         diskPutCalls.some((call) => call[0].includes(promptSpan!.id)),
       ).toBe(true)

apps/gateway/src/routes/api/v3/projects/versions/documents/logs/create.handler.ts

@@ -12,6 +12,7 @@ import {
 } from '@latitude-data/core/constants'
 import { cache as redis } from '@latitude-data/core/cache'
 import { diskFactory } from '@latitude-data/core/lib/disk'
+import { compressString } from '@latitude-data/core/lib/disk/compression'
 import { publishSpanCreated } from '@latitude-data/core/services/tracing/publishSpanCreated'
 import { AppRouteHandler } from '$/openApi/types'
 import { CreateLogRoute } from './create.route'
@@ -232,8 +233,11 @@ async function createSpansFromLogData({
     traceId,
     promptSpanId,
   )
+  const compressedPromptMetadata = await compressString(
+    JSON.stringify(promptMetadata),
+  )
   await disk
-    .put(promptMetadataKey, JSON.stringify(promptMetadata))
+    .putBuffer(promptMetadataKey, compressedPromptMetadata)
     .then((r) => r.unwrap())
   await cache.del(promptMetadataKey)
 
@@ -242,8 +246,11 @@ async function createSpansFromLogData({
     traceId,
     completionSpanId,
   )
+  const compressedCompletionMetadata = await compressString(
+    JSON.stringify(completionMetadata),
+  )
   await disk
-    .put(completionMetadataKey, JSON.stringify(completionMetadata))
+    .putBuffer(completionMetadataKey, compressedCompletionMetadata)
     .then((r) => r.unwrap())
   await cache.del(completionMetadataKey)
 

apps/web/src/app/(admin)/backoffice/_components/BackofficeTabs/index.tsx

@@ -21,11 +21,6 @@ export function BackofficeTabs({ children }: { children: ReactNode }) {
               value: BackofficeRoutes.search,
               route: ROUTES.backoffice.search.root,
             },
-            {
-              label: 'Usage overview',
-              value: BackofficeRoutes.usageOverview,
-              route: ROUTES.backoffice.usageOverview.root,
-            },
             {
               label: 'Features',
               value: BackofficeRoutes.features,