proper ch migrations (#978)

* proper ch migrations

* tmp

* removed

* fixes

Author: skull8888888

frontend/instrumentation.ts

@@ -47,46 +47,23 @@ export async function register() {
 
       const initializeClickHouse = async () => {
         try {
-          const { clickhouseClient } = await import("@/lib/clickhouse/client.ts");
-          const { readFileSync, readdirSync } = await import("fs");
+          const { migration } = await import("clickhouse-migrations");
           const { join } = await import("path");
 
-          for (const file of readdirSync("lib/clickhouse/migrations")) {
-            if (!file.endsWith(".sql")) {
-              continue;
-            }
-            const schemaSql = readFileSync(join(process.cwd(), "lib/clickhouse/migrations", file), "utf-8");
-            const statements = schemaSql
-              .split(";")
-              .map((s) => s.trim())
-              .filter((s) => s.length > 0);
+          const migrationsHome = join(process.cwd(), "lib/clickhouse/migrations");
 
-            for (const statement of statements) {
-              try {
-                await clickhouseClient.exec({ query: statement });
-              } catch (error) {
-                if ((error as { type: string }).type === "DUPLICATE_COLUMN") {
-                  console.warn(
-                    "[WARNING] Failed to apply ClickHouse statement:",
-                    statement,
-                    "because column already exists"
-                  );
-                  continue;
-                } else if ((error as { type: string }).type === "TABLE_ALREADY_EXISTS") {
-                  console.warn(
-                    "[WARNING] Failed to apply ClickHouse statement:",
-                    statement,
-                    "because table already exists"
-                  );
-                  continue;
-                } else {
-                  throw error;
-                }
-              }
-            }
-          }
+          await migration(
+            migrationsHome,
+            process.env.CLICKHOUSE_URL || "http://localhost:8123",
+            process.env.CLICKHOUSE_USER || "ch_user",
+            process.env.CLICKHOUSE_PASSWORD || "ch_passwd",
+            process.env.CLICKHOUSE_DB || "default",
+            "ENGINE=Atomic", // db_engine
+            String(Number(process.env.CH_MIGRATIONS_TIMEOUT) || 30000), // timeout as string
+          );
         } catch (error) {
-          console.error("Failed to apply ClickHouse schema:", error);
+          console.error("Failed to apply ClickHouse migrations:", error);
+          throw error;
         }
       };
       // Run Postgres migrations and data initialization

frontend/lib/clickhouse/migrations/0009-labels-to-tags.sql

@@ -0,0 +1,48 @@
+-- Migration: Rename labels table to tags and label_source column to source
+-- This migration is idempotent and handles the case where it's already been applied
+
+-- Step 1: Create tags table if it doesn't exist (in case labels was already renamed)
+CREATE TABLE IF NOT EXISTS default.tags
+(
+    `project_id` UUID,
+    `class_id` UUID,
+    `created_at` DateTime64(9, 'UTC'),
+    `id` UUID,
+    `name` String,
+    `label_source` UInt8,
+    `span_id` UUID
+)
+ENGINE = MergeTree
+PRIMARY KEY (project_id, class_id, span_id)
+ORDER BY (project_id, class_id, span_id, created_at, id)
+SETTINGS index_granularity = 8192;
+
+-- Step 2: Copy data from labels to tags (only if labels exists and tags is empty)
+-- This will copy all data from labels if it exists
+-- On re-run, if labels doesn't exist, this will fail - but that's okay, it means data was already migrated
+INSERT INTO default.tags 
+SELECT 
+    project_id,
+    class_id,
+    created_at,
+    id,
+    name,
+    label_source,
+    span_id
+FROM default.labels
+WHERE EXISTS (
+    SELECT 1 
+    FROM system.tables 
+    WHERE database = 'default' AND name = 'labels'
+)
+AND NOT EXISTS (
+    SELECT 1 
+    FROM default.tags 
+    LIMIT 1
+);
+
+-- Step 3: Rename column (idempotent with IF EXISTS)
+ALTER TABLE default.tags RENAME COLUMN IF EXISTS label_source TO source;
+
+-- Step 4: Drop the old labels table if it still exists
+DROP TABLE IF EXISTS default.labels;

…/clickhouse/migrations/0008-squashed.sql …lib/clickhouse/migrations/1_squashed.sqlfrontend/lib/clickhouse/migrations/0008-squashed.sql renamed to frontend/lib/clickhouse/migrations/1_squashed.sql frontend/lib/clickhouse/migrations/0008-squashed.sql renamed to frontend/lib/clickhouse/migrations/1_squashed.sql

@@ -20,9 +20,9 @@
     "@ai-sdk/anthropic": "^2.0.13",
     "@ai-sdk/azure": "^2.0.26",
     "@ai-sdk/google": "^2.0.11",
-    "@ai-sdk/openai": "^2.0.23",
     "@ai-sdk/groq": "^2.0.17",
     "@ai-sdk/mistral": "^2.0.13",
+    "@ai-sdk/openai": "^2.0.23",
     "@ai-sdk/provider": "2.0.0",
     "@ai-sdk/react": "^2.0.29",
     "@aws-sdk/client-s3": "^3.864.0",
@@ -77,9 +77,10 @@
     "@uiw/codemirror-themes": "^4.25.1",
     "@uiw/react-codemirror": "^4.25.1",
     "@vercel/otel": "^1.13.0",
-    "ai": "5.0.28",
     "@xyflow/react": "^12.8.3",
+    "ai": "5.0.28",
     "class-variance-authority": "^0.7.1",
+    "clickhouse-migrations": "^1.1.1",
     "clsx": "^2.1.1",
     "cmdk": "^1.1.1",
     "dagre": "^0.8.5",
@@ -125,11 +126,11 @@
     "shiki": "^3.12.2",
     "streamdown": "^1.2.0",
     "stripe": "^16.12.0",
-    "use-stick-to-bottom": "^1.1.1",
-    "zod": "^4.1.5",
     "swr": "^2.3.6",
+    "use-stick-to-bottom": "^1.1.1",
     "uuid": "^9.0.1",
     "yaml": "^2.8.1",
+    "zod": "^4.1.5",
     "zustand": "^4.5.7"
   },
   "devDependencies": {