Clickhouse performance improvements (#2175)

matt-aitken · web-flow · commit a0815c8e3e47 · 2025-06-14T11:10:28.000+01:00
* Runs filter by org id and add created at to ordering

* CopyableText can accept an alternative value for copying

* The runs table now shows the ID instead of run number

* Paginating back/forwards fix

* The task stats need org id and project id too
diff --git a/apps/webapp/app/components/primitives/CopyableText.tsx b/apps/webapp/app/components/primitives/CopyableText.tsx
@@ -4,9 +4,17 @@ import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { useCopy } from "~/hooks/useCopy";
 import { cn } from "~/utils/cn";
 
-export function CopyableText({ value, className }: { value: string; className?: string }) {
+export function CopyableText({
+  value,
+  copyValue,
+  className,
+}: {
+  value: string;
+  copyValue?: string;
+  className?: string;
+}) {
   const [isHovered, setIsHovered] = useState(false);
-  const { copy, copied } = useCopy(value);
+  const { copy, copied } = useCopy(copyValue ?? value);
 
   return (
     <span
diff --git a/apps/webapp/app/components/runs/v3/TaskRunsTable.tsx b/apps/webapp/app/components/runs/v3/TaskRunsTable.tsx
@@ -52,6 +52,8 @@ import {
   TaskRunStatusCombo,
 } from "./TaskRunStatus";
 import { useEnvironment } from "~/hooks/useEnvironment";
+import { CopyableText } from "~/components/primitives/CopyableText";
+import { ClipboardField } from "~/components/primitives/ClipboardField";
 
 type RunsTableProps = {
   total: number;
@@ -134,7 +136,7 @@ export function TaskRunsTable({
               )}
             </TableHeaderCell>
           )}
-          <TableHeaderCell alignment="right">Run #</TableHeaderCell>
+          <TableHeaderCell>ID</TableHeaderCell>
           <TableHeaderCell>Task</TableHeaderCell>
           <TableHeaderCell>Version</TableHeaderCell>
           <TableHeaderCell
@@ -306,8 +308,21 @@ export function TaskRunsTable({
                     />
                   </TableCell>
                 )}
-                <TableCell to={path} alignment="right" isTabbableCell>
-                  {formatNumber(run.number)}
+                <TableCell to={path} isTabbableCell>
+                  <SimpleTooltip
+                    content={run.friendlyId}
+                    button={
+                      <span className="flex h-6 items-center gap-1">
+                        <CopyableText
+                          value={run.friendlyId.slice(-8)}
+                          copyValue={run.friendlyId}
+                          className="font-mono"
+                        />
+                      </span>
+                    }
+                    asChild
+                    disableHoverableContent
+                  />
                 </TableCell>
                 <TableCell to={path}>
                   <span className="flex items-center gap-x-1">
diff --git a/apps/webapp/app/presenters/v3/NextRunListPresenter.server.ts b/apps/webapp/app/presenters/v3/NextRunListPresenter.server.ts
@@ -45,6 +45,7 @@ export class NextRunListPresenter {
   ) {}
 
   public async call(
+    organizationId: string,
     environmentId: string,
     {
       userId,
@@ -190,6 +191,7 @@ export class NextRunListPresenter {
     });
 
     const { runs, pagination } = await runsRepository.listRuns({
+      organizationId,
       environmentId,
       projectId,
       tasks,
diff --git a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts
@@ -1,16 +1,16 @@
 import {
-  PrismaClientOrTransaction,
-  RuntimeEnvironmentType,
+  type PrismaClientOrTransaction,
+  type RuntimeEnvironmentType,
   type TaskTriggerSource,
 } from "@trigger.dev/database";
 import { $replica } from "~/db.server";
 import { clickhouseClient } from "~/services/clickhouseInstance.server";
 import {
-  AverageDurations,
+  type AverageDurations,
   ClickHouseEnvironmentMetricsRepository,
-  CurrentRunningStats,
-  DailyTaskActivity,
-  EnvironmentMetricsRepository,
+  type CurrentRunningStats,
+  type DailyTaskActivity,
+  type EnvironmentMetricsRepository,
   PostgrestEnvironmentMetricsRepository,
 } from "~/services/environmentMetricsRepository.server";
 import { singleton } from "~/utils/singleton";
@@ -32,9 +32,13 @@ export class TaskListPresenter {
   ) {}
 
   public async call({
+    organizationId,
+    projectId,
     environmentId,
     environmentType,
   }: {
+    organizationId: string;
+    projectId: string;
     environmentId: string;
     environmentType: RuntimeEnvironmentType;
   }) {
@@ -76,18 +80,24 @@ export class TaskListPresenter {
     // IMPORTANT: Don't await these, we want to return the promises
     // so we can defer the loading of the data
     const activity = this.environmentMetricsRepository.getDailyTaskActivity({
+      organizationId,
+      projectId,
       environmentId,
       days: 6, // This actually means 7 days, because we want to show the current day too
       tasks: slugs,
     });
 
     const runningStats = this.environmentMetricsRepository.getCurrentRunningStats({
+      organizationId,
+      projectId,
       environmentId,
       days: 6,
       tasks: slugs,
     });
 
     const durations = this.environmentMetricsRepository.getAverageDurations({
+      organizationId,
+      projectId,
       environmentId,
       days: 6,
       tasks: slugs,
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx
@@ -125,6 +125,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
   try {
     const { tasks, activity, runningStats, durations } = await taskListPresenter.call({
+      organizationId: project.organizationId,
+      projectId: project.id,
       environmentId: environment.id,
       environmentType: environment.type,
     });
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.next.runs._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.next.runs._index/route.tsx
@@ -128,7 +128,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   }
 
   const presenter = new NextRunListPresenter($replica, clickhouseClient);
-  const list = presenter.call(environment.id, {
+  const list = presenter.call(project.organizationId, environment.id, {
     userId,
     projectId: project.id,
     tasks,
diff --git a/apps/webapp/app/services/environmentMetricsRepository.server.ts b/apps/webapp/app/services/environmentMetricsRepository.server.ts
@@ -10,18 +10,24 @@ export type AverageDurations = Record<string, number>;
 
 export interface EnvironmentMetricsRepository {
   getDailyTaskActivity(options: {
+    organizationId: string;
+    projectId: string;
     environmentId: string;
     days: number;
     tasks: string[];
   }): Promise<DailyTaskActivity>;
 
   getCurrentRunningStats(options: {
+    organizationId: string;
+    projectId: string;
     environmentId: string;
     days: number;
     tasks: string[];
   }): Promise<CurrentRunningStats>;
 
   getAverageDurations(options: {
+    organizationId: string;
+    projectId: string;
     environmentId: string;
     days: number;
     tasks: string[];
@@ -177,10 +183,14 @@ export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetric
   constructor(private readonly options: ClickHouseEnvironmentMetricsRepositoryOptions) {}
 
   public async getDailyTaskActivity({
+    organizationId,
+    projectId,
     environmentId,
     days,
     tasks,
   }: {
+    organizationId: string;
+    projectId: string;
     environmentId: string;
     days: number;
     tasks: string[];
@@ -190,6 +200,8 @@ export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetric
     }
 
     const [queryError, activity] = await this.options.clickhouse.taskRuns.getTaskActivity({
+      organizationId,
+      projectId,
       environmentId,
       days,
     });
@@ -210,10 +222,14 @@ export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetric
   }
 
   public async getCurrentRunningStats({
+    organizationId,
+    projectId,
     environmentId,
     days,
     tasks,
   }: {
+    organizationId: string;
+    projectId: string;
     environmentId: string;
     days: number;
     tasks: string[];
@@ -223,6 +239,8 @@ export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetric
     }
 
     const [queryError, stats] = await this.options.clickhouse.taskRuns.getCurrentRunningStats({
+      organizationId,
+      projectId,
       environmentId,
       days,
     });
@@ -242,10 +260,14 @@ export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetric
   }
 
   public async getAverageDurations({
+    organizationId,
+    projectId,
     environmentId,
     days,
     tasks,
   }: {
+    organizationId: string;
+    projectId: string;
     environmentId: string;
     days: number;
     tasks: string[];
@@ -255,6 +277,8 @@ export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetric
     }
 
     const [queryError, durations] = await this.options.clickhouse.taskRuns.getAverageDurations({
+      organizationId,
+      projectId,
       environmentId,
       days,
     });
diff --git a/apps/webapp/app/services/runsRepository.server.ts b/apps/webapp/app/services/runsRepository.server.ts
@@ -1,8 +1,8 @@
-import { ClickHouse } from "@internal/clickhouse";
-import { Tracer } from "@internal/tracing";
-import { Logger, LogLevel } from "@trigger.dev/core/logger";
-import { TaskRunStatus } from "@trigger.dev/database";
-import { PrismaClient } from "~/db.server";
+import { type ClickHouse } from "@internal/clickhouse";
+import { type Tracer } from "@internal/tracing";
+import { type Logger, type LogLevel } from "@trigger.dev/core/logger";
+import { type TaskRunStatus } from "@trigger.dev/database";
+import { type PrismaClient } from "~/db.server";
 
 export type RunsRepositoryOptions = {
   clickhouse: ClickHouse;
@@ -13,6 +13,7 @@ export type RunsRepositoryOptions = {
 };
 
 export type ListRunsOptions = {
+  organizationId: string;
   projectId: string;
   environmentId: string;
   //filters
@@ -43,11 +44,14 @@ export class RunsRepository {
   async listRuns(options: ListRunsOptions) {
     const queryBuilder = this.options.clickhouse.taskRuns.queryBuilder();
     queryBuilder
-      .where("environment_id = {environmentId: String}", {
-        environmentId: options.environmentId,
+      .where("organization_id = {organizationId: String}", {
+        organizationId: options.organizationId,
       })
       .where("project_id = {projectId: String}", {
         projectId: options.projectId,
+      })
+      .where("environment_id = {environmentId: String}", {
+        environmentId: options.environmentId,
       });
 
     if (options.tasks && options.tasks.length > 0) {
@@ -115,17 +119,17 @@ export class RunsRepository {
       if (options.page.direction === "forward") {
         queryBuilder
           .where("run_id < {runId: String}", { runId: options.page.cursor })
-          .orderBy("run_id DESC")
+          .orderBy("created_at DESC, run_id DESC")
           .limit(options.page.size + 1);
       } else {
         queryBuilder
           .where("run_id > {runId: String}", { runId: options.page.cursor })
-          .orderBy("run_id DESC")
+          .orderBy("created_at ASC, run_id ASC")
           .limit(options.page.size + 1);
       }
     } else {
       // Initial page - no cursor provided
-      queryBuilder.orderBy("run_id DESC").limit(options.page.size + 1);
+      queryBuilder.orderBy("created_at DESC, run_id DESC").limit(options.page.size + 1);
     }
 
     const [queryError, result] = await queryBuilder.execute();
@@ -143,38 +147,33 @@ export class RunsRepository {
     let previousCursor: string | null = null;
 
     //get cursors for next and previous pages
-    if (options.page.cursor) {
-      switch (options.page.direction) {
-        case "forward":
-          previousCursor = runIds.at(0) ?? null;
-          if (hasMore) {
-            // The next cursor should be the last run ID from this page
-            nextCursor = runIds[options.page.size - 1];
-          }
-          break;
-        case "backward":
-          // No need to reverse since we're using DESC ordering consistently
-          if (hasMore) {
-            previousCursor = runIds[options.page.size - 1];
-          }
-          nextCursor = runIds.at(0) ?? null;
-          break;
-        default:
-          // This shouldn't happen if cursor is provided, but handle it
-          if (hasMore) {
-            nextCursor = runIds[options.page.size - 1];
-          }
-          break;
+    const direction = options.page.direction ?? "forward";
+    switch (direction) {
+      case "forward": {
+        previousCursor = options.page.cursor ? runIds.at(0) ?? null : null;
+        if (hasMore) {
+          // The next cursor should be the last run ID from this page
+          nextCursor = runIds[options.page.size - 1];
+        }
+        break;
       }
-    } else {
-      // Initial page - no cursor
-      if (hasMore) {
-        // The next cursor should be the last run ID from this page
-        nextCursor = runIds[options.page.size - 1];
+      case "backward": {
+        const reversedRunIds = [...runIds].reverse();
+        if (hasMore) {
+          previousCursor = reversedRunIds.at(1) ?? null;
+          nextCursor = reversedRunIds.at(options.page.size) ?? null;
+        } else {
+          nextCursor = reversedRunIds.at(options.page.size - 1) ?? null;
+        }
+
+        break;
       }
     }
 
-    const runIdsToReturn = hasMore ? runIds.slice(0, -1) : runIds;
+    const runIdsToReturn =
+      options.page.direction === "backward" && hasMore
+        ? runIds.slice(1, options.page.size + 1)
+        : runIds.slice(0, options.page.size);
 
     const runs = await this.options.prisma.taskRun.findMany({
       where: {
diff --git a/apps/webapp/test/runsRepository.test.ts b/apps/webapp/test/runsRepository.test.ts
diff --git a/internal-packages/clickhouse/src/taskRuns.ts b/internal-packages/clickhouse/src/taskRuns.ts

Original file line number	Diff line number	Diff line change
`@@ -128,7 +128,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {`
`128`	`128`	`}`
`129`	`129`
`130`	`130`	`const presenter = new NextRunListPresenter($replica, clickhouseClient);`
`131`		`- const list = presenter.call(environment.id, {`
	`131`	`+ const list = presenter.call(project.organizationId, environment.id, {`
`132`	`132`	`userId,`
`133`	`133`	`projectId: project.id,`
`134`	`134`	`tasks,`