updated clickhouse config for logs

fixed admin access to logs feature
added alpha badge to logs

Author: mpcgrid

apps/webapp/app/components/logs/LogsTable.tsx

@@ -1,5 +1,4 @@
 import { ArrowPathIcon } from "@heroicons/react/20/solid";
-import { Link } from "@remix-run/react";
 import { formatDurationNanoseconds } from "@trigger.dev/core/v3";
 import { type ReactNode, useEffect, useRef, useState } from "react";
 import { cn } from "~/utils/cn";
@@ -104,13 +103,11 @@ function highlightText(text: string, searchTerm: string | undefined): ReactNode
 export function LogsTable({
   logs,
   hasFilters,
-  filters,
   searchTerm,
   isLoading = false,
   isLoadingMore = false,
   hasMore = false,
   onLoadMore,
-  variant = "dimmed",
   selectedLogId,
   onLogSelect,
 }: LogsTableProps) {

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -269,13 +269,16 @@ export function SideMenu({
               to={v3DeploymentsPath(organization, project, environment)}
               data-action="deployments"
             />
-            <SideMenuItem
-              name="Logs"
-              icon={LogsIcon}
-              activeIconColor="text-logs"
-              to={v3LogsPath(organization, project, environment)}
-              data-action="logs"
-            />
+            {(isAdmin || user.isImpersonating) && (
+              <SideMenuItem
+                name="Logs"
+                icon={LogsIcon}
+                activeIconColor="text-logs"
+                to={v3LogsPath(organization, project, environment)}
+                data-action="logs"
+                badge={<AlphaBadge />}
+              />
+            )}
             <SideMenuItem
               name="Test"
               icon={BeakerIcon}

apps/webapp/app/env.server.ts

@@ -1175,6 +1175,18 @@ const EnvironmentSchema = z
     CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
     CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"),
 
+    // Logs List Query Settings (for paginated log views)
+    CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE: z.coerce.number().int().default(2_000_000_000),
+    CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT: z.coerce.number().int().default(1_000_000_000),
+    CLICKHOUSE_LOGS_LIST_MAX_THREADS: z.coerce.number().int().default(4),
+    CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ: z.coerce.number().int().optional(),
+    CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME: z.coerce.number().int().optional(),
+
+    // Logs Detail Query Settings (for single log views)
+    CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE: z.coerce.number().int().default(500_000_000),
+    CLICKHOUSE_LOGS_DETAIL_MAX_THREADS: z.coerce.number().int().default(2),
+    CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME: z.coerce.number().int().optional(),
+
     EVENTS_CLICKHOUSE_URL: z
       .string()
       .optional()

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx

@@ -1,4 +1,4 @@
-import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { type LoaderFunctionArgs , redirect} from "@remix-run/server-runtime";
 import { type MetaFunction, useFetcher, useNavigation, useLocation } from "@remix-run/react";
 import {
   TypedAwait,
@@ -62,6 +62,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = user.id;
   const isAdmin = user.admin || user.isImpersonating;
 
+  if (!isAdmin) {
+    throw redirect("/");
+  }
+
   const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params);
 
   const project = await findProjectBySlug(organizationSlug, projectParam, userId);

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts

@@ -1,14 +1,13 @@
 import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/node";
-import { requireUserId } from "~/services/session.server";
+import { requireUser, requireUserId } from "~/services/session.server";
 import { EnvironmentParamSchema } from "~/utils/pathBuilder";
 import { findProjectBySlug } from "~/models/project.server";
 import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server";
 import { LogsListPresenter, type LogLevel } from "~/presenters/v3/LogsListPresenter.server";
 import { $replica } from "~/db.server";
 import { clickhouseClient } from "~/services/clickhouseInstance.server";
-import { getUserById } from "~/models/user.server";
 
 // Valid log levels for filtering
 const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"];
@@ -33,9 +32,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     throw new Response("Environment not found", { status: 404 });
   }
 
-  const user = await requireUserId(request);
-  const userRecord = await getUserById(user);
-  const isAdmin = userRecord?.admin || userRecord?.isImpersonating;
+  const user = await requireUser(request);
+  const isAdmin = user?.admin || user?.isImpersonating;
 
   const filters = await getRunFiltersFromRequest(request);
 

apps/webapp/app/services/clickhouseInstance.server.ts

@@ -12,6 +12,28 @@ function initializeClickhouseClient() {
 
   console.log(`🗃️  Clickhouse service enabled to host ${url.host}`);
 
+  // Build logs query settings from environment variables
+  const logsQuerySettings = {
+    list: {
+      max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
+      max_bytes_before_external_sort: env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
+      max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
+      ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
+        max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
+      }),
+      ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
+        max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
+      }),
+    },
+    detail: {
+      max_memory_usage: env.CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE.toString(),
+      max_threads: env.CLICKHOUSE_LOGS_DETAIL_MAX_THREADS,
+      ...(env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME && {
+        max_execution_time: env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME,
+      }),
+    },
+  };
+
   const clickhouse = new ClickHouse({
     url: url.toString(),
     name: "clickhouse-instance",
@@ -24,6 +46,7 @@ function initializeClickhouseClient() {
       request: true,
     },
     maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
+    logsQuerySettings,
   });
 
   return clickhouse;

internal-packages/clickhouse/src/index.ts

@@ -46,6 +46,11 @@ export {
 } from "./client/tsql.js";
 export type { OutputColumnMetadata } from "@internal/tsql";
 
+export type LogsQuerySettings = {
+  list?: ClickHouseSettings;
+  detail?: ClickHouseSettings;
+};
+
 export type ClickhouseCommonConfig = {
   keepAlive?: {
     enabled?: boolean;
@@ -60,6 +65,7 @@ export type ClickhouseCommonConfig = {
     response?: boolean;
   };
   maxOpenConnections?: number;
+  logsQuerySettings?: LogsQuerySettings;
 };
 
 export type ClickHouseConfig =
@@ -83,9 +89,11 @@ export class ClickHouse {
   public readonly writer: ClickhouseWriter;
   private readonly logger: Logger;
   private _splitClients: boolean;
+  private readonly logsQuerySettings?: LogsQuerySettings;
 
   constructor(config: ClickHouseConfig) {
     this.logger = config.logger ?? new Logger("ClickHouse", config.logLevel ?? "debug");
+    this.logsQuerySettings = config.logsQuerySettings;
 
     if (config.url) {
       const url = new URL(config.url);
@@ -197,8 +205,8 @@ export class ClickHouse {
       traceSummaryQueryBuilder: getTraceSummaryQueryBuilderV2(this.reader),
       traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilderV2(this.reader),
       spanDetailsQueryBuilder: getSpanDetailsQueryBuilderV2(this.reader),
-      logsListQueryBuilder: getLogsListQueryBuilder(this.reader),
-      logDetailQueryBuilder: getLogDetailQueryBuilder(this.reader),
+      logsListQueryBuilder: getLogsListQueryBuilder(this.reader, this.logsQuerySettings?.list),
+      logDetailQueryBuilder: getLogDetailQueryBuilder(this.reader, this.logsQuerySettings?.detail),
     };
   }
 }

internal-packages/clickhouse/src/taskEvents.ts

@@ -276,12 +276,7 @@ export function getLogsListQueryBuilder(ch: ClickhouseReader, settings?: ClickHo
       "metadata",
       "attributes"
     ],
-    settings: {
-      max_memory_usage: "2000000000", // 2GB per query limit
-      max_bytes_before_external_sort: "1000000000", // 1GB before spill to disk
-      max_threads: 4, // Limit parallelism to reduce memory
-      ...settings,
-    },
+    settings,
   });
 }