add etl logs (supabase#37930)

* add etl logs

* fix missing desc

* rename to etl-replication-logs

* rename sidebar

* Add links to ETL logs

* Fix

* Smol

---------

Co-authored-by: Joshen Lim <[email protected]>

Author: jordienr

apps/studio/components/interfaces/Database/Replication/Pipeline.utils.ts

@@ -22,44 +22,44 @@ export const getStatusName = (
   return undefined
 }
 
-export const PIPELINE_STATE_MESSAGES = {
+const PIPELINE_STATE_MESSAGES = {
   enabling: {
-    title: 'Pipeline Enabling',
+    title: 'Pipeline enabling',
     message: 'Starting the pipeline. Table replication will resume once enabled.',
     badge: 'Enabling',
   },
   disabling: {
-    title: 'Pipeline Disabling',
+    title: 'Pipeline disabling',
     message: 'Stopping the pipeline. Table replication will be paused once disabled.',
     badge: 'Disabling',
   },
   failed: {
-    title: 'Pipeline Failed',
-    message: 'Replication has encountered an error. Check the logs for details.',
+    title: 'Pipeline failed',
+    message: 'Replication has encountered an error.',
     badge: 'Failed',
   },
   stopped: {
-    title: 'Pipeline Stopped',
+    title: 'Pipeline stopped',
     message: 'Replication is paused. Enable the pipeline to resume data synchronization.',
     badge: 'Stopped',
   },
   starting: {
-    title: 'Pipeline Starting',
+    title: 'Pipeline starting',
     message: 'Initializing replication. Table status will be available once running.',
     badge: 'Starting',
   },
   running: {
-    title: 'Pipeline Running',
+    title: 'Pipeline running',
     message: 'Replication is active and processing data',
     badge: 'Running',
   },
   unknown: {
-    title: 'Pipeline Status Unknown',
-    message: 'Unable to determine replication status. Check the logs for more information.',
+    title: 'Pipeline status unknown',
+    message: 'Unable to determine replication status.',
     badge: 'Unknown',
   },
   notRunning: {
-    title: 'Pipeline Not Running',
+    title: 'Pipeline not running',
     message: 'Replication is not active. Enable the pipeline to start data synchronization.',
     badge: 'Disabled',
   },

apps/studio/components/interfaces/Database/Replication/PipelineStatus.tsx

@@ -1,3 +1,5 @@
+import { useParams } from 'common'
+import { InlineLink } from 'components/ui/InlineLink'
 import { ReplicationPipelineStatusData } from 'data/replication/pipeline-status-query'
 import { AlertTriangle, Loader2 } from 'lucide-react'
 import { PipelineStatusRequestStatus } from 'state/replication-pipeline-request-status'
@@ -31,6 +33,8 @@ export const PipelineStatus = ({
   isSuccess,
   requestStatus,
 }: PipelineStatusProps) => {
+  const { ref } = useParams()
+
   // Map backend statuses to UX-friendly display
   const getStatusConfig = () => {
     const statusName =
@@ -138,7 +142,17 @@ export const PipelineStatus = ({
               <span>{statusConfig.label}</span>
             </div>
           </TooltipTrigger>
-          <TooltipContent side="bottom">{statusConfig.tooltip}</TooltipContent>
+          <TooltipContent side="bottom">
+            {statusConfig.tooltip}
+            {['unknown', 'failed'].includes(pipelineStatus?.name ?? '') && (
+              <>
+                {' '}
+                Check the{' '}
+                <InlineLink href={`/project/${ref}/logs/etl-replication-logs`}>logs</InlineLink> for
+                more information.
+              </>
+            )}
+          </TooltipContent>
         </Tooltip>
       )}
     </>

apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus.tsx

@@ -174,6 +174,9 @@ export const ReplicationPipelineStatus = () => {
               }
             />
           </div>
+          <Button asChild type="default">
+            <Link href={`/project/${projectRef}/logs/etl-replication-logs`}>View logs</Link>
+          </Button>
           <Button
             type={statusName === 'stopped' ? 'primary' : 'default'}
             onClick={() => onTogglePipeline()}

apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts

@@ -372,6 +372,7 @@ export enum LogsTableName {
   PGBOUNCER = 'pgbouncer_logs',
   PG_UPGRADE = 'pg_upgrade_logs',
   PG_CRON = 'pg_cron_logs',
+  ETL = 'etl_replication_logs',
 }
 
 export const LOGS_TABLES = {
@@ -387,6 +388,7 @@ export const LOGS_TABLES = {
   pg_upgrade: LogsTableName.PG_UPGRADE,
   pg_cron: LogsTableName.POSTGRES,
   pgbouncer: LogsTableName.PGBOUNCER,
+  etl: LogsTableName.ETL,
 }
 
 export const LOGS_SOURCE_DESCRIPTION = {
@@ -402,6 +404,7 @@ export const LOGS_SOURCE_DESCRIPTION = {
   [LogsTableName.PGBOUNCER]: 'Dedicated connection pooler for PostgreSQL',
   [LogsTableName.PG_UPGRADE]: 'Logs generated by the Postgres version upgrade process',
   [LogsTableName.PG_CRON]: 'Postgres logs from pg_cron cron jobs',
+  [LogsTableName.ETL]: 'Logs from the ETL replication process',
 }
 
 export const FILTER_OPTIONS: FilterTableSet = {

apps/studio/components/interfaces/Settings/Logs/Logs.types.ts

@@ -93,6 +93,7 @@ export type QueryType =
   | 'pg_upgrade'
   | 'pg_cron'
   | 'pgbouncer'
+  | 'etl'
 
 export type Mode = 'simple' | 'custom'
 

apps/studio/components/layouts/LogsLayout/LogsSidebarMenuV2.tsx

@@ -33,6 +33,7 @@ import {
 } from 'ui-patterns/InnerSideMenu'
 import { GenericSkeletonLoader } from 'ui-patterns/ShimmeringLoader'
 import { FeaturePreviewSidebarPanel } from '../../ui/FeaturePreviewSidebarPanel'
+import { useReplicationSourcesQuery } from 'data/replication/sources-query'
 
 const SupaIcon = ({ className }: { className?: string }) => {
   return (
@@ -91,6 +92,19 @@ export function LogsSidebarMenuV2() {
     realtimeAll: realtimeEnabled,
   } = useIsFeatureEnabled(['project_storage:all', 'project_auth:all', 'realtime:all'])
 
+  const enablePgReplicate = useFlag('enablePgReplicate')
+  const { data: etlData, isLoading: isETLLoading } = useReplicationSourcesQuery(
+    {
+      projectRef: ref,
+    },
+    {
+      enabled: enablePgReplicate,
+    }
+  )
+
+  // [Jordi] We only want to show ETL logs if the user has the feature enabled AND they're using the feature aka they've created a source.
+  const showETLLogs = enablePgReplicate && (etlData?.sources?.length ?? 0) > 0 && !isETLLoading
+
   const { plan: orgPlan, isLoading: isOrgPlanLoading } = useCurrentOrgPlan()
   const isFreePlan = !isOrgPlanLoading && orgPlan?.id === 'free'
 
@@ -183,6 +197,14 @@ export function LogsSidebarMenuV2() {
       url: `/project/${ref}/logs/pgcron-logs`,
       items: [],
     },
+    showETLLogs
+      ? {
+          name: 'ETL Replication',
+          key: 'etl_replication_logs',
+          url: `/project/${ref}/logs/etl-replication-logs`,
+          items: [],
+        }
+      : null,
   ].filter((x) => x !== null)
 
   const OPERATIONAL_COLLECTIONS = IS_PLATFORM

apps/studio/pages/project/[ref]/logs/etl-replication-logs.tsx

@@ -0,0 +1,27 @@
+import { LogsTableName } from 'components/interfaces/Settings/Logs/Logs.constants'
+import LogsPreviewer from 'components/interfaces/Settings/Logs/LogsPreviewer'
+import LogsLayout from 'components/layouts/LogsLayout/LogsLayout'
+import DefaultLayout from 'components/layouts/DefaultLayout'
+import type { NextPageWithLayout } from 'types'
+import { useParams } from 'common'
+
+export const LogPage: NextPageWithLayout = () => {
+  const { ref } = useParams()
+
+  return (
+    <LogsPreviewer
+      condensedLayout
+      queryType="etl"
+      projectRef={ref!}
+      tableName={LogsTableName.ETL}
+    />
+  )
+}
+
+LogPage.getLayout = (page) => (
+  <DefaultLayout>
+    <LogsLayout title="ETL Replication Logs">{page}</LogsLayout>
+  </DefaultLayout>
+)
+
+export default LogPage