feat(cron): add build status reconciliation cron job

supabase/functions/_backend/triggers/cron_reconcile_build_status.ts

@@ -0,0 +1,150 @@
+import type { MiddlewareKeyVariables } from '../utils/hono.ts'
+import { Hono } from 'hono/tiny'
+import { BRES, middlewareAPISecret } from '../utils/hono.ts'
+import { cloudlog, cloudlogErr } from '../utils/logging.ts'
+import { recordBuildTime, supabaseAdmin } from '../utils/supabase.ts'
+import { getEnv } from '../utils/utils.ts'
+
+interface BuilderStatusResponse {
+  job: {
+    status: string
+    started_at: number | null
+    completed_at: number | null
+    error: string | null
+  }
+  machine: Record<string, unknown> | null
+  uploadUrl?: string
+}
+
+const TERMINAL_STATUSES = new Set(['succeeded', 'failed', 'expired', 'released', 'cancelled'])
+const STALE_THRESHOLD_MINUTES = 5
+const ORPHAN_THRESHOLD_HOURS = 1
+const BATCH_LIMIT = 50
+
+export const app = new Hono<MiddlewareKeyVariables>()
+
+app.post('/', middlewareAPISecret, async (c) => {
+  const startTime = Date.now()
+  let reconciled = 0
+  let orphaned = 0
+  let errors = 0
+
+  const supabase = supabaseAdmin(c)
+  const builderUrl = getEnv(c, 'BUILDER_URL')
+  const builderApiKey = getEnv(c, 'BUILDER_API_KEY')
+
+  const { data: staleBuilds, error: queryError } = await supabase
+    .from('build_requests')
+    .select('id, builder_job_id, app_id, owner_org, requested_by, platform, status, created_at')
+    .not('status', 'in', `(${[...TERMINAL_STATUSES].join(',')})`)
+    .lt('updated_at', new Date(Date.now() - STALE_THRESHOLD_MINUTES * 60 * 1000).toISOString())
+    .order('updated_at', { ascending: true })
+    .limit(BATCH_LIMIT)
+
+  if (queryError) {
+    cloudlogErr({ requestId: c.get('requestId'), message: 'Failed to query stale build_requests', error: queryError.message })
+    return c.json(BRES)
+  }
+
+  if (!staleBuilds || staleBuilds.length === 0) {
+    cloudlog({ requestId: c.get('requestId'), message: 'No stale builds to reconcile' })
+    return c.json(BRES)
+  }
+
+  cloudlog({ requestId: c.get('requestId'), message: `Found ${staleBuilds.length} stale builds to reconcile` })
+
+  for (const build of staleBuilds) {
+    if (!build.builder_job_id) {
+      const createdAt = new Date(build.created_at).getTime()
+      const orphanCutoff = Date.now() - ORPHAN_THRESHOLD_HOURS * 60 * 60 * 1000
+      if (createdAt < orphanCutoff) {
+        const { error: updateError } = await supabase
+          .from('build_requests')
+          .update({
+            status: 'failed',
+            last_error: 'Build request was never submitted to builder',
+            updated_at: new Date().toISOString(),
+          })
+          .eq('id', build.id)
+
+        if (updateError) {
+          cloudlogErr({ requestId: c.get('requestId'), message: 'Failed to mark orphan build as failed', buildId: build.id, error: updateError.message })
+          errors++
+        }
+        else {
+          orphaned++
+        }
+      }
+      continue
+    }
+
+    try {
+      const response = await fetch(`${builderUrl}/jobs/${build.builder_job_id}`, {
+        method: 'GET',
+        headers: { 'x-api-key': builderApiKey },
+      })
+
+      if (!response.ok) {
+        cloudlogErr({ requestId: c.get('requestId'), message: 'Builder status fetch failed', buildId: build.id, jobId: build.builder_job_id, status: response.status })
+        errors++
+        continue
+      }
+
+      const builderJob = await response.json() as BuilderStatusResponse
+      const jobStatus = builderJob.job.status
+
+      const { error: updateError } = await supabase
+        .from('build_requests')
+        .update({
+          status: jobStatus,
+          last_error: builderJob.job.error || null,
+          updated_at: new Date().toISOString(),
+        })
+        .eq('id', build.id)
+
+      if (updateError) {
+        cloudlogErr({ requestId: c.get('requestId'), message: 'Failed to update build_requests status', buildId: build.id, error: updateError.message })
+        errors++
+        continue
+      }
+
+      reconciled++
+
+      if (
+        TERMINAL_STATUSES.has(jobStatus)
+        && builderJob.job.started_at
+        && builderJob.job.completed_at
+      ) {
+        const buildTimeSeconds = Math.floor((builderJob.job.completed_at - builderJob.job.started_at) / 1000)
+        const resolvedPlatform = (build.platform === 'ios' || build.platform === 'android')
+          ? build.platform
+          : 'ios'
+
+        await recordBuildTime(
+          c,
+          build.owner_org,
+          build.requested_by,
+          build.builder_job_id,
+          resolvedPlatform as 'ios' | 'android',
+          buildTimeSeconds,
+        )
+      }
+    }
+    catch (err) {
+      cloudlogErr({ requestId: c.get('requestId'), message: 'Error reconciling build', buildId: build.id, jobId: build.builder_job_id, error: String(err) })
+      errors++
+    }
+  }
+
+  cloudlog({
+    requestId: c.get('requestId'),
+    message: 'Build status reconciliation completed',
+    duration_ms: Date.now() - startTime,
+    total: staleBuilds.length,
+    reconciled,
+    orphaned,
+    errors,
+  })
+
+  return c.json(BRES)
+})

supabase/functions/triggers/index.ts

@@ -1,6 +1,7 @@
 import { app as credit_usage_alerts } from '../_backend/triggers/credit_usage_alerts.ts'
 import { app as cron_clean_orphan_images } from '../_backend/triggers/cron_clean_orphan_images.ts'
 import { app as cron_clear_versions } from '../_backend/triggers/cron_clear_versions.ts'
+import { app as cron_reconcile_build_status } from '../_backend/triggers/cron_reconcile_build_status.ts'
 import { app as cron_email } from '../_backend/triggers/cron_email.ts'
 import { app as cron_stat_app } from '../_backend/triggers/cron_stat_app.ts'
 import { app as cron_stat_org } from '../_backend/triggers/cron_stat_org.ts'
@@ -48,6 +49,7 @@ appGlobal.route('/cron_stat_org', cron_stat_org)
 appGlobal.route('/cron_sync_sub', cron_sync_sub)
 appGlobal.route('/cron_clear_versions', cron_clear_versions)
 appGlobal.route('/cron_clean_orphan_images', cron_clean_orphan_images)
+appGlobal.route('/cron_reconcile_build_status', cron_reconcile_build_status)
 appGlobal.route('/credit_usage_alerts', credit_usage_alerts)
 appGlobal.route('/on_organization_delete', on_organization_delete)
 appGlobal.route('/on_deploy_history_create', on_deploy_history_create)

supabase/migrations/20260216102420_add_build_status_reconciliation_cron.sql

@@ -0,0 +1,75 @@
+SELECT pgmq.create('cron_reconcile_build_status');
+
+INSERT INTO public.cron_tasks (
+    name,
+    description,
+    task_type,
+    target,
+    batch_size,
+    second_interval,
+    minute_interval,
+    hour_interval,
+    run_at_hour,
+    run_at_minute,
+    run_at_second,
+    run_on_dow,
+    run_on_day
+) VALUES (
+    'reconcile_build_status',
+    'Send build status reconciliation job to queue every 15 minutes',
+    'queue',
+    'cron_reconcile_build_status',
+    null,
+    null,
+    15,
+    null,
+    null,
+    null,
+    0,
+    null,
+    null
+)
+ON CONFLICT (name) DO UPDATE SET
+    description = EXCLUDED.description,
+    task_type = EXCLUDED.task_type,
+    target = EXCLUDED.target,
+    minute_interval = EXCLUDED.minute_interval,
+    run_at_second = EXCLUDED.run_at_second,
+    updated_at = NOW();
+
+INSERT INTO public.cron_tasks (
+    name,
+    description,
+    task_type,
+    target,
+    batch_size,
+    second_interval,
+    minute_interval,
+    hour_interval,
+    run_at_hour,
+    run_at_minute,
+    run_at_second,
+    run_on_dow,
+    run_on_day
+) VALUES (
+    'reconcile_build_status_queue',
+    'Process build status reconciliation queue',
+    'function_queue',
+    '["cron_reconcile_build_status"]',
+    null,
+    null,
+    5,
+    null,
+    null,
+    null,
+    0,
+    null,
+    null
+)
+ON CONFLICT (name) DO UPDATE SET
+    description = EXCLUDED.description,
+    task_type = EXCLUDED.task_type,
+    target = EXCLUDED.target,
+    minute_interval = EXCLUDED.minute_interval,
+    run_at_second = EXCLUDED.run_at_second,
+    updated_at = NOW();