fix(api): defer docker startup tasks until ready (#1952)

## Summary
- move Docker startup work out of the Nest bootstrap critical path and
schedule it only after the API is listening
- replace the `@Timeout` decorators with a dedicated startup task helper
that logs failures without blocking readiness
- trim the temporary bootstrap progress logs while keeping the
ready-signal and deferred Docker startup behavior intact
- add focused coverage for delayed execution, rejection logging, and
missing-provider behavior

## Testing
- `pnpm --dir api test
src/unraid-api/graph/resolvers/docker/docker-startup-tasks.spec.ts`
- `pnpm --dir api type-check`

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Chores**
* Increased application startup timeout budget (15s → 30s) to improve
initialization robustness.
* Added safeguards to prevent duplicate "process ready" signals and
centralized readiness signaling.

* **New Features**
* Added a background startup task scheduler and listener to run template
scanning and container digest refresh after the app is ready; failures
are logged.

* **Tests**
* Added tests for startup task scheduling, execution timing, and failure
logging.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: elibosley

api/src/index.ts

@@ -30,8 +30,7 @@ import { StateManager } from '@app/store/watch/state-watch.js';
 
 let server: NestFastifyApplication<RawServerDefault> | null = null;
 
-// PM2 listen_timeout is 15 seconds (ecosystem.config.json)
-// We use 13 seconds as our total budget to ensure our timeout triggers before PM2 kills us
+// PM2 listen_timeout is 30 seconds (ecosystem.config.json)
 const TOTAL_STARTUP_BUDGET_MS = 30_000;
 // Reserve time for the NestJS bootstrap (the most critical and time-consuming operation)
 const BOOTSTRAP_RESERVED_MS = 20_000;

api/src/unraid-api/app/app-lifecycle.events.ts

@@ -0,0 +1,5 @@
+export const APP_READY_EVENT = 'app.ready';
+
+export interface AppReadyEvent {
+    reason: 'nestjs-server-listening';
+}

api/src/unraid-api/graph/resolvers/docker/container-status.job.ts

@@ -1,5 +1,5 @@
 import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
-import { SchedulerRegistry, Timeout } from '@nestjs/schedule';
+import { SchedulerRegistry } from '@nestjs/schedule';
 
 import { CronJob } from 'cron';
 
@@ -41,7 +41,6 @@ export class ContainerStatusJob {
     /**
      * Refresh container digests 5 seconds after application start.
      */
-    @Timeout(5_000)
     async refreshContainerDigestsAfterStartup() {
         if (!this.dockerConfigService.enabled()) return;
         await this.dockerManifestService.refreshDigests();

api/src/unraid-api/graph/resolvers/docker/docker-startup-tasks.spec.ts

@@ -0,0 +1,123 @@
+import { Test } from '@nestjs/testing';
+
+import { describe, expect, it, vi } from 'vitest';
+
+import type { AppReadyEvent } from '@app/unraid-api/app/app-lifecycle.events.js';
+import { ContainerStatusJob } from '@app/unraid-api/graph/resolvers/docker/container-status.job.js';
+import {
+    DockerStartupTasksListener,
+    scheduleDockerStartupTasks,
+} from '@app/unraid-api/graph/resolvers/docker/docker-startup-tasks.js';
+import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js';
+
+describe('scheduleDockerStartupTasks', () => {
+    it('schedules docker startup work after the provided delay', async () => {
+        vi.useFakeTimers();
+
+        const bootstrapScan = vi.fn().mockResolvedValue(undefined);
+        const refreshContainerDigestsAfterStartup = vi.fn().mockResolvedValue(undefined);
+        const logger = {
+            info: vi.fn(),
+            warn: vi.fn(),
+        };
+
+        scheduleDockerStartupTasks(
+            {
+                dockerTemplateScannerService: { bootstrapScan },
+                containerStatusJob: { refreshContainerDigestsAfterStartup },
+            },
+            logger,
+            250
+        );
+
+        expect(bootstrapScan).not.toHaveBeenCalled();
+        expect(refreshContainerDigestsAfterStartup).not.toHaveBeenCalled();
+
+        await vi.advanceTimersByTimeAsync(250);
+
+        expect(bootstrapScan).toHaveBeenCalledTimes(1);
+        expect(refreshContainerDigestsAfterStartup).toHaveBeenCalledTimes(1);
+
+        vi.useRealTimers();
+    });
+
+    it('warns when a background docker startup task rejects', async () => {
+        vi.useFakeTimers();
+
+        const backgroundError = new Error('docker unavailable');
+        const logger = {
+            info: vi.fn(),
+            warn: vi.fn(),
+        };
+
+        scheduleDockerStartupTasks(
+            {
+                dockerTemplateScannerService: {
+                    bootstrapScan: vi.fn().mockRejectedValue(backgroundError),
+                },
+                containerStatusJob: {
+                    refreshContainerDigestsAfterStartup: vi.fn().mockResolvedValue(undefined),
+                },
+            },
+            logger,
+            250
+        );
+
+        await vi.advanceTimersByTimeAsync(250);
+        await vi.runAllTicks();
+
+        expect(logger.warn).toHaveBeenCalledWith(
+            backgroundError,
+            'Docker template bootstrap scan failed'
+        );
+
+        vi.useRealTimers();
+    });
+
+    it('does nothing when docker providers are unavailable', () => {
+        const logger = {
+            info: vi.fn(),
+            warn: vi.fn(),
+        };
+
+        expect(() => scheduleDockerStartupTasks({}, logger)).not.toThrow();
+        expect(logger.info).not.toHaveBeenCalled();
+        expect(logger.warn).not.toHaveBeenCalled();
+    });
+});
+
+describe('DockerStartupTasksListener', () => {
+    it('schedules docker startup work when the app ready event is emitted', async () => {
+        vi.useFakeTimers();
+
+        const bootstrapScan = vi.fn().mockResolvedValue(undefined);
+        const refreshContainerDigestsAfterStartup = vi.fn().mockResolvedValue(undefined);
+        const module = await Test.createTestingModule({
+            providers: [
+                DockerStartupTasksListener,
+                {
+                    provide: DockerTemplateScannerService,
+                    useValue: { bootstrapScan },
+                },
+                {
+                    provide: ContainerStatusJob,
+                    useValue: { refreshContainerDigestsAfterStartup },
+                },
+            ],
+        }).compile();
+        const listener = module.get(DockerStartupTasksListener);
+        const event: AppReadyEvent = {
+            reason: 'nestjs-server-listening',
+        };
+
+        listener.handleAppReady(event);
+
+        await vi.advanceTimersByTimeAsync(5_000);
+
+        expect(bootstrapScan).toHaveBeenCalledTimes(1);
+        expect(refreshContainerDigestsAfterStartup).toHaveBeenCalledTimes(1);
+
+        await module.close();
+        vi.useRealTimers();
+    });
+});

api/src/unraid-api/graph/resolvers/docker/docker-startup-tasks.ts

@@ -0,0 +1,67 @@
+import { Injectable } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+
+import type { AppReadyEvent } from '@app/unraid-api/app/app-lifecycle.events.js';
+import { apiLogger } from '@app/core/log.js';
+import { APP_READY_EVENT } from '@app/unraid-api/app/app-lifecycle.events.js';
+import { ContainerStatusJob } from '@app/unraid-api/graph/resolvers/docker/container-status.job.js';
+import { DockerTemplateScannerService } from '@app/unraid-api/graph/resolvers/docker/docker-template-scanner.service.js';
+
+const DEFAULT_DOCKER_STARTUP_DELAY_MS = 5_000;
+
+interface DockerStartupTasksDependencies {
+    containerStatusJob?: Pick<ContainerStatusJob, 'refreshContainerDigestsAfterStartup'> | null;
+    dockerTemplateScannerService?: Pick<DockerTemplateScannerService, 'bootstrapScan'> | null;
+}
+
+interface DockerStartupLogger {
+    info: (message: string, ...args: unknown[]) => void;
+    warn: (error: unknown, message: string, ...args: unknown[]) => void;
+}
+
+export const scheduleDockerStartupTasks = (
+    { dockerTemplateScannerService, containerStatusJob }: DockerStartupTasksDependencies,
+    logger: DockerStartupLogger,
+    delayMs = DEFAULT_DOCKER_STARTUP_DELAY_MS
+): void => {
+    if (!dockerTemplateScannerService && !containerStatusJob) {
+        return;
+    }
+
+    logger.info('Scheduling Docker startup tasks to run in %dms', delayMs);
+
+    if (dockerTemplateScannerService) {
+        setTimeout(() => {
+            void dockerTemplateScannerService.bootstrapScan().catch((error: unknown) => {
+                logger.warn(error, 'Docker template bootstrap scan failed');
+            });
+        }, delayMs);
+    }
+
+    if (containerStatusJob) {
+        setTimeout(() => {
+            void containerStatusJob.refreshContainerDigestsAfterStartup().catch((error: unknown) => {
+                logger.warn(error, 'Docker container digest refresh after startup failed');
+            });
+        }, delayMs);
+    }
+};
+
+@Injectable()
+export class DockerStartupTasksListener {
+    constructor(
+        private readonly dockerTemplateScannerService: DockerTemplateScannerService,
+        private readonly containerStatusJob: ContainerStatusJob
+    ) {}
+
+    @OnEvent(APP_READY_EVENT)
+    handleAppReady(_event: AppReadyEvent): void {
+        scheduleDockerStartupTasks(
+            {
+                dockerTemplateScannerService: this.dockerTemplateScannerService,
+                containerStatusJob: this.containerStatusJob,
+            },
+            apiLogger
+        );
+    }
+}

api/src/unraid-api/graph/resolvers/docker/docker-template-scanner.service.ts

@@ -1,5 +1,4 @@
 import { Injectable, Logger } from '@nestjs/common';
-import { Timeout } from '@nestjs/schedule';
 import { readdir, readFile } from 'fs/promises';
 import { join } from 'path';
 
@@ -33,7 +32,6 @@ export class DockerTemplateScannerService {
         private readonly dockerService: DockerService
     ) {}
 
-    @Timeout(5_000)
     async bootstrapScan(attempt = 1, maxAttempts = 5): Promise<void> {
         if (!ENABLE_NEXT_DOCKER_RELEASE) {
             return;

api/src/unraid-api/graph/resolvers/docker/docker.module.ts

@@ -10,6 +10,7 @@ import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/do
 import { DockerNetworkService } from '@app/unraid-api/graph/resolvers/docker/docker-network.service.js';
 import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js';
 import { DockerPortService } from '@app/unraid-api/graph/resolvers/docker/docker-port.service.js';
+import { DockerStartupTasksListener } from '@app/unraid-api/graph/resolvers/docker/docker-startup-tasks.js';
 import { DockerStatsService } from '@app/unraid-api/graph/resolvers/docker/docker-stats.service.js';
 import { DockerTailscaleService } from '@app/unraid-api/graph/resolvers/docker/docker-tailscale.service.js';
 import { DockerTemplateIconService } from '@app/unraid-api/graph/resolvers/docker/docker-template-icon.service.js';
@@ -35,6 +36,7 @@ import { ServicesModule } from '@app/unraid-api/graph/services/services.module.j
         DockerConfigService,
         DockerTemplateScannerService,
         DockerTemplateIconService,
+        DockerStartupTasksListener,
         DockerStatsService,
         DockerTailscaleService,
         DockerLogService,

api/src/unraid-api/main.ts

@@ -2,20 +2,66 @@ import type { NestFastifyApplication } from '@nestjs/platform-fastify';
 import { ValidationPipe } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { NestFactory } from '@nestjs/core';
+import { EventEmitter2 } from '@nestjs/event-emitter';
 import { FastifyAdapter } from '@nestjs/platform-fastify/index.js';
 
 import fastifyCookie from '@fastify/cookie';
 import fastifyHelmet from '@fastify/helmet';
 import { LoggerErrorInterceptor, Logger as PinoLogger } from 'nestjs-pino';
 
+import type { AppReadyEvent } from '@app/unraid-api/app/app-lifecycle.events.js';
 import { apiLogger } from '@app/core/log.js';
 import { LOG_LEVEL, PORT } from '@app/environment.js';
-import { AppModule } from '@app/unraid-api/app/app.module.js';
-import { GraphQLExceptionsFilter } from '@app/unraid-api/exceptions/graphql-exceptions.filter.js';
-import { HttpExceptionFilter } from '@app/unraid-api/exceptions/http-exceptions.filter.js';
+import { APP_READY_EVENT } from '@app/unraid-api/app/app-lifecycle.events.js';
+
+const READY_SIGNAL_SENT_SYMBOL = Symbol.for('unraid-api.processReadySignalSent');
+
+type ReadySignalAwareGlobal = typeof globalThis & {
+    [READY_SIGNAL_SENT_SYMBOL]?: boolean;
+};
+
+const readySignalState = globalThis as ReadySignalAwareGlobal;
+
+const signalProcessReady = (reason: string): boolean => {
+    if (readySignalState[READY_SIGNAL_SENT_SYMBOL] === true) {
+        return false;
+    }
+
+    if (!process.send) {
+        apiLogger.warn(
+            new Error('process.send is unavailable'),
+            'Unable to send PM2 ready signal while %s',
+            reason
+        );
+        return false;
+    }
+
+    try {
+        const readySignalSent = process.send('ready');
+
+        if (!readySignalSent) {
+            apiLogger.warn(
+                'process.send returned false while sending PM2 ready signal while %s',
+                reason
+            );
+            return false;
+        }
+
+        readySignalState[READY_SIGNAL_SENT_SYMBOL] = true;
+        apiLogger.info('Sent PM2 ready signal while %s', reason);
+        return true;
+    } catch (error: unknown) {
+        apiLogger.error(error, 'Failed to send PM2 ready signal while %s', reason);
+        return false;
+    }
+};
 
 export async function bootstrapNestServer(): Promise<NestFastifyApplication> {
-    apiLogger.debug('Creating Nest Server');
+    const [{ AppModule }, { GraphQLExceptionsFilter }, { HttpExceptionFilter }] = await Promise.all([
+        import('@app/unraid-api/app/app.module.js'),
+        import('@app/unraid-api/exceptions/graphql-exceptions.filter.js'),
+        import('@app/unraid-api/exceptions/http-exceptions.filter.js'),
+    ]);
 
     const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter(), {
         bufferLogs: false,
@@ -144,14 +190,12 @@ export async function bootstrapNestServer(): Promise<NestFastifyApplication> {
     // This 'ready' signal tells pm2 that the api has started.
     // PM2 documents this as Graceful Start or Clean Restart.
     // See https://pm2.keymetrics.io/docs/usage/signals-clean-restart/
-    if (process.send) {
-        process.send('ready');
-    } else {
-        apiLogger.warn(
-            'Warning: process.send is unavailable. This will affect IPC communication with PM2.'
-        );
-    }
+    signalProcessReady('NestJS server is now listening');
     apiLogger.info('Nest Server is now listening');
+    const appReadyEvent: AppReadyEvent = {
+        reason: 'nestjs-server-listening',
+    };
+    app.get(EventEmitter2).emit(APP_READY_EVENT, appReadyEvent);
 
     return app;
 }