Add OpenAPI health endpoint and a prometheus health gauge (#1468)

Co-authored-by: Simon Rogers <[email protected]>

Author: scriptorian

meteor/server/api/rest/v1/index.ts

@@ -20,6 +20,8 @@ import { registerRoutes as registerSystemRoutes } from './system'
 import { registerRoutes as registerBucketsRoutes } from './buckets'
 import { registerRoutes as registerSnapshotRoutes } from './snapshots'
 import { APIFactory, ServerAPIContext } from './types'
+import { getSystemStatus } from '../../../systemStatus/systemStatus'
+import { Component, ExternalStatus } from '@sofie-automation/meteor-lib/dist/api/systemStatus'
 
 function restAPIUserEvent(
 	ctx: Koa.ParameterizedContext<
@@ -193,6 +195,82 @@ koaRouter.get('/', async (ctx, next) => {
 	await next()
 })
 
+koaRouter.get('/health', async (ctx, next) => {
+	ctx.type = 'application/json'
+	const systemStatus = await getSystemStatus(null)
+	const coreVersion = systemStatus._internal.versions['core'] ?? 'unknown'
+	const blueprint = Object.keys(systemStatus._internal.versions).find((component) =>
+		component.startsWith('blueprint')
+	)
+	const blueprintsVersion = blueprint ? systemStatus._internal.versions[blueprint] : 'unknown'
+
+	interface ComponentStatus {
+		name: string
+		updated: string
+		status: ExternalStatus
+		version?: string
+		components?: ComponentStatus[]
+		statusMessage?: string
+	}
+
+	// Array of all devices that have a parentId
+	const subComponents =
+		systemStatus.components?.filter((c) => c.instanceId !== undefined && c.parentId !== undefined) ?? []
+
+	function mapComponents(components?: Component[]): ComponentStatus[] | undefined {
+		return (
+			components?.map((c) => {
+				const version = c._internal.versions['_process']
+				const children = subComponents.filter((sub) => sub.parentId === c.instanceId)
+				return {
+					name: c.name,
+					updated: c.updated,
+					status: c.status,
+					version: version ?? undefined,
+					components: children.length ? mapComponents(children) : undefined,
+					statusMessage: c.statusMessage?.length ? c.statusMessage : undefined,
+				}
+			}) ?? undefined
+		)
+	}
+
+	// Patch the component statusMessage to be from the _internal field if required
+	const allComponentsPatched = systemStatus.components?.map((c) => {
+		return {
+			...c,
+			statusMessage: c.statusMessage ?? (c.status !== 'OK' ? c._internal.messages.join(', ') : undefined),
+		}
+	})
+
+	// Report status for all devices that are not children and any non-devices that are not OK
+	const componentStatus =
+		mapComponents(
+			allComponentsPatched?.filter(
+				(c) => (c.instanceId !== undefined || c.status !== 'OK') && c.parentId === undefined
+			)
+		) ?? []
+
+	const allStatusMessages =
+		allComponentsPatched // include children by not using componentStatus here
+			?.filter((c) => c.statusMessage !== undefined)
+			.map((c) => `${c.name}: ${c.statusMessage}`)
+			.join('; ') ?? ''
+
+	const response = ClientAPI.responseSuccess({
+		name: systemStatus.name,
+		updated: systemStatus.updated,
+		status: systemStatus.status,
+		version: coreVersion,
+		blueprintsVersion: blueprintsVersion,
+		components: componentStatus,
+		statusMessage: allStatusMessages,
+	})
+
+	ctx.body = JSON.stringify({ status: response.success, result: response.result })
+	ctx.status = response.success
+	await next()
+})
+
 registerBlueprintsRoutes(sofieAPIRequest)
 registerDevicesRoutes(sofieAPIRequest)
 registerPlaylistsRoutes(sofieAPIRequest)

meteor/server/main.ts

@@ -37,6 +37,7 @@ import './migration/migrations'
 import './api/playout/debug'
 import './performanceMonitor'
 import './systemStatus/api'
+import './systemStatus/prometheusHealthGauge'
 import './api/user'
 import './api/organizations'
 import './api/serviceMessages/api'

meteor/server/systemStatus/prometheusHealthGauge.ts

@@ -0,0 +1,24 @@
+import { MetricsGauge } from '@sofie-automation/corelib/dist/prometheus'
+import { getSystemStatus } from './systemStatus'
+
+export const healthGauge = new MetricsGauge({
+	name: 'sofie_health_status',
+	help: 'Health status of Sofie application and its components',
+	labelNames: ['name', 'version'] as const,
+	async collect() {
+		const systemStatus = await getSystemStatus(null)
+
+		const statusValues = { OK: 0, FAIL: 1, WARNING: 2, UNDEFINED: 3 }
+		this.labels({
+			name: systemStatus.name,
+			version: systemStatus._internal.versions['core'],
+		}).set(statusValues[systemStatus.status])
+
+		systemStatus.components?.forEach((c) => {
+			this.labels({
+				name: c.name,
+				version: c._internal.versions['_process'] ?? '',
+			}).set(statusValues[c.status])
+		})
+	},
+})

meteor/server/systemStatus/systemStatus.ts

@@ -137,6 +137,7 @@ function getSystemStatusForDevice(device: PeripheralDevice): StatusResponse {
 	const so: StatusResponse = {
 		name: device.name,
 		instanceId: device._id,
+		parentId: device.parentDeviceId ?? undefined,
 		status: 'UNDEFINED',
 		updated: new Date(device.lastSeen).toISOString(),
 		_status: deviceStatus,

packages/meteor-lib/src/api/systemStatus.ts

@@ -27,6 +27,7 @@ export interface StatusResponseBase {
 
 	statusMessage?: string // Tekstlig beskrivelse av status. (Eks: OK, Running, Standby, Completed successfully, 2/3 nodes running, Slow response time).
 	instanceId?: ProtectedString<any>
+	parentId?: ProtectedString<any>
 	utilises?: Array<string>
 	consumers?: Array<string>
 	version?: '3' // version of healthcheck

packages/openapi/api/actions.yaml

@@ -20,6 +20,8 @@ tags:
 paths:
   /:
     $ref: 'definitions/sofie.yaml#/resources/index'
+  /health:
+    $ref: 'definitions/sofie.yaml#/resources/health'
   /system/blueprint:
     $ref: 'definitions/sofie.yaml#/resources/systemBlueprint'
   /system/migrations:

packages/openapi/api/definitions/sofie.yaml

@@ -131,6 +131,21 @@ resources:
                     type: string
         500:
           $ref: '#/components/responses/internalServerError'
+  health:
+    get:
+      operationId: getHealth
+      tags:
+        - sofie
+      summary: Gets the current health status of Sofie and all its components
+      responses:
+        200:
+          description: Command successfully handled - returns an object with detailed health status
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/healthStatus'
+        500:
+          $ref: '#/components/responses/internalServerError'
 components:
   schemas:
     migrationInputs:
@@ -204,6 +219,88 @@ components:
         - attributeId
         - migrationValue
       additionalProperties: false
+    healthStatus:
+      type: object
+      properties:
+        name:
+          type: string
+          description: Name of the running system
+          example: Sofie Automation system
+        updated:
+          type: string
+          format: date-time
+          description: Time when the status of Sofie was updated
+          example: '2023-11-29T16:50:06.057Z'
+        status:
+          type: string
+          enum: [OK, FAIL, WARNING, UNDEFINED]
+          description: Sofie status string
+          example: OK
+        version:
+          type: string
+          description: Sofie core software version
+          example: '1.50.10'
+        blueprintsVersion:
+          type: string
+          description: Sofie blueprints version
+          example: '1.0.60'
+        components:
+          type: array
+          description: Array of components that are part of the Sofie software
+          items:
+            $ref: '#/components/schemas/healthComponentStatus'
+        statusMessage:
+          type: string
+          description: Concatenation of Sofie status and all component statuses, separated by semicolons
+          example: 'Playout gateway: Disconnected'
+      required:
+        - name
+        - updated
+        - status
+        - version
+        - blueprintsVersion
+        - components
+        - statusMessage
+      additionalProperties: false
+    healthComponentStatus:
+      type: object
+      properties:
+        name:
+          type: string
+          description: Name of the component
+          example: Playout Gateway
+        updated:
+          type: string
+          format: date-time
+          description: Time when the component status was updated
+        status:
+          type: string
+          enum: [OK, FAIL, WARNING, UNDEFINED]
+          description: Component status string
+          example: OK
+        version:
+          type: string
+          description: Component software version
+          example: '0.1.13'
+        components:
+          type: array
+          description: Array of components that are children of this component. Can recurse - components with no child will have no components member
+          items:
+            type: object
+            description: Components conforming to the same definition as the parent object
+          example:
+            - name: atem
+              updated: '2023-11-28T15:17:21.712Z'
+              status: OK
+        statusMessage:
+          type: string
+          description: Status messages for this component
+          example: Disconnected
+      required:
+        - name
+        - updated
+        - status
+      additionalProperties: false
   responses:
     postSuccess:
       description: POST success.

packages/openapi/install_swagger.js renamed to packages/openapi/install_swagger.mjs

@@ -1,7 +1,7 @@
-const fs = require('fs/promises')
-const fs_constants = require('fs').constants
-// eslint-disable-next-line n/no-unpublished-require
-const wget = require('wget-improved')
+import fs from 'fs/promises'
+import { exit } from 'process'
+// eslint-disable-next-line n/no-unpublished-import
+import wget from 'wget-improved'
 
 async function get(url, path) {
 	let totalBytes = 0
@@ -23,7 +23,7 @@ async function get(url, path) {
 			}
 		})
 		download.on('end', function (output) {
-			process.stdout.write(`Downloaded 100% of '${path}'. Total length ${totalBytes} bytes.\n`)
+			process.stdout.write(`${output}, total length ${totalBytes} bytes.\n`)
 			resolve(output)
 		})
 	})
@@ -37,11 +37,13 @@ async function checkInstall() {
 	})
 
 	const srcPath =
-		'https://repo1.maven.org/maven2/io/swagger/codegen/v3/swagger-codegen-cli/3.0.34/swagger-codegen-cli-3.0.34.jar'
+		'https://repo1.maven.org/maven2/io/swagger/codegen/v3/swagger-codegen-cli/3.0.51/swagger-codegen-cli-3.0.51.jar'
 	const swaggerFilename = 'swagger-codegen-cli.jar'
 	await fs
-		.access(`jars/${swaggerFilename}`, fs_constants.R_OK)
+		.access(`jars/${swaggerFilename}`, fs.constants.R_OK)
 		.catch(async () => get(srcPath, `jars/${swaggerFilename}`))
 }
 
-checkInstall()
+await checkInstall()
+
+exit(0)

packages/openapi/package.json

@@ -19,26 +19,27 @@
 		"lint-fix": "run lint --fix",
 		"genclient:ts": "run -T rimraf client/ts && openapi-generator-cli generate -i ./api/actions.yaml -o client/ts -g typescript-fetch -p supportsES6=true",
 		"genclient:rs": "run -T rimraf client/rs && openapi-generator-cli generate -i ./api/actions.yaml -o client/rs -g rust",
-		"gendocs": "run -T rimraf docs && node install_swagger.js && java -jar ./jars/swagger-codegen-cli.jar generate -i ./api/actions.yaml -l html2 -o ./docs",
-		"genserver": "run -T rimraf server && node install_swagger.js && java -jar ./jars/swagger-codegen-cli.jar generate -i ./api/actions.yaml -l nodejs-server -o server && cd server && npm install && cd ../",
+		"genclient:cs": "run -T rimraf client/cs && openapi-generator-cli generate -i ./api/actions.yaml -o client/cs -g csharp",
+		"gendocs": "run -T rimraf docs && node install_swagger.mjs && java -jar ./jars/swagger-codegen-cli.jar generate -i ./api/actions.yaml -l html2 -o ./docs",
+		"genserver": "run -T rimraf server && node install_swagger.mjs && java -jar ./jars/swagger-codegen-cli.jar generate -i ./api/actions.yaml -l nodejs-server -o server && cd server && npm install && cd ../",
 		"runserver": "run genserver && cd server && node index.js",
 		"test": "run lint && run genclient:ts && run unit",
 		"unit:no-server": "node --experimental-fetch ../node_modules/jest/bin/jest.js --detectOpenHandles --forceExit"
 	},
 	"prettier": "@sofie-automation/code-standard-preset/.prettierrc.json",
 	"engines": {
-		"node": ">=21"
+		"node": ">=22.13.1"
 	},
 	"files": [
 		"/api",
 		"/dist",
-		"install_swagger.js"
+		"install_swagger.mjs"
 	],
 	"dependencies": {
 		"tslib": "^2.8.1"
 	},
 	"devDependencies": {
-		"@openapitools/openapi-generator-cli": "^2.16.3",
+		"@openapitools/openapi-generator-cli": "^2.20.2",
 		"eslint": "^9.18.0",
 		"eslint-plugin-yml": "^1.16.0",
 		"js-yaml": "^4.1.0",
@@ -51,5 +52,6 @@
 		"*.{ts,tsx,js,jsx,yaml}": [
 			"yarn lint:raw"
 		]
-	}
+	},
+	"packageManager": "[email protected]"
 }