Merge pull request Sofie-Automation#1533 from nrkno/feat/health-endpoints-upstream

Add health endpoints to Mos- & Playout-Gateway

Author: nytamin

packages/mos-gateway/src/connector.ts

@@ -5,19 +5,27 @@ import {
 	PeripheralDeviceId,
 	loadCertificatesFromDisk,
 	CertificatesConfig,
+	stringifyError,
+	HealthConfig,
+	HealthEndpoints,
+	IConnector,
 } from '@sofie-automation/server-core-integration'
 
 export interface Config {
 	certificates: CertificatesConfig
 	device: DeviceConfig
 	core: CoreConfig
 	mos: MosConfig
+	health: HealthConfig
 }
 export interface DeviceConfig {
 	deviceId: PeripheralDeviceId
 	deviceToken: string
 }
-export class Connector {
+export class Connector implements IConnector {
+	public initialized = false
+	public initializedError: string | undefined = undefined
+
 	private mosHandler: MosHandler | undefined
 	private coreHandler: CoreHandler | undefined
 	private _config: Config | undefined
@@ -38,11 +46,18 @@ export class Connector {
 			this._logger.info('Initializing Core...')
 			await this.initCore(certificates)
 
+			if (!this.coreHandler) throw Error('coreHandler is undefined!')
+
+			new HealthEndpoints(this, this.coreHandler, config.health)
+
 			this._logger.info('Initializing Mos...')
 			await this.initMos()
 
 			this._logger.info('Initialization done')
+			this.initialized = true
 		} catch (e: any) {
+			this.initializedError = stringifyError(e)
+
 			this._logger.error('Error during initialization:', e, e.stack)
 
 			this._logger.info('Shutting down in 10 seconds!')

packages/mos-gateway/src/coreHandler.ts

@@ -10,6 +10,7 @@ import {
 	stringifyError,
 	PeripheralDevicePubSub,
 	PeripheralDevicePubSubCollectionsNames,
+	ICoreHandler,
 } from '@sofie-automation/server-core-integration'
 import * as Winston from 'winston'
 
@@ -29,14 +30,16 @@ export interface CoreConfig {
 /**
  * Represents a connection between mos-integration and Core
  */
-export class CoreHandler {
+export class CoreHandler implements ICoreHandler {
 	core: CoreConnection | undefined
 	logger: Winston.Logger
 	public _observers: Array<Observer<any>> = []
+	public connectedToCore = false
 	private _deviceOptions: DeviceConfig
 	private _coreMosHandlers: Array<CoreMosDeviceHandler> = []
 	private _onConnected?: () => any
 	private _isInitialized = false
+	private _isDestroyed = false
 	private _executedFunctions = new Set<PeripheralDeviceCommandId>()
 	private _coreConfig?: CoreConfig
 	private _certificates?: Buffer[]
@@ -54,10 +57,12 @@ export class CoreHandler {
 
 		this.core.onConnected(() => {
 			this.logger.info('Core Connected!')
+			this.connectedToCore = true
 			if (this._isInitialized) this.onConnectionRestored()
 		})
 		this.core.onDisconnected(() => {
 			this.logger.info('Core Disconnected!')
+			this.connectedToCore = false
 		})
 		this.core.onError((err) => {
 			this.logger.error('Core Error: ' + (typeof err === 'string' ? err : err.message || err.toString()))
@@ -74,31 +79,45 @@ export class CoreHandler {
 		}
 		await this.core.init(ddpConfig)
 
-		if (!this.core) {
-			throw Error('core is undefined!')
-		}
-
-		this.core
-			.setStatus({
-				statusCode: StatusCode.GOOD,
-				// messages: []
-			})
-			.catch((e) => this.logger.warn('Error when setting status:' + e))
-		// nothing
-
 		await this.setupSubscriptionsAndObservers()
 
 		this._isInitialized = true
+
+		await this.updateCoreStatus()
+	}
+	getCoreStatus(): {
+		statusCode: StatusCode
+		messages: string[]
+	} {
+		let statusCode = StatusCode.GOOD
+		const messages: string[] = []
+
+		if (!this._isInitialized) {
+			statusCode = StatusCode.BAD
+			messages.push('Starting up...')
+		}
+		if (this._isDestroyed) {
+			statusCode = StatusCode.FATAL
+			messages.push('Shut down')
+		}
+		return {
+			statusCode,
+			messages,
+		}
 	}
+	async updateCoreStatus(): Promise<void> {
+		if (!this.core) throw Error('core is undefined!')
+
+		await this.core.setStatus(this.getCoreStatus())
+	}
+
 	async dispose(): Promise<void> {
+		this._isDestroyed = true
 		if (!this.core) {
 			throw Error('core is undefined!')
 		}
 
-		await this.core.setStatus({
-			statusCode: StatusCode.FATAL,
-			messages: ['Shutting down'],
-		})
+		await this.updateCoreStatus()
 
 		await Promise.all(
 			this._coreMosHandlers.map(async (cmh: CoreMosDeviceHandler) => {

packages/mos-gateway/src/index.ts

@@ -14,6 +14,7 @@ let deviceToken: string = process.env.DEVICE_TOKEN || ''
 let disableWatchdog: boolean = process.env.DISABLE_WATCHDOG === '1' || false
 let unsafeSSL: boolean = process.env.UNSAFE_SSL === '1' || false
 const certs: string[] = (process.env.CERTIFICATES || '').split(';') || []
+let healthPort: number | undefined = parseInt(process.env.HEALTH_PORT + '') || undefined
 let debug = false
 let printHelp = false
 
@@ -46,6 +47,8 @@ process.argv.forEach((val) => {
 	} else if (val.match(/-unsafeSSL/i)) {
 		// Will cause the Node applocation to blindly accept all certificates. Not recommenced unless in local, controlled networks.
 		unsafeSSL = true
+	} else if (prevProcessArg.match(/-healthPort/i)) {
+		healthPort = parseInt(val)
 	}
 	prevProcessArg = nextPrevProcessArg + ''
 })
@@ -207,6 +210,9 @@ const config: Config = {
 		port: port,
 		watchdog: !disableWatchdog,
 	},
+	health: {
+		port: healthPort,
+	},
 	mos: {
 		self: {
 			debug: debug,

packages/playout-gateway/src/config.ts

@@ -20,6 +20,8 @@ let influxUser: string | undefined = process.env.INFLUX_USER || 'sofie'
 let influxPassword: string | undefined = process.env.INFLUX_PASSWORD || undefined
 let influxDatabase: string | undefined = process.env.INFLUX_DB || 'sofie'
 
+let healthPort: number | undefined = parseInt(process.env.HEALTH_PORT + '') || undefined
+
 let prevProcessArg = ''
 process.argv.forEach((val) => {
 	val = val + ''
@@ -50,14 +52,18 @@ process.argv.forEach((val) => {
 		influxPassword = val
 	} else if (prevProcessArg.match(/-influxDatabase/i)) {
 		influxDatabase = val
+	} else if (prevProcessArg.match(/-healthPort/i)) {
+		healthPort = parseInt(val)
 
 		// arguments with no options:
 	} else if (val.match(/-disableWatchdog/i)) {
 		disableWatchdog = true
 	} else if (val.match(/-disableAtemUpload/i)) {
 		disableAtemUpload = true
 	} else if (val.match(/-unsafeSSL/i)) {
-		// Will cause the Node applocation to blindly accept all certificates. Not recommenced unless in local, controlled networks.
+		// Will cause the Node application to blindly accept all certificates.
+		// Not recommenced unless in local, controlled networks.
+		// Instead use "-certificates cert1 cert2"
 		unsafeSSL = true
 	}
 	prevProcessArg = nextPrevProcessArg + ''
@@ -85,6 +91,9 @@ const config: Config = {
 		password: influxPassword,
 		database: influxDatabase,
 	},
+	health: {
+		port: healthPort,
+	},
 }
 
 export { config, logPath, logLevel, disableWatchdog, disableAtemUpload }

packages/playout-gateway/src/connector.ts

@@ -6,6 +6,10 @@ import {
 	CertificatesConfig,
 	PeripheralDeviceId,
 	loadCertificatesFromDisk,
+	stringifyError,
+	HealthConfig,
+	HealthEndpoints,
+	IConnector,
 } from '@sofie-automation/server-core-integration'
 
 export interface Config {
@@ -14,13 +18,17 @@ export interface Config {
 	core: CoreConfig
 	tsr: TSRConfig
 	influx: InfluxConfig
+	health: HealthConfig
 }
 
 export interface DeviceConfig {
 	deviceId: PeripheralDeviceId
 	deviceToken: string
 }
-export class Connector {
+export class Connector implements IConnector {
+	public initialized = false
+	public initializedError: string | undefined = undefined
+
 	private tsrHandler: TSRHandler | undefined
 	private coreHandler: CoreHandler | undefined
 	private _logger: Logger
@@ -38,6 +46,8 @@ export class Connector {
 
 			this._logger.info('Initializing Core...')
 			this.coreHandler = new CoreHandler(this._logger, config.device)
+			new HealthEndpoints(this, this.coreHandler, config.health)
+
 			await this.coreHandler.init(config.core, this._certificates)
 			this._logger.info('Core initialized')
 
@@ -47,12 +57,15 @@ export class Connector {
 			this._logger.info('TSR initialized')
 
 			this._logger.info('Initialization done')
+			this.initialized = true
 			return
 		} catch (e: any) {
 			this._logger.error('Error during initialization:')
 			this._logger.error(e)
 			this._logger.error(e.stack)
 
+			this.initializedError = stringifyError(e)
+
 			try {
 				if (this.coreHandler) {
 					this.coreHandler.destroy().catch(this._logger.error)

packages/playout-gateway/src/coreHandler.ts

@@ -11,6 +11,7 @@ import {
 	stringifyError,
 	PeripheralDevicePubSub,
 	PeripheralDevicePubSubCollectionsNames,
+	ICoreHandler,
 } from '@sofie-automation/server-core-integration'
 import { MediaObject, DeviceOptionsAny, ActionExecutionResult } from 'timeline-state-resolver'
 import _ from 'underscore'
@@ -40,7 +41,7 @@ export interface MemoryUsageReport {
 /**
  * Represents a connection between the Gateway and Core
  */
-export class CoreHandler {
+export class CoreHandler implements ICoreHandler {
 	core!: CoreConnection
 	logger: Logger
 	public _observers: Array<Observer<any>> = []
@@ -59,6 +60,8 @@ export class CoreHandler {
 	private _statusInitialized = false
 	private _statusDestroyed = false
 
+	public connectedToCore = false
+
 	constructor(logger: Logger, deviceOptions: DeviceConfig) {
 		this.logger = logger
 		this._deviceOptions = deviceOptions
@@ -73,11 +76,13 @@ export class CoreHandler {
 
 		this.core.onConnected(() => {
 			this.logger.info('Core Connected!')
+			this.connectedToCore = true
 
 			if (this._onConnected) this._onConnected()
 		})
 		this.core.onDisconnected(() => {
 			this.logger.warn('Core Disconnected!')
+			this.connectedToCore = false
 		})
 		this.core.onError((err: any) => {
 			this.logger.error('Core Error: ' + (typeof err === 'string' ? err : err.message || err.toString() || err))
@@ -375,9 +380,12 @@ export class CoreHandler {
 
 		return Object.fromEntries(this._tsrHandler.getDebugStates().entries())
 	}
-	async updateCoreStatus(): Promise<any> {
+	getCoreStatus(): {
+		statusCode: StatusCode
+		messages: string[]
+	} {
 		let statusCode = StatusCode.GOOD
-		const messages: Array<string> = []
+		const messages: string[] = []
 
 		if (!this._statusInitialized) {
 			statusCode = StatusCode.BAD
@@ -387,11 +395,13 @@ export class CoreHandler {
 			statusCode = StatusCode.BAD
 			messages.push('Shut down')
 		}
-
-		return this.core.setStatus({
-			statusCode: statusCode,
-			messages: messages,
-		})
+		return {
+			statusCode,
+			messages,
+		}
+	}
+	async updateCoreStatus(): Promise<any> {
+		return this.core.setStatus(this.getCoreStatus())
 	}
 }
 

packages/server-core-integration/package.json

@@ -69,11 +69,17 @@
 		"rundown",
 		"production"
 	],
+	"devDependencies": {
+		"@types/koa": "^3.0.0",
+		"@types/koa__router": "^12.0.4"
+	},
 	"dependencies": {
+		"@koa/router": "^14.0.0",
 		"@sofie-automation/shared-lib": "1.53.0-in-development",
 		"ejson": "^2.2.3",
 		"faye-websocket": "^0.11.4",
 		"got": "^11.8.6",
+		"koa": "^3.0.1",
 		"tslib": "^2.8.1",
 		"underscore": "^1.13.7"
 	},

packages/server-core-integration/src/index.ts

@@ -1,6 +1,8 @@
 export * from './lib/coreConnection.js'
 export * from './lib/configManifest.js'
 export * from './lib/ddpClient.js'
+export * from './lib/gateway-types.js'
+export * from './lib/health.js'
 export * from './lib/methods.js'
 export * from './lib/process.js'
 export { SubscriptionId } from './lib/subscriptions.js'

packages/server-core-integration/src/lib/gateway-types.ts

@@ -0,0 +1,11 @@
+import { StatusCode } from '@sofie-automation/shared-lib/dist/lib/status'
+
+export interface IConnector {
+	initialized: boolean
+	initializedError: string | undefined
+}
+
+export interface ICoreHandler {
+	getCoreStatus: () => { statusCode: StatusCode; messages: string[] }
+	connectedToCore: boolean
+}