fix: add retry for waiting on tools server

Author: CasLubbers

src/jwt-verification.ts

@@ -1,10 +1,17 @@
 import { createRemoteJWKSet, JWTPayload, jwtVerify } from 'jose'
 import Debug from 'debug'
 import retry from 'async-retry'
-import { cleanEnv, JWT_AUDIENCE, SSO_ISSUER, SSO_JWKS_URI } from 'src/validators'
+import {
+  cleanEnv,
+  JWT_AUDIENCE,
+  SSO_ISSUER,
+  SSO_JWKS_URI,
+  STARTUP_RETRY_COUNT,
+  STARTUP_RETRY_INTERVAL_MS,
+} from 'src/validators'
 
 const debug = Debug('otomi:jwt')
-const env = cleanEnv({ SSO_ISSUER, JWT_AUDIENCE, SSO_JWKS_URI })
+const env = cleanEnv({ SSO_ISSUER, JWT_AUDIENCE, SSO_JWKS_URI, STARTUP_RETRY_COUNT, STARTUP_RETRY_INTERVAL_MS })
 const JWKS_URL = env.SSO_JWKS_URI
 
 // Create remote JWKS - automatically caches and refreshes keys
@@ -42,7 +49,7 @@ export async function verifyJwt(token: string): Promise<AppJWTPayload> {
     }
   } catch (error: any) {
     debug('JWT verification failed:', error.message)
-    throw new Error(`JWT verification failed: ${error.message}`)
+    throw error
   }
 }
 
@@ -74,9 +81,9 @@ export async function waitForJwksReady(): Promise<void> {
       }
     },
     {
-      retries: 300, // 10 minutes total with 2s minTimeout
-      minTimeout: 2000,
-      maxTimeout: 2000,
+      retries: env.STARTUP_RETRY_COUNT,
+      minTimeout: env.STARTUP_RETRY_INTERVAL_MS,
+      maxTimeout: env.STARTUP_RETRY_INTERVAL_MS,
     },
   )
 }

src/middleware/jwt.ts

@@ -87,7 +87,6 @@ export function jwtMiddleware(): RequestHandler {
 
       return next()
     } catch (error: any) {
-      debug('JWT verification failed:', error.message)
       return res.status(401).send({
         message: 'Unauthorized',
         error: `JWT verification failed: ${error.message}`,

src/utils.ts

@@ -1,4 +1,6 @@
 import axios from 'axios'
+import retry from 'async-retry'
+import Debug from 'debug'
 import cleanDeep, { CleanOptions } from 'clean-deep'
 import { pathExists } from 'fs-extra'
 import { readdir, readFile } from 'fs/promises'
@@ -7,10 +9,14 @@ import cloneDeep from 'lodash/cloneDeep'
 import { Cluster, Dns } from 'src/otomi-models'
 import { parse, stringify } from 'yaml'
 import { BASEURL } from './constants'
-import { cleanEnv, GIT_PASSWORD } from './validators'
+import { cleanEnv, GIT_PASSWORD, STARTUP_RETRY_COUNT, STARTUP_RETRY_INTERVAL_MS } from './validators'
+
+const debug = Debug('otomi:utils')
 
 const env = cleanEnv({
   GIT_PASSWORD,
+  STARTUP_RETRY_COUNT,
+  STARTUP_RETRY_INTERVAL_MS,
 })
 
 export function arrayToObject(array: [] = [], keyName = 'name', keyValue = 'value'): Record<string, unknown> {
@@ -85,7 +91,25 @@ const valuesSchemaEndpointUrl = `${BASEURL}/apl/schema`
 let valuesSchema: Record<string, any>
 
 export const getValuesSchema = async (): Promise<Record<string, any>> => {
-  const res = await axios.get(valuesSchemaEndpointUrl)
+  debug('Fetching values schema from tools server...')
+
+  const res = await retry(
+    async () => {
+      try {
+        return await axios.get(valuesSchemaEndpointUrl)
+      } catch (error: any) {
+        debug(`Tools server not ready yet (${error.code}), retrying...`)
+        throw error
+      }
+    },
+    {
+      retries: env.STARTUP_RETRY_COUNT,
+      minTimeout: env.STARTUP_RETRY_INTERVAL_MS,
+      maxTimeout: env.STARTUP_RETRY_INTERVAL_MS,
+    },
+  )
+
+  debug('Values schema fetched successfully')
   valuesSchema = omit(res.data, ['definitions'])
   return valuesSchema
 }

src/validators.ts

@@ -146,6 +146,14 @@ export const AGENT_KIND = str({
   desc: 'The kind for AkamaiAgent custom resources',
   default: 'AkamaiAgent',
 })
+export const STARTUP_RETRY_COUNT = num({
+  desc: 'Number of retries for startup dependencies (JWKS, tools server)',
+  default: 300,
+})
+export const STARTUP_RETRY_INTERVAL_MS = num({
+  desc: 'Retry interval in milliseconds for startup dependencies',
+  default: 1000,
+})
 const { env } = process
 export function cleanEnv<T>(validators: { [K in keyof T]: ValidatorSpec<T[K]> }, options: CleanOptions<T> = {}) {
   if (env.NODE_ENV === 'test') {