feat: new readiness logic

Author: Simone Sanfratello

README.md

@@ -52,7 +52,11 @@ _Variables in bold are required._
 | P2P_CONNECTION_HANDLER_MAX_INBOUND_STREAMS | `1024` | p2p handler max incoming streams limit at the same time on each connection |
 | P2P_CONNECTION_HANDLER_MAX_OUTBOUND_STREAMS | `1024` | p2p handler max outgoing streams limit at the same time on each connection |
 | P2P_CONNECTION_TAGGED_PEERS_VALUE | `100` | p2p tagged peers default value, see [tagged peers](#tagged-peers). |
-| TELEMETRY_PORT        | `3001`        | The telemetry port number for the OpenTelemetry server to listen on.     |
+| READINESS_MAX_CONNECTIONS || TODO |
+| READINESS_MAX_PENDING_REQUEST_BLOCKS || |
+| READINESS_MAX_EVENT_LOOP_UTILIZATION || |
+| READINESS_MAX_RESPONSE_DURATION || |
+| HTTP_PORT        | `3001`        | The telemetry port number for the OpenTelemetry server to listen on.     |
 | NODE_DEBUG            |               | If it contains `aws-ipfs`, debug mode is enabled.                        |
 | LOG_LEVEL            | `info` | Logging level. |
 | LOG_PRETTY            | `false` | Enable pretty logging. |

package-lock.json

@@ -16,10 +16,10 @@
     "@aws-sdk/client-sqs": "3.231.0",
     "@aws-sdk/node-http-handler": "3.226.0",
     "@chainsafe/libp2p-noise": "10.2.0",
-    "@libp2p/mplex": "7.1.0",
-    "@libp2p/peer-id": "1.1.17",
-    "@libp2p/peer-id-factory": "1.0.19",
-    "@libp2p/websockets": "5.0.1",
+    "@libp2p/mplex": "7.1.1",
+    "@libp2p/peer-id": "1.1.18",
+    "@libp2p/peer-id-factory": "1.0.20",
+    "@libp2p/websockets": "5.0.2",
     "dotenv": "16.0.3",
     "e-ipfs-core-lib": "0.5.0",
     "it-length-prefixed": "8.0.3",

package.json

@@ -46,7 +46,13 @@ export function makeConfig () {
 
     peerAnnounceAddr: process.env.PEER_ANNOUNCE_ADDR,
     port: process.env.PORT ? parseInt(process.env.PORT) : 3000,
-    httpPort: process.env.HTTP_PORT ? parseInt(process.env.PORT) : 3001,
+    httpPort: process.env.HTTP_PORT ? parseInt(process.env.HTTP_PORT) : 3001,
+
+    // readiness
+    readinessMaxConnections: process.env.READINESS_MAX_CONNECTIONS ? parseInt(process.env.READINESS_MAX_CONNECTIONS) : 200,
+    readinessMaxPendingRequestBlocks: process.env.READINESS_MAX_PENDING_REQUEST_BLOCKS ? parseInt(process.env.READINESS_MAX_PENDING_REQUEST_BLOCKS) : 1e3,
+    readinessMaxEventLoopUtilization: process.env.READINESS_MAX_EVENT_LOOP_UTILIZATION ? parseInt(process.env.READINESS_MAX_EVENT_LOOP_UTILIZATION) : 0.5, // 0 to 1
+    readinessMaxResponseDuration: process.env.READINESS_MAX_RESPONSE_DURATION ? parseInt(process.env.READINESS_MAX_RESPONSE_DURATION) : 5e3, // 5 sec
 
     // p2p
     p2pConnectionMaxConnections: process.env.P2P_CONNECTION_MAX_CONNECTIONS ? parseInt(process.env.P2P_CONNECTION_MAX_CONNECTIONS) : 10e3,

src/config.js

@@ -1,13 +1,47 @@
+import config from './config.js'
+import { telemetry } from './telemetry.js'
 
-const SUCCESS_CODE = 200
-const ERROR_CODE = 503
+/**
+ * `bitswap-request-duration` is been reset every telemetry.export call (in /metrics)
+ */
+export function getHealthCheckValues () {
+  return {
+    connections: telemetry.getGaugeValue('bitswap-active-connections'),
+    pendingRequestBlocks: telemetry.getGaugeValue('bitswap-pending-entries'),
+    eventLoopUtilization: telemetry.getGaugeValue('bitswap-elu'),
+    responseDuration: telemetry.getHistogramValue('bitswap-request-duration') ?? -1
+  }
+}
+
+/**
+ * called every 1 second
+ */
+export function checkReadiness (logger) {
+  const resources = getHealthCheckValues()
+
+  if (resources.connections > config.readinessMaxConnections) {
+    logger.warn({ connections: resources.connections, maxConnections: config.readinessMaxConnections },
+      'Service is not ready due to max connections')
+    return false
+  }
+
+  if (resources.pendingRequestBlocks > config.readinessMaxPendingRequestBlocks) {
+    logger.warn({ pendingRequestBlocks: resources.pendingRequestBlocks, maxPendingRequestBlocks: config.readinessMaxPendingRequestBlocks },
+      'Service is not ready due to max pending request blocks')
+    return false
+  }
+
+  if (resources.eventLoopUtilization > config.readinessMaxEventLoopUtilization) {
+    logger.warn({ eventLoopUtilization: resources.eventLoopUtilization, maxEventLoopUtilization: config.readinessMaxEventLoopUtilization },
+      'Service is not ready due to max event loop utilization')
+    return false
+  }
 
-export async function checkReadiness ({ logger }) {
-  // TODO
+  if (resources.responseDuration > config.readinessMaxResponseDuration) {
+    logger.warn({ responseDuration: resources.pendingRequestBlocks, maxResponseDuration: config.readinessMaxResponseDuration },
+      'Service is not ready due to max response duration')
+    return false
+  }
 
-  return SUCCESS_CODE
-  // } catch (err) {
-  //   logger.error({ err }, 'Readiness Probe Failed')
-  //   return ERROR_CODE
-  // }
+  return true
 }

src/health-check.js

@@ -2,10 +2,13 @@
 import { createServer } from 'node:http'
 import { URL } from 'node:url'
 import { logger } from './logging.js'
-import { checkReadiness } from './health-check.js'
+import { getHealthCheckValues, checkReadiness } from './health-check.js'
 import { telemetry } from './telemetry.js'
 import { version } from './util.js'
 
+const SUCCESS_CODE = 200
+const ERROR_CODE = 503
+
 class HttpServer {
   startServer ({ port }) {
     if (this.server) {
@@ -20,25 +23,16 @@ class HttpServer {
           res.end()
           break
         case '/readiness': {
-          checkReadiness({ logger })
-            .then(httpStatus => {
-              res.writeHead(httpStatus).end()
-            })
+          res.writeHead(checkReadiness(logger) ? SUCCESS_CODE : ERROR_CODE)
+            .end()
           break
         }
         case '/load': {
           res.writeHead(200, {
             connection: 'close',
             'content-type': 'application/json'
           })
-
-          const resources = {
-            connections: telemetry.getGaugeValue('bitswap-active-connections'),
-            pendingRequestBlocks: telemetry.getGaugeValue('bitswap-pending-entries'),
-            eventLoopUtilization: telemetry.getGaugeValue('bitswap-elu'),
-            // note: duration it's been reset every /metrics call
-            responseDuration: telemetry.getHistogramValue('bitswap-request-duration') ?? -1
-          }
+          const resources = getHealthCheckValues()
 
           res.end(JSON.stringify(resources))
           break

src/http-server.js

@@ -8,10 +8,6 @@ import { getPeerId } from './peer-id.js'
 import { createConnectionConfig } from './util.js'
 
 async function boot () {
-  const readinessConfig = {
-    // TODO
-  }
-
   try {
     const awsClient = await createAwsClient(config, logger)
 

src/index.js

@@ -37,6 +37,10 @@ t.test('config - defaults', async t => {
     peerAnnounceAddr: undefined,
     port: 3000,
     httpPort: 3001,
+    readinessMaxConnections: 200,
+    readinessMaxPendingRequestBlocks: 1000,
+    readinessMaxEventLoopUtilization: 0.5,
+    readinessMaxResponseDuration: 5000,
     p2pConnectionMaxConnections: 10000,
     p2pConnectionMinConnections: 0,
     p2pConnectionPollInterval: 2000,
@@ -90,6 +94,10 @@ t.test('config - all by env vars', async t => {
   process.env.PEER_ANNOUNCE_ADDR = '/dns4/elastic-dev.dag.house/tcp/443/wss'
   process.env.PORT = '3123'
   process.env.HTTP_PORT = '3258'
+  process.env.READINESS_MAX_CONNECTIONS = '1'
+  process.env.READINESS_MAX_PENDING_REQUEST_BLOCKS = '1'
+  process.env.READINESS_MAX_EVENT_LOOP_UTILIZATION = '0.1'
+  process.env.READINESS_MAX_RESPONSE_DURATION = '1'
   process.env.P2P_CONNECTION_MAX_CONNECTIONS = '99999'
   process.env.P2P_CONNECTION_MIN_CONNECTIONS = '1'
   process.env.P2P_CONNECTION_POLL_INTERVAL = '1000'
@@ -143,7 +151,11 @@ t.test('config - all by env vars', async t => {
     peerIdS3Region: 'aws-s3',
     peerAnnounceAddr: '/dns4/elastic-dev.dag.house/tcp/443/wss',
     port: 3123,
-    httpPort: 3123,
+    httpPort: 3258,
+    readinessMaxConnections: 1,
+    readinessMaxPendingRequestBlocks: 1,
+    readinessMaxEventLoopUtilization: 0,
+    readinessMaxResponseDuration: 1,
     p2pConnectionMaxConnections: 99999,
     p2pConnectionMinConnections: 1,
     p2pConnectionPollInterval: 1000,