Instrumentation: Add gauge of total number of requests in flight (#281)

* Instrumentation: Add gauge of total number of requests in flight

* remove useless call to next

* Update package.json

Author: AgnesToulet

package.json

@@ -29,6 +29,7 @@
     "lodash": "^4.17.21",
     "minimist": "^1.2.0",
     "morgan": "^1.9.0",
+    "on-finished": "^2.3.0",
     "prom-client": "^11.5.3",
     "puppeteer": "^10.0.0",
     "puppeteer-cluster": "^0.22.0",

src/service/http-server.ts

@@ -8,7 +8,7 @@ import * as promClient from 'prom-client';
 import { Logger } from '../logger';
 import { Browser, createBrowser } from '../browser';
 import { ServiceConfig } from '../config';
-import { metricsMiddleware } from './metrics_middleware';
+import { setupHttpServerMetrics } from './metrics_middleware';
 import { RenderOptions, RenderCSVOptions, HTTPHeaders, Metrics } from '../browser/browser';
 
 export interface RenderRequest {
@@ -58,7 +58,10 @@ export class HttpServer {
         stream: this.log.errorWriter,
       })
     );
-    this.app.use(metricsMiddleware(this.config.service.metrics, this.log));
+
+    if (this.config.service.metrics.enabled) {
+      setupHttpServerMetrics(this.app, this.config.service.metrics, this.log);
+    }
     this.app.get('/', (req: express.Request, res: express.Response) => {
       res.send('Grafana Image Renderer');
     });

src/service/metrics_middleware.ts

@@ -1,14 +1,12 @@
 import * as promBundle from 'express-prom-bundle';
+import * as promClient from 'prom-client';
+import * as onFinished from 'on-finished';
+import express = require('express');
+
 import { MetricsConfig } from '../config';
 import { Logger } from '../logger';
 
-export const metricsMiddleware = (config: MetricsConfig, log: Logger) => {
-  if (!config.enabled) {
-    return (req, res, next) => {
-      next();
-    };
-  }
-
+export const setupHttpServerMetrics = (app: express.Express, config: MetricsConfig, log: Logger) => {
   log.info(
     'Metrics enabled',
     'collectDefaultMetrics',
@@ -17,13 +15,15 @@ export const metricsMiddleware = (config: MetricsConfig, log: Logger) => {
     config.requestDurationBuckets.join(',')
   );
 
+  const excludeRegExp = /^((?!(render)).)*$/;
+
   const opts = {
     httpDurationMetricName: 'grafana_image_renderer_service_http_request_duration_seconds',
     metricType: 'histogram',
     buckets: config.requestDurationBuckets,
-    excludeRoutes: [/^((?!(render)).)*$/],
+    excludeRoutes: [excludeRegExp],
     promClient: {},
-    formatStatusCode: res => {
+    formatStatusCode: (res) => {
       if (res && res.req && res.req.aborted) {
         // Nginx non-standard code 499 Client Closed Request
         // Used when the client has closed the request before
@@ -39,6 +39,28 @@ export const metricsMiddleware = (config: MetricsConfig, log: Logger) => {
     opts.promClient.collectDefaultMetrics = {};
   }
 
-  const bundle = promBundle(opts);
-  return bundle;
+  const metricsMiddleware = promBundle(opts);
+  app.use(metricsMiddleware);
+
+  const httpRequestsInFlight = new promClient.Gauge({
+    name: 'grafana_image_renderer_http_request_in_flight',
+    help: 'A gauge of requests currently being served by the image renderer.',
+  });
+  app.use(requestsInFlightMiddleware(httpRequestsInFlight, excludeRegExp));
+};
+
+const requestsInFlightMiddleware = (httpRequestsInFlight: promClient.Gauge, excludeRegExp: RegExp) => {
+  return (req, res, next) => {
+    const path = req.originalUrl || req.url;
+    if (path.match(excludeRegExp)) {
+      return next();
+    }
+
+    httpRequestsInFlight.inc();
+    onFinished(res, () => {
+      httpRequestsInFlight.dec();
+    });
+
+    next();
+  };
 };