feat: add WPT queue WST selected region monitoring (DELO-4838) (#97)

Optional opentelemetry integration. When `start-with-instrumentation` is used, some WPT related metrics are exported (by default on `localhost:6060/metrics` endpoint).

Author: lukaszczerpak-cloudinary

instrumentation.js

@@ -0,0 +1,17 @@
+const opentelemetry = require('@opentelemetry/sdk-node');
+const {Resource} = require('@opentelemetry/resources');
+const {
+    ATTR_SERVICE_NAME,
+    ATTR_SERVICE_VERSION
+} = require('@opentelemetry/semantic-conventions');
+const {PrometheusExporter} = require('@opentelemetry/exporter-prometheus');
+const sdk = new opentelemetry.NodeSDK({
+    resource: new Resource({
+        [ATTR_SERVICE_NAME]: 'web-speed-test-server',
+        [ATTR_SERVICE_VERSION]: require('./package.json').version
+    }),
+    metricReader: new PrometheusExporter({
+        port: 6060,
+    }),
+});
+sdk.start();

package.json

@@ -5,9 +5,15 @@
   "scripts": {
     "test": "mocha",
     "start": "node start.js",
+    "start-with-instrumentation": "node --require ./instrumentation.js start.js",
     "postinstall": "patch-package"
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-prometheus": "^0.56.0",
+    "@opentelemetry/resources": "^1.29.0",
+    "@opentelemetry/sdk-node": "^0.56.0",
+    "@opentelemetry/semantic-conventions": "^1.28.0",
     "async": "^3.2.6",
     "async-mutex": "^0.5.0",
     "body-parser": "~1.20.3",

routes/wpt.js

@@ -6,6 +6,10 @@ const locationSelector = require("../wtp/locationSelector");
 const logger = require('../logger').logger;
 const {LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR, LOG_LEVEL_CRITICAL, LOG_LEVEL_DEBUG} = require('../logger');
 const path = require('path');
+const opentelemetry = require('@opentelemetry/api');
+
+const WstMeter = opentelemetry.metrics.getMeter('default');
+const testrunCounter = WstMeter.createCounter('testrun.total');
 
 const routeCallback = (error, result, res, rollBarMsg) => {
   if (error) {
@@ -52,11 +56,11 @@ const wtp = (app) => {
     });
   });
 
-
   app.post('/test/run', (req, res) => {
     let rollBarMsg = {testId: "N/A", thirdPartyErrorCode: "", file: path.basename((__filename))};
     if (!req.body) {
       logger.error('Could not run test missing request body', rollBarMsg, req);
+      testrunCounter.add(1, {"status": "BAD_REQUEST"});
       routeCallback({statusCode: 400}, null, res, rollBarMsg);
       return;
     }
@@ -65,6 +69,7 @@ const wtp = (app) => {
     rollBarMsg.analyzedUrl = testUrl;
     if (!testUrl) {
       logger.error('Could not run test missing test url',rollBarMsg, req);
+      testrunCounter.add(1, {"status": "BAD_REQUEST"});
       routeCallback({statusCode: 400}, null, res, rollBarMsg);
       return;
     }
@@ -75,6 +80,7 @@ const wtp = (app) => {
     }*/
     logger.info('Started test called from webspeedtest', rollBarMsg, req);
     apiCaller.runWtpTest(testUrl, mobile, (error, result, response, rollBarMsg) => {
+      testrunCounter.add(1, {"status": error ? "FAILURE" : "OK"});
       routeCallback(error, result, res, rollBarMsg)
     });
   });

wtp/locationSelector.js

@@ -3,10 +3,29 @@ const config = require('config');
 const {Mutex, withTimeout, E_TIMEOUT} = require('async-mutex');
 const apiKeys = require('./apiKey');
 const path = require("path");
+const opentelemetry = require("@opentelemetry/api");
 const logger = require('../logger').logger;
 
 const GET_LOCATIONS = 'http://www.webpagetest.org/getLocations.php?f=json';
 
+const WstMeter = opentelemetry.metrics.getMeter('default');
+const locationMetrics = {
+    total: WstMeter.createGauge('location.total'),
+    lastUpdate: WstMeter.createGauge('location.last_update'),
+    ratio: WstMeter.createGauge('location.ratio'),
+    score: WstMeter.createGauge('location.score'),
+    selected: WstMeter.createGauge('location.selected'),
+    agent: {
+        total: WstMeter.createGauge('location.agent.total'),
+        idle: WstMeter.createGauge('location.agent.idle'),
+        queued: WstMeter.createGauge('location.agent.queued'),
+        highprio: WstMeter.createGauge('location.agent.highprio'),
+        lowprio: WstMeter.createGauge('location.agent.lowprio'),
+        testing: WstMeter.createGauge('location.agent.testing'),
+        blocking: WstMeter.createGauge('location.agent.blocking')
+    },
+};
+
 class LocationSelector {
     constructor() {
         if (!LocationSelector.instance) {
@@ -111,6 +130,26 @@ class LocationSelector {
         this.location = this.getBestLocationId(filtered);
         this.cachedAllLocations = filtered;
         this.lastUpdated = Date.now();
+
+        // telemetry
+        locationMetrics.total.record(this.cachedAllLocations.length);
+        locationMetrics.lastUpdate.record(this.lastUpdated);
+        this.cachedAllLocations.forEach((loc) => {
+            let labels = {location: loc.location};
+
+            locationMetrics.ratio.record(loc.PendingTests.TestAgentRatio, labels);
+            locationMetrics.score.record(loc.score, labels);
+
+            locationMetrics.selected.record(loc.location === this.location ? 1 : 0, labels);
+
+            locationMetrics.agent.total.record(loc.PendingTests.Total, labels);
+            locationMetrics.agent.idle.record(loc.PendingTests.Idle, labels);
+            locationMetrics.agent.queued.record(loc.PendingTests.Queued, labels);
+            locationMetrics.agent.highprio.record(loc.PendingTests.HighPriority, labels);
+            locationMetrics.agent.lowprio.record(loc.PendingTests.LowPriority, labels);
+            locationMetrics.agent.testing.record(loc.PendingTests.Testing, labels);
+            locationMetrics.agent.blocking.record(loc.PendingTests.Blocking, labels);
+        });
     };
 
     async getLocation() {