Introduced Prometheus metrics endpoint for scraping (#1466)

* Initial Python support

* .NET support

* Introduced Prometheus metrics endpoint for scraping

* Streamline port numbers

* Remove test application files and add new .NET and Python test applications with Helm charts

- Deleted the TypeScript test application including package.json, server.ts, and tsconfig.json.
- Added a new .NET test application with Dockerfile, project file, and source code.
- Created a Helm chart for the .NET test application including deployment and service definitions.
- Added a new Python test application with Dockerfile, source code, and Helm chart.
- Each application includes a simple HTTP server that can call a target URL and handle errors.

* Cleanup

* Minor refactoring

* Cleanup

* Enable private preview languages in validation apps

* Refactor validate-mutation script to use dynamic environment variable and init container arrays

* Add support for Python and .NET targets in the test app configuration

* Enhance error handling in call-target route with logging

* Refactor error handling in call-target route to throw exceptions instead of writing error messages

* Add build and push instructions to Dockerfiles for various test apps

* Remove redundant status code setting for unexpected errors in call-target route

* Ensure imagePullPolicy is set to Always for nodejs-source-app container

* Add global exception handler for unhandled exceptions in the .NET application

* Refactor error handling in /call-target endpoint to log exceptions and return error responses

* Remove deprecated Kubernetes resource definitions and related configurations for app monitoring

* Remove obsolete configuration files and change log for app monitoring

* Move .NET test app to MVC

* Refactor /call-target endpoint to log exceptions and improve error handling

* Enhance error handling in /call-target endpoint with OpenTelemetry integration

* Refactor telemetry validation in validate_ai.sh to conditionally skip exceptions for Dotnet app and simplify error handling in dotnet-test-app.cs

* Improve health checks for app-monitoring-webhook deployment by validating pod readiness and container status

* Enhance deployment health check for app-monitoring-webhook by tracking rollout duration and verifying running pods

* Add validation output for MutatingWebhookConfiguration in test-cronjob.sh

* Enhance test-cronjob.sh by adding validation outputs for secret store and MutatingWebhookConfiguration states after modifications

* Enhance deployment health checks for app-monitoring-webhook by validating pod and container states specific to the latest ReplicaSet

* Fix secret store reference in test-cronjob.sh to use correct secret name

* Refactor deployment health check for app-monitoring-webhook to improve rollout status validation and error handling

* Refactor message in test-cronjob.sh for clarity on secret store repair process

* Fix caBundle value in MutatingWebhookConfiguration for app-monitoring-webhook

* Fix casing of caBundle in test-cronjob.sh output messages for consistency

* Update Helm chart from aks-rp repo

* Update README.md to include R2D process completion step and clarify rollout monitoring instructions

* Update README.md to clarify the R2D process and enhance instructions for monitoring AKS RP rollouts

* Update DotNet SDK image tag to 1.0.0-beta5

* Better coverage for Python and DotNet in tests

* Improve error logging in Mutator by including exception message in heartbeat metric

* Refactor server to use http for Prom endpoint server

* Expose CR count and instrumented namespace count metrics to prom scraper

* Update Python agent image tag to 1.0.0b25-aks

* Revert "Update Python agent image tag to 1.0.0b25-aks"

This reverts commit 4793286.

Author: tokaplan

appmonitoring/ts/src/Dockerfile

@@ -9,6 +9,12 @@ ENV IMAGE_TAG=$IMAGE_TAG
 ARG TARGETARCH
 ENV ARCH=$TARGETARCH
 
+ARG WEBHOOK_PORT=1337
+ENV WEBHOOK_PORT=$WEBHOOK_PORT
+
+ARG PROM_PORT=4000
+ENV PROM_PORT=$PROM_PORT
+
 RUN tdnf upgrade -y && tdnf clean all
 
 # Smoke test ensuring that NodeJS was installed.
@@ -20,5 +26,6 @@ COPY ./out/* /mutating-webhook
 COPY ./package.json /mutating-webhook
 COPY ./node_modules /mutating-webhook/node_modules
 
-EXPOSE 1337
+EXPOSE $WEBHOOK_PORT
+EXPOSE $PROM_PORT
 CMD [ "node", "server.js" ]

appmonitoring/ts/src/K8sWatcher.ts

@@ -18,7 +18,11 @@ export class K8sWatcher {
         const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi);
         const watch = new k8s.Watch(kc);
 
-        logger.registerWatchdog(Watchdogs.SecondsSinceLastSuccessfulCRList, () => (new Date().getTime() - K8sWatcher.lastSuccessfulListTimestamp.getTime()) / 1000.0);
+        logger.registerWatchdog(Watchdogs.SecondsSinceLastSuccessfulCRList, () => { 
+            const secondsElapsed = (new Date().getTime() - K8sWatcher.lastSuccessfulListTimestamp.getTime()) / 1000.0;
+            logger.SecondsSinceLastSuccessfulCRListSummary.observe(secondsElapsed);
+            return secondsElapsed;
+        });
 
         let latestResourceVersion: string = null;
         while (true) { // eslint-disable-line

appmonitoring/ts/src/LoggerWrapper.ts

@@ -6,6 +6,8 @@ import { PodInfo } from "./RequestDefinition.js";
 import log4js from "log4js";
 import { InstrumentationCRsCollection } from "./InstrumentationCRsCollection.js";
 
+import { Registry, collectDefaultMetrics, Summary } from "prom-client";
+
 const { configure, getLogger } = log4js;
 
 configure({
@@ -128,6 +130,11 @@ class LocalLogger {
         }
     }
 
+    public Register: Registry;
+    public SecondsSinceLastSuccessfulCRListSummary: Summary;
+    public CRCountSummary: Summary;
+    public InstrumentedNamespaceCountSummary: Summary;
+    
     private static instance: LocalLogger = null;
 
     private isUnitTestMode = false;
@@ -146,6 +153,28 @@ class LocalLogger {
         this.clusterMetadata = new ClusterMetadata(clusterArmId, clusterArmRegion, podName, imageTag, arch);
 
         this.log.info(`Application Insights has been set up and started. Default telemetry client is: ${this.client}, cluster metadata: ${JSON.stringify(this.clusterMetadata)}`);
+
+        this.Register = new Registry();
+        
+        collectDefaultMetrics({ register: this.Register });
+
+        this.SecondsSinceLastSuccessfulCRListSummary = new Summary({
+            name: 'cr_list_seconds_since_last_successful',
+            help: 'Seconds elapsed since the last successful CR list call'
+        });
+        this.CRCountSummary = new Summary({
+            name: 'cr_count',
+            help: 'Number of CRs in the cluster'
+        });
+        this.InstrumentedNamespaceCountSummary = new Summary({
+            name: 'instrumented_namespace_count',
+            help: 'Number of namespaces in the cluster that have at least one CR'
+        });
+        this.Register.registerMetric(this.SecondsSinceLastSuccessfulCRListSummary);
+        this.Register.registerMetric(this.CRCountSummary);
+        this.Register.registerMetric(this.InstrumentedNamespaceCountSummary);
+
+        this.log.info(`Prometheus client has been set up.`);
     }
 
     public trace(message: string, operationId: string, requestMetadata: RequestMetadata) {

appmonitoring/ts/src/package-lock.json

@@ -20,7 +20,8 @@
     "@types/npm": "^7.19.0",
     "applicationinsights": "^3.7.0",
     "log4js": "^6.9.1",
-    "node-forge": "^1.3.1"
+    "node-forge": "^1.3.1",
+    "prom-client": "^15.1.3"
   },
   "engines": {
     "node": "^20.4.0"

appmonitoring/ts/src/package.json

@@ -1,4 +1,5 @@
-import * as https from "https";
+import * as http from "http";
+import * as https from "https";
 import { Mutator } from "./Mutator.js";
 import { Events, HeartbeatMetrics, HeartbeatLogs, logger, RequestMetadata } from "./LoggerWrapper.js";
 import { InstrumentationCR, IAdmissionReview } from "./RequestDefinition.js";
@@ -91,8 +92,25 @@ try {
     throw e;
 }
 
-const port = process.env.port || 1337;
-logger.info(`listening on port ${port}`, operationId, null);
+const promPort = process.env.PROM_PORT;
+logger.info(`Prom endpoint is available on port ${promPort}`, operationId, null);
+const promServer = http.createServer(async (req, res) => {
+    if (req.method === "GET" && (req.url === "/metrics" || req.url === "/metrics/") && logger?.Register) {
+        try {
+            res.writeHead(200, { "Content-Type": logger.Register.contentType });
+            res.end(await logger.Register.metrics());
+        } catch (e) {
+            res.writeHead(500, { "Content-Type": "application/json" });
+            res.end(JSON.stringify(e));
+        }
+    } else {
+        res.writeHead(404);
+        res.end('Not Found');
+    }
+}).listen(promPort);
+
+const port = process.env.WEBHOOK_PORT;
+logger.info(`Webhook is listening on port ${port}`, operationId, null);
 
 const server = https.createServer(options, (req, res) => {
     logger.info(`Received request with url: ${req.url}, method: ${req.method}, content-type: ${req.headers["content-type"]}`, operationId, null);
@@ -150,12 +168,19 @@ const server = https.createServer(options, (req, res) => {
         res.writeHead(404);
         res.end();
     }
-
 }).listen(port);
 
 logger.info(`Server created on port ${port}`, null, null);
 
-function shutdownServer() {
+function shutdownServers() {
+    promServer.close((err) => {
+        if (err) {
+            logger.error(`Error shutting down prom server: ${err}`, operationId, null);
+        } else {
+            logger.info("Prom server has shut down gracefully", operationId, null);
+        }
+    });
+
     server.close((err) => {
         if (err) {
             logger.error(`Error shutting down server: ${err}`, operationId, null);
@@ -168,8 +193,8 @@ function shutdownServer() {
 }
 
 // listen for process termination signals
-process.on("SIGINT", shutdownServer);
-process.on("SIGTERM", shutdownServer);
+process.on("SIGINT", shutdownServers);
+process.on("SIGTERM", shutdownServers);
 
 const keepAlive = new Promise<void>((resolve) => {
     process.on('SIGINT', resolve);
@@ -184,9 +209,11 @@ logger.info("Server shut down, exiting now", operationId, null);
 function logCRs(crs: InstrumentationCRsCollection) {
     const items: InstrumentationCR[] = crs.ListCRs();
     logger.setHeartbeatMetric(HeartbeatMetrics.CRCount, items.length);
+    logger.CRCountSummary.observe(items.length);
 
     const uniqueNamespaces = new Set<string>(items.map(cr => cr.metadata.namespace, this));
     logger.setHeartbeatMetric(HeartbeatMetrics.InstrumentedNamespaceCount, uniqueNamespaces.size);
+    logger.InstrumentedNamespaceCountSummary.observe(uniqueNamespaces.size);
 
     let log = "CRs: [";
     for (let i = 0; i < items.length; i++) {