Added monitoring

Author: hkirat

mobile-magic/apps/k8s-orchestrator/index.ts

@@ -6,6 +6,15 @@ import cors from 'cors';
 import { Writable } from 'stream';
 import { DOMAIN } from './config';
 import { prismaClient } from "db/client";
+import promClient from 'prom-client';
+
+const containerCreateBucket = new promClient.Histogram({
+    name: 'container_create_bucket',
+    help: 'Number of times a container was created',
+    labelNames: ['type'],
+    buckets: [50, 100, 250, 500, 1000, 2500, 5000, 10000, 20000],
+});
+
 
 const kc = new KubeConfig();
 const app = express();
@@ -16,8 +25,6 @@ const PROJECT_TYPE_TO_BASE_FOLDER = {
     REACT_NATIVE: "/tmp/mobile-app"
 }
 
-const POD_EXPIRY = 1000 * 60 * 5; // 5 minutes
-const EMPTY_POD_BUFFER_SIZE = 3;
 kc.loadFromDefault();
 
 app.use(cors());
@@ -192,8 +199,10 @@ app.get("/worker/:projectId", async (req, res) => {
     }
 
     console.log("Project found, assigning to pod");
-    await assignPodToProject(projectId, "REACT"); // project.type);
+    const startTime = Date.now();
+    await assignPodToProject(projectId, "REACT");
     console.log("Pod assigned, sending response");
+    containerCreateBucket.observe({ type: project.type }, Date.now() - startTime);
 
     res.json({ 
         sessionUrl: `https://session-${projectId}.${DOMAIN}`, 
@@ -202,6 +211,11 @@ app.get("/worker/:projectId", async (req, res) => {
     });
 });
 
+app.get("/metrics", async (req, res) => {
+    res.setHeader('Content-Type', promClient.register.contentType);
+    res.end(await promClient.register.metrics());
+});
+
 app.listen(7001, () => {
     console.log("Server is running on port 7001");
 });

mobile-magic/apps/k8s-orchestrator/package.json

@@ -13,8 +13,9 @@
     "@types/cors": "^2.8.17",
     "@types/express": "^5.0.0",
     "cors": "^2.8.5",
-    "express": "^4.21.2",
     "db": "*",
+    "express": "^4.21.2",
+    "prom-client": "^15.1.3",
     "redis": "^4.7.0"
   }
 }

mobile-magic/apps/primary-backend/index.ts

@@ -2,8 +2,7 @@ import { prismaClient } from "db/client";
 import express from "express";
 import cors from "cors";
 import { authMiddleware } from "common/middleware";
-const promMid = require('express-prometheus-middleware')
-
+import promMid from 'express-prometheus-middleware'
 
 const app = express();
 

mobile-magic/apps/primary-backend/package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "@types/cors": "^2.8.17",
     "@types/express": "^5.0.0",
+    "@types/express-prometheus-middleware": "^1.2.3",
     "@types/jsonwebtoken": "^9.0.9",
     "common": "*",
     "cors": "^2.8.5",

mobile-magic/bun.lock

@@ -59,6 +59,7 @@
         "cors": "^2.8.5",
         "db": "*",
         "express": "^4.21.2",
+        "prom-client": "^15.1.3",
         "redis": "^4.7.0",
       },
       "devDependencies": {
@@ -73,6 +74,7 @@
       "dependencies": {
         "@types/cors": "^2.8.17",
         "@types/express": "^5.0.0",
+        "@types/express-prometheus-middleware": "^1.2.3",
         "@types/jsonwebtoken": "^9.0.9",
         "common": "*",
         "cors": "^2.8.5",
@@ -678,6 +680,8 @@
 
     "@types/express": ["@types/[email protected]", "", { "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", "@types/qs": "*", "@types/serve-static": "*" } }, "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ=="],
 
+    "@types/express-prometheus-middleware": ["@types/[email protected]", "", { "dependencies": { "@types/express": "*" } }, "sha512-3B+/27E/pxkZOzTWHqSQOhi5IctzoEyX6F1izBxAjiC5oe6tHLDy0kNQAj31SGR3wjxfvH1dgZi9fzmKX90OBw=="],
+
     "@types/express-serve-static-core": ["@types/[email protected]", "", { "dependencies": { "@types/node": "*", "@types/qs": "*", "@types/range-parser": "*", "@types/send": "*" } }, "sha512-3xhRnjJPkULekpSzgtoNYYcTWgEZkp4myc+Saevii5JPnHNvHMRlBSHDbs7Bh1iPPoVTERHEZXyhyLbMEsExsA=="],
 
     "@types/glob": ["@types/[email protected]", "", { "dependencies": { "@types/minimatch": "*", "@types/node": "*" } }, "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA=="],

ops/app/primary-backend/deployment.yml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
         - name: primary-backend
-          image: 100xdevs/antidevs-primary-backend:latest
+          image: 100xdevs/antidevs-primary-backend:7eab897aab7517fde2625afe89e265e175f9b678
           ports:
             - containerPort: 9090
           env:

ops/monitoring/grafana-values.yaml

@@ -0,0 +1,9 @@
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+      - name: Prometheus
+        type: prometheus
+        url: http://prometheus-server
+        access: proxy
+        isDefault: true

ops/monitoring/primary-backend-servicemonitor.yaml

@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: primary-backend-monitor
+spec:
+  selector:
+    matchLabels:
+      app: primary-backend
+  endpoints:
+  - port: http
+    path: /metrics
+    interval: 15s

ops/monitoring/prometheus-values.yaml

@@ -0,0 +1,20 @@
+prometheus:
+  prometheusSpec:
+    additionalScrapeConfigs:
+      - job_name: 'primary-backend'
+        kubernetes_sd_configs:
+          - role: pod
+            namespaces:
+              names:
+                - app
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_label_app]
+            regex: primary-backend
+            action: keep
+          - source_labels: [__meta_kubernetes_pod_ip]
+            target_label: __address__
+            replacement: $1:9090
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace

ops/monitoring/script.sh

@@ -0,0 +1,12 @@
+kubectl create namespace monitoring
+helm install prometheus prometheus-community/kube-prometheus-stack -f prometheus-values.yaml -n monitoring
+
+## Get grafana password
+kubectl --namespace monitoring get secrets prometheus-grafana -o jsonpath="{.data.admin-password}" | base64 -d ; echo
+
+## Get grafana url
+export POD_NAME=$(kubectl --namespace monitoring get pod -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=prometheus" -oname)
+kubectl --namespace monitoring port-forward $POD_NAME 3000
+
+## Get prometheus url
+kubectl port-forward svc/prometheus-kube-prometheus-prometheus 9090:9090