containers_running metric

Author: tshmieldev

prometheus.yml

@@ -4,3 +4,4 @@ scrape_configs:
           - targets: ["host.docker.internal:3001"]
       metrics_path: /metrics
       bearer_token: "5678"
+      scrape_interval: 1s

src/lib/docker.ts

@@ -1,6 +1,7 @@
 import { resolve } from "path";
 import { DEFAULT_CONFIG as CONFIG, DOCKER_CONFIG } from "../config";
 import { type Config } from "./validation";
+import { incrementContainers, decrementContainers } from "./prometheus";
 
 // Track active containers
 let activeContainers = new Set<string>();
@@ -47,10 +48,15 @@ export function getContainerName(sessionId: string): string {
 
 export function trackContainer(containerName: string): void {
     activeContainers.add(containerName);
+    incrementContainers();
 }
 
 export function untrackContainer(containerName: string): void {
     activeContainers.delete(containerName);
+    // This is for prometheus to be able to track the container
+    setTimeout(() => {
+        decrementContainers();
+    }, 1200); // Scrape interval = 1000, 200 is for flexibility
 }
 
 export async function cleanupContainer(containerName: string): Promise<void> {
@@ -67,6 +73,7 @@ export async function cleanupContainer(containerName: string): Promise<void> {
             stderr: "ignore",
         });
         activeContainers.delete(containerName);
+        decrementContainers();
     } catch (err) {
         // This is non-crucial as containers have the --rm flag
         console.error(`Error cleaning up container ${containerName}:`, err);

src/lib/prometheus.ts

@@ -1,7 +1,39 @@
+import { Registry, collectDefaultMetrics, Gauge } from "prom-client";
 import { prometheus } from "@hono/prometheus";
 
-const { printMetrics, registerMetrics } = prometheus({
-    collectDefaultMetrics: true,
+// Create custom registry
+const register = new Registry();
+collectDefaultMetrics({ register });
+
+const prom = prometheus({
+    collectDefaultMetrics: false, // We handle it manually
+    registry: register,
 });
 
+const { printMetrics, registerMetrics } = prom;
+
+// Custom gauge for running containers
+const containersRunning = new Gauge({
+    name: "containers_running",
+    help: "Number of currently running containers",
+    registers: [register],
+});
+
+register.registerMetric(containersRunning);
+
+// Initialize to 0 to ensure it appears in metrics
+containersRunning.set(0);
+
+export function incrementContainers() {
+    containersRunning.inc();
+}
+
+export function decrementContainers() {
+    containersRunning.dec();
+}
+
+export function setContainers(count: number) {
+    containersRunning.set(count);
+}
+
 export { printMetrics, registerMetrics };