- updated notifications and plots (moved alerting to prometheus, deleted from grafana, added more plots to grafana)

- add more precise calculation of selected metrics

Author: HardMax71

backend/alertmanager/alertmanager.yml

@@ -0,0 +1,16 @@
+global:
+  smtp_smarthost: 'localhost:587'
+  smtp_from: '[email protected]'
+
+route:
+  group_by: ['alertname']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+  receiver: 'web.hook'
+
+receivers:
+  - name: 'web.hook'
+    webhook_configs:
+      - url: 'http://backend:443/api/v1/health'
+        send_resolved: true

backend/app/core/metrics.py

@@ -35,7 +35,7 @@ def labels(self, *args: Any, **kwargs: Any) -> 'BoundedLabelsCounter':
 
 # Resource usage metrics
 MEMORY_USAGE = Gauge(
-    "script_memory_usage_bytes", "Memory usage per script execution", ["lang_and_version"]
+    "script_memory_usage_mib", "Memory usage (MiB) per script execution", ["lang_and_version"]
 )
 
 ERROR_COUNTER = Counter(

backend/app/services/execution_service.py

@@ -10,6 +10,7 @@
     ACTIVE_EXECUTIONS,
     ERROR_COUNTER,
     EXECUTION_DURATION,
+    MEMORY_USAGE,
     SCRIPT_EXECUTIONS,
 )
 from app.db.repositories.execution_repository import (
@@ -143,16 +144,22 @@ async def _try_finalize_execution(self, execution: ExecutionInDB) -> Optional[Ex
             peak_kib = float(res_usage.get("peak_memory_kb", 0) or 0)
             peak_mib = peak_kib / 1024.0
 
+            MEMORY_USAGE.labels(lang_and_version=execution.lang + "-" + execution.lang_version).set(
+                peak_mib * 1024 * 1024)
+
             final_resource_usage = {
                 "execution_time": round(wall_s, 6),
                 "cpu_usage": round(cpu_millicores, 2),
                 "memory_usage": round(peak_mib, 2),
             }
 
             final_resource_usage["pod_phase"] = final_phase
+            status: ExecutionStatus = ExecutionStatus.COMPLETED if exit_code == 0 else ExecutionStatus.ERROR
+            if status == ExecutionStatus.ERROR:
+                ERROR_COUNTER.labels(error_type="NonZeroExitCode").inc()
 
             update_data = {
-                "status": ExecutionStatus.COMPLETED if exit_code == 0 else ExecutionStatus.ERROR,
+                "status": status,
                 "output": metrics.get("stdout", ""),
                 "errors": metrics.get("stderr", ""),
                 "resource_usage": final_resource_usage,

backend/grafana/grafana.ini

@@ -7,8 +7,6 @@ allow_sign_up = false
 allow_org_create = false
 auto_assign_org = true
 
-# idk why need host, took idea from https://github.com/grafana/grafana/blob/main/conf/defaults.ini
-# if removed - "SMTP not configured, check your grafana.ini config file's [smtp] section"
 [smtp]
 enabled = false
 
@@ -24,10 +22,7 @@ org_role = Viewer
 enabled = false
 
 [unified_alerting]
-enabled = true
-
-[alerting]
 enabled = false
 
-;[paths]
-;provisioning = /etc/grafana/provisioning
+[alerting]
+enabled = false

backend/grafana/provisioning/alerting/alerts.yml

@@ -0,0 +1,3 @@
+apiVersion: 1
+contactPoints: []
+policies: []