feat: prometheus metrics (#582)

yanksyoon · yanksyoon · cbartz · web-flow · commit 426b54e422c2 · 2025-07-09T13:09:42.000Z
* add prometheus client to dependency &amp; order deps alphabetically

* add metrics route

* refactor internal function

* add reconcile metrics

* implement reconcile metrics

* initial implementation

* test: add integration test

* test: test flavor label

* fix: enable allure test to collect reports before running tests

* remove dep

* debug: try two tests

* test: mark openstack test

* test: move integration test pre run script

* fix: juju microk8s setup script

* test: dont switch back to original controller

* switch back to original controller
git push

* unset juju model env var

* raw json output

* debug

* fix: label name

* test: fix offer name relation

* test: do not use juju controller from env var

* test: do not raise on error while waiting for idle

* test: model integrate &amp; consume

* test: keep microk8s model

* test: deploy cos agent

* test: add grafana agent series

* test: fix typo

* test: assign grafana agent channel

* test: juju ops lib is ...

* test: use series

* test: swap out ops_test with jubilant

* test: generator type hint fix

* test: add prometheus datasource

* test: separate micrlk8s juju from lxd juju

* test: add controller prefix to model

* test: store model &amp; controller name

* test: remove duplicate model naming w/ controller name prefix

* test: add fixture dependency

* test: switch controller fixture for juju offers

* test: switch controller select model

* test: modify juju controller &amp; model env vars

* test: try direct cli_bin call

* test: try no env patch

* test: assert subprocess result

* test: set additional juju envs

* test: use controller &amp; model params

* test: try consume model w/ model params

* test: use direct consume command

* test: setup microstack metallb for ingress

* test: check for metrics availability

* test: add schema to request

* test: fix syntax err

* test: fix prometheus ip

* test: wait for openstack metrics longer

* test: test only metrics that have been generated

* test: restore tests

* chore: move integration test script

* ci: separate tests depending on k8s

* test: fix lints

* fix: lint

* fix: lint

* ci: remove unused setup script

* docs: add changelog &amp; increment version

* ci: separate test names

* chore: refactor the metrics to be fetched without side effects

* chore: increment metrics by 1

* chore: move code

* fix: lint fixe

* fix integration test wf file

---------

Co-authored-by: yanksyoon &lt;yanksyoon@canonical.com&gt;
Co-authored-by: Christopher Bartz &lt;christopher.bartz@canonical.com&gt;
diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml
@@ -16,7 +16,6 @@ jobs:
     secrets: inherit
     with:
       juju-channel: 3.6/stable
-      pre-run-script: scripts/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.6
       modules: '["test_e2e"]'
diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml
@@ -12,35 +12,34 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  openstack-interface-tests-private-endpoint:
-    name: openstack interface test using private-endpoint
+  openstack-integration-tests-private-endpoint:
+    name: Integration test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
     secrets: inherit
     with:
       juju-channel: 3.6/stable
-      pre-run-script: scripts/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.6
-      modules: '["test_runner_manager_openstack"]'
-      extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"'
+      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade",  "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive"]'
+      extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"'
       self-hosted-runner: true
       self-hosted-runner-label: stg-private-endpoint
-  openstack-integration-tests-private-endpoint:
-    name: Integration test using private-endpoint
+  openstack-integration-tests-cross-controller-private-endpoint:
+    name: Cross controller integration test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
     secrets: inherit
     with:
       juju-channel: 3.6/stable
-      pre-run-script: scripts/setup-integration-tests.sh
+      pre-run-script: tests/integration/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.6
-      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade",  "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive"]'
+      modules: '["test_prometheus_metrics"]'
       extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"'
       self-hosted-runner: true
       self-hosted-runner-label: stg-private-endpoint
   allure-report:
     if: ${{ (success() || failure()) && github.event_name == 'schedule' }}
     needs:
-      - openstack-interface-tests-private-endpoint
       - openstack-integration-tests-private-endpoint
+      - openstack-integration-tests-cross-controller-private-endpoint
     uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -4,6 +4,7 @@ This changelog documents user-relevant changes to the GitHub runner charm.
 
 ### 2025-06-30
 - New configuration options aproxy-exclude-addresses and aproxy-redirect-ports for allowing aproxy to redirect arbitrary TCP traffic
+- Added prometheus metrics to the GitHub runner manager application.
 
 ## 2025-06-26
 
diff --git a/github-runner-manager/pyproject.toml b/github-runner-manager/pyproject.toml
@@ -3,7 +3,7 @@
 
 [project]
 name = "github-runner-manager"
-version = "0.5.0"
+version = "0.6.0"
 authors = [
     { name = "Canonical IS DevOps", email = "is-devops-team@canonical.com" },
 ]
diff --git a/github-runner-manager/requirements.txt b/github-runner-manager/requirements.txt
@@ -1,9 +1,10 @@
+click==8.2.1
 fabric >=3,<4
+flask==3.1.1
 ghapi
 jinja2
 kombu==5.5.3
 openstacksdk==4.5.0
+prometheus-client==0.22.1
 pydantic < 2
 pymongo==4.13.0
-click==8.2.1
-flask==3.1.1
diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py
@@ -12,6 +12,7 @@
 from threading import Lock
 
 from flask import Flask, request
+from prometheus_client import generate_latest
 
 from github_runner_manager.configuration import ApplicationConfiguration
 from github_runner_manager.errors import CloudError, LockError
@@ -44,7 +45,7 @@ def check_runner() -> tuple[str, int]:
     Returns:
         Information on the runners in JSON format.
     """
-    app_config = app.config[APP_CONFIG_NAME]
+    app_config: ApplicationConfiguration = app.config[APP_CONFIG_NAME]
     app.logger.info("Checking runners...")
     runner_scaler = get_runner_scaler(app_config)
     try:
@@ -72,7 +73,7 @@ def flush_runner() -> tuple[str, int]:
     if flush_busy_str in ("True", "true"):
         flush_busy = True
 
-    lock = get_lock()
+    lock = _get_lock()
     with lock:
         app.logger.info("Flushing runners...")
         runner_scaler = get_runner_scaler(app_config)
@@ -87,7 +88,7 @@ def flush_runner() -> tuple[str, int]:
     return ("", 204)
 
 
-def get_lock() -> Lock:
+def _get_lock() -> Lock:
     """Get the lock representing modification access to the set of runners.
 
     Raises:
@@ -103,6 +104,16 @@ def get_lock() -> Lock:
     raise LockError("Lock not configured")
 
 
+@app.route("/metrics", methods=["GET"])
+def metrics() -> bytes:
+    """Return prometheus metrics from default registry.
+
+    Returns:
+        The latest metrics from the default Prometheus registry.
+    """
+    return generate_latest()
+
+
 @dataclass
 class FlaskArgs:
     """Arguments for Flask HTTP server.
diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py
@@ -23,6 +23,7 @@
 from github_runner_manager.metrics import events as metric_events
 from github_runner_manager.metrics import github as github_metrics
 from github_runner_manager.metrics import runner as runner_metrics
+from github_runner_manager.metrics.reconcile import CLEANED_RUNNERS_TOTAL
 from github_runner_manager.metrics.runner import RunnerMetrics
 from github_runner_manager.openstack_cloud.constants import CREATE_SERVER_TIMEOUT
 from github_runner_manager.platform.platform_provider import (
@@ -348,6 +349,7 @@ def _delete_cloud_runners(
 
             logging.info("Delete runner in cloud: %s", cloud_runner.instance_id)
             runner_metric = self._cloud.delete_runner(cloud_runner.instance_id)
+            CLEANED_RUNNERS_TOTAL.labels(self.manager_name).inc(1)
             if not runner_metric:
                 logger.error("No metrics returned after deleting %s", cloud_runner.instance_id)
             else:
diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py
@@ -8,10 +8,7 @@
 from dataclasses import dataclass
 
 import github_runner_manager.reactive.runner_manager as reactive_runner_manager
-from github_runner_manager.configuration import (
-    ApplicationConfiguration,
-    UserInfo,
-)
+from github_runner_manager.configuration import ApplicationConfiguration, UserInfo
 from github_runner_manager.constants import GITHUB_SELF_HOSTED_ARCH_LABELS
 from github_runner_manager.errors import (
     CloudError,
@@ -28,6 +25,12 @@
     RunnerMetadata,
 )
 from github_runner_manager.metrics import events as metric_events
+from github_runner_manager.metrics.reconcile import (
+    BUSY_RUNNERS_COUNT,
+    EXPECTED_RUNNERS_COUNT,
+    IDLE_RUNNERS_COUNT,
+    RECONCILE_DURATION_SECONDS,
+)
 from github_runner_manager.openstack_cloud.models import OpenStackServerConfig
 from github_runner_manager.openstack_cloud.openstack_runner_manager import (
     OpenStackRunnerManager,
@@ -216,6 +219,8 @@ def __init__(  # pylint: disable=too-many-arguments, too-many-positional-argumen
         self._platform_name = platform_name
         self._python_path = python_path
 
+        EXPECTED_RUNNERS_COUNT.labels(self._manager.manager_name).set(self._base_quantity)
+
     def get_runner_info(self) -> RunnerInfo:
         """Get information on the runners.
 
@@ -321,7 +326,10 @@ def reconcile(self) -> int:
                 flavor=self._manager.manager_name,
                 expected_runner_quantity=expected_runner_quantity,
             )
-            _issue_reconciliation_metric(reconcile_metric_data)
+            RECONCILE_DURATION_SECONDS.labels(self._manager.manager_name).observe(
+                end_timestamp - start_timestamp
+            )
+            _issue_reconciliation_metric(reconcile_metric_data, self._manager.manager_name)
 
         logger.info("Finished reconciliation.")
 
@@ -403,12 +411,13 @@ def _log_runners(runner_list: tuple[RunnerInstance]) -> None:
 
 
 def _issue_reconciliation_metric(
-    reconcile_metric_data: _ReconcileMetricData,
+    reconcile_metric_data: _ReconcileMetricData, manager_name: str
 ) -> None:
     """Issue the reconciliation metric.
 
     Args:
         reconcile_metric_data: The data used to issue the reconciliation metric.
+        manager_name: The name of the manager.
     """
     idle_runners = {
         runner.name
@@ -431,6 +440,9 @@ def _issue_reconciliation_metric(
     logger.info("Current available runners (idle + healthy offline): %s", available_runners)
     logger.info("Current active runners: %s", active_runners)
 
+    BUSY_RUNNERS_COUNT.labels(manager_name).set(len(active_runners))
+    IDLE_RUNNERS_COUNT.labels(manager_name).set(len(idle_runners))
+
     try:
 
         metric_events.issue_event(
diff --git a/github-runner-manager/src/github_runner_manager/metrics/reconcile.py b/github-runner-manager/src/github_runner_manager/metrics/reconcile.py
@@ -0,0 +1,34 @@
+#  Copyright 2025 Canonical Ltd.
+#  See LICENSE file for licensing details.
+
+"""Module for collecting metrics related to the reconciliation process."""
+
+from prometheus_client import Gauge, Histogram
+
+LABEL_FLAVOR = "flavor"
+
+RECONCILE_DURATION_SECONDS = Histogram(
+    name="reconcile_duration_seconds",
+    documentation="Duration of reconciliation (seconds)",
+    labelnames=[LABEL_FLAVOR],
+)
+EXPECTED_RUNNERS_COUNT = Gauge(
+    name="expected_runners_count",
+    documentation="Expected number of runners",
+    labelnames=[LABEL_FLAVOR],
+)
+BUSY_RUNNERS_COUNT = Gauge(
+    name="busy_runners_count",
+    documentation="Number of busy runners",
+    labelnames=[LABEL_FLAVOR],
+)
+IDLE_RUNNERS_COUNT = Gauge(
+    name="idle_runners_count",
+    documentation="Number of idle runners",
+    labelnames=[LABEL_FLAVOR],
+)
+CLEANED_RUNNERS_TOTAL = Gauge(
+    name="cleaned_runners_total",
+    documentation="Total number of runners cleaned up",
+    labelnames=[LABEL_FLAVOR],
+)
diff --git a/scripts/setup-integration-tests.sh b/scripts/setup-integration-tests.sh
diff --git a/src/charm.py b/src/charm.py
@@ -186,7 +186,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self._log_charm_status()
 
-        self._grafana_agent = COSAgentProvider(self)
+        self._grafana_agent = COSAgentProvider(
+            self,
+            metrics_endpoints=[
+                {"path": "/metrics", "port": int(manager_service.GITHUB_RUNNER_MANAGER_PORT)}
+            ],
+        )
 
         self._stored.set_default(
             path=self.config[PATH_CONFIG_NAME],  # for detecting changes
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
@@ -9,7 +9,7 @@
 import string
 from pathlib import Path
 from time import sleep
-from typing import Any, AsyncIterator, Generator, Iterator, Optional, cast
+from typing import Any, AsyncGenerator, AsyncIterator, Generator, Iterator, Optional, cast
 
 import jubilant
 import nest_asyncio
@@ -750,8 +750,10 @@ async def instance_helper_fixture(request: pytest.FixtureRequest) -> OpenStackIn
     return OpenStackInstanceHelper(openstack_connection=openstack_connection)
 
 
-@pytest.fixture(scope="module")
-def juju(request: pytest.FixtureRequest, model: Model) -> Generator[jubilant.Juju, None, None]:
+@pytest_asyncio.fixture(scope="module")
+async def juju(
+    request: pytest.FixtureRequest, model: Model
+) -> AsyncGenerator[jubilant.Juju, None]:
     """Pytest fixture that wraps :meth:`jubilant.with_model`."""
 
     def show_debug_log(juju: jubilant.Juju):
@@ -764,14 +766,19 @@ def show_debug_log(juju: jubilant.Juju):
             log = juju.debug_log(limit=1000)
             print(log, end="")
 
+    controller = await model.get_controller()
     if model:
-        juju = jubilant.Juju(model=model.name)
+        # Currently juju has no way of switching controller context, this is required to operate
+        # in the right controller's right model when using multiple controllers.
+        # See: https://github.com/canonical/jubilant/issues/158
+        juju = jubilant.Juju(model=f"{controller.controller_name}:{model.name}")
         yield juju
         show_debug_log(juju)
         return
 
     keep_models = cast(bool, request.config.getoption("--keep-models"))
-    with jubilant.temp_model(keep=keep_models) as juju:
+    with jubilant.temp_model(keep=keep_models, controller=controller.controller_name) as juju:
+        juju.model = f"{controller.controller_name}:{juju.model}"
         juju.wait_timeout = 10 * 60
         yield juju
         show_debug_log(juju)
diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt
@@ -1,5 +1,6 @@
 GitPython>3,<4
-pygithub
+jubilant==1.1.*
 kombu==5.*
+pygithub
 pymongo==4.*
-jubilant==1.1.*
+tenacity==9.1.2
diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright 2025 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Pre-run script for integration test operator-workflows action.
+# https://github.com/canonical/operator-workflows/blob/main/.github/workflows/integration_test.yaml
+
+# The COS observability stack are deployed on K8s models.
+
+# save original controller that is used for testing
+ORIGINAL_CONTROLLER=$(juju controllers --format json | jq -r '.controllers | keys | .[0]')
+
+echo "bootstrapping microk8s juju controller"
+sudo snap install microk8s --channel=1.32-strict/stable
+GROUP=snap_microk8s
+sudo usermod -a -G "$GROUP" "$USER"
+if [ "$(id -gn)" != "$GROUP" ]; then
+  exec sg "$GROUP" "$0" "$*"
+fi
+
+# Get preferred source IP address for metallb
+IPADDR=$( { ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc'; } )
+sudo microk8s enable "metallb:$IPADDR-$IPADDR" "hostpath-storage"
+microk8s status --wait-ready
+
+unset JUJU_CONTROLLER
+unset JUJU_MODEL
+juju bootstrap microk8s microk8s
+juju switch "$ORIGINAL_CONTROLLER"
diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py
diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@`
`3`	`3`
`4`	`4`	`[project]`
`5`	`5`	`name = "github-runner-manager"`
`6`		`-version = "0.5.0"`
	`6`	`+version = "0.6.0"`
`7`	`7`	`authors = [`
`8`	`8`	`{ name = "Canonical IS DevOps", email = "[email protected]" },`
`9`	`9`	`]`