[gitlab_runner] Add gitlab_runner_job_queue_duration_seconds metric

Add support for the `gitlab_runner_job_queue_duration_seconds` histogram
metric which tracks how long jobs wait in the queue before being picked
up by a runner.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: mahipdeora25

gitlab_runner/assets/configuration/spec.yaml

@@ -21,6 +21,7 @@ files:
         - gitlab_runner_autoscaling_machine_creation_duration_seconds
         - gitlab_runner_autoscaling_machine_states
         - gitlab_runner_errors_total
+        - gitlab_runner_job_queue_duration_seconds
         - gitlab_runner_jobs
         - gitlab_runner_version_info
         - go_gc_duration_seconds

gitlab_runner/metadata.csv

@@ -13,6 +13,8 @@ gitlab_runner.ci_ssh_docker_machines_provider_machine_states,gauge,,request,seco
 gitlab_runner.gitlab_runner_autoscaling_machine_creation_duration_seconds,gauge,,request,second,A histogram of Docker machine creation time. Applies to GitLab Runner 1.11.0+,0,gitlab_runner,auto docker machine creation time,
 gitlab_runner.gitlab_runner_autoscaling_machine_states,gauge,,request,second,The current number of machines per state in this provider. Applies to GitLab Runner 1.11.0+,0,gitlab_runner,auto total docker machines per state,
 gitlab_runner.gitlab_runner_errors_total,count,,request,second,The number of caught errors. Applies to GitLab Runner 1.11.0+,0,gitlab_runner,errors,
+gitlab_runner.gitlab_runner_job_queue_duration_seconds.count,gauge,,,,Histogram bucket and count of job queue duration observations.,0,gitlab_runner,job queue duration count,
+gitlab_runner.gitlab_runner_job_queue_duration_seconds.sum,gauge,,second,,Sum of job queue duration in seconds.,0,gitlab_runner,job queue duration sum,
 gitlab_runner.gitlab_runner_jobs,gauge,,,,The current number of running builds. Applies to GitLab Runner 1.11.0+,0,gitlab_runner,current running builds,
 gitlab_runner.gitlab_runner_jobs_total,count,,,,The total number of jobs executed.,1,gitlab_runner,total jobs,
 gitlab_runner.gitlab_runner_version_info,gauge,,request,,A metric with a constant '1' value labeled by different build stats fields. Applies to GitLab Runner 1.11.0+,0,gitlab_runner,version info,

gitlab_runner/tests/conftest.py

@@ -4,6 +4,7 @@
 
 import os
 
+import mock
 import pytest
 
 from datadog_checks.dev import docker_run
@@ -17,6 +18,7 @@
     GITLAB_RUNNER_URL,
     GITLAB_TEST_TOKEN,
     HERE,
+    HOST,
 )
 
 # Needed to mount volume for logging
@@ -57,3 +59,30 @@ def dd_environment():
         conditions=conditions,
     ):
         yield CONFIG, E2E_METADATA
+
+
+def _mocked_requests_get(*args, **kwargs):
+    url = args[0]
+
+    if url == 'http://{}:{}/metrics'.format(HOST, GITLAB_LOCAL_RUNNER_PORT):
+        fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures', 'metrics.txt')
+        with open(fixtures_path, 'r') as f:
+            text_data = f.read()
+            return mock.MagicMock(
+                status_code=200,
+                iter_lines=lambda **kwargs: text_data.split("\n"),
+                headers={'Content-Type': "text/plain"},
+            )
+    elif url == 'http://{}:{}/ci'.format(HOST, GITLAB_LOCAL_MASTER_PORT):
+        return mock.MagicMock(status_code=200)
+
+    return mock.MagicMock(status_code=404)
+
+
+@pytest.fixture()
+def mock_data():
+    with mock.patch(
+        'requests.Session.get',
+        side_effect=_mocked_requests_get,
+    ):
+        yield

gitlab_runner/tests/fixtures/metrics.txt

@@ -0,0 +1,37 @@
+# HELP ci_runner_version_info A metric with a constant '1' value labeled by different build stats fields
+# TYPE ci_runner_version_info gauge
+ci_runner_version_info{architecture="amd64",branch="main",built_at="2023-01-01T00:00:00+0000",go_version="go1.20",name="gitlab-runner",os="linux",revision="abc123",version="10.8.0"} 1
+# HELP ci_runner_errors The number of caught errors
+# TYPE ci_runner_errors counter
+ci_runner_errors{level="warning"} 2
+# HELP process_max_fds Maximum number of open file descriptors
+# TYPE process_max_fds gauge
+process_max_fds 1.048576e+06
+# HELP process_open_fds Number of open file descriptors
+# TYPE process_open_fds gauge
+process_open_fds 12
+# HELP process_resident_memory_bytes Resident memory size in bytes
+# TYPE process_resident_memory_bytes gauge
+process_resident_memory_bytes 2.7394048e+07
+# HELP process_start_time_seconds Start time of the process since unix epoch in seconds
+# TYPE process_start_time_seconds gauge
+process_start_time_seconds 1.67281956837e+09
+# HELP process_virtual_memory_bytes Virtual memory size in bytes
+# TYPE process_virtual_memory_bytes gauge
+process_virtual_memory_bytes 7.61856e+08
+# HELP gitlab_runner_job_queue_duration_seconds Histogram of the time in seconds a job is queued before being picked up by a runner
+# TYPE gitlab_runner_job_queue_duration_seconds histogram
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="0.3"} 5
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="0.6"} 8
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="1"} 10
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="5"} 12
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="10"} 14
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="30"} 15
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="60"} 15
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="300"} 15
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="600"} 15
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="1800"} 15
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="3600"} 15
+gitlab_runner_job_queue_duration_seconds_bucket{runner="test-runner",le="+Inf"} 15
+gitlab_runner_job_queue_duration_seconds_sum{runner="test-runner"} 45.5
+gitlab_runner_job_queue_duration_seconds_count{runner="test-runner"} 15

gitlab_runner/tests/test_unit.py

@@ -7,6 +7,7 @@
 import mock
 import pytest
 
+from datadog_checks.dev.utils import get_metadata_metrics
 from datadog_checks.gitlab_runner import GitlabRunnerCheck
 
 from . import common
@@ -44,3 +45,25 @@ def test_timeout(test_case, timeout_config, expected_timeout):
             verify=mock.ANY,
             allow_redirects=mock.ANY,
         )
+
+
+@pytest.mark.unit
+def test_job_queue_duration_metric(aggregator, dd_run_check, mock_data):
+    """
+    Test that the gitlab_runner_job_queue_duration_seconds histogram metric
+    is properly collected from the Prometheus endpoint.
+    """
+    allowed_metrics = list(common.ALLOWED_METRICS) + ['gitlab_runner_job_queue_duration_seconds']
+    config = deepcopy(common.CONFIG)
+    config['init_config']['allowed_metrics'] = allowed_metrics
+
+    check = GitlabRunnerCheck('gitlab_runner', config['init_config'], instances=config['instances'])
+    dd_run_check(check)
+    dd_run_check(check)
+
+    # Histogram buckets and count (reported as .count with upper_bound tags)
+    aggregator.assert_metric('gitlab_runner.gitlab_runner_job_queue_duration_seconds.count')
+    # Histogram sum
+    aggregator.assert_metric('gitlab_runner.gitlab_runner_job_queue_duration_seconds.sum')
+
+    aggregator.assert_metrics_using_metadata(get_metadata_metrics())