feat: pre-spawned mode for jobmanager platform (#567)

* fix reconciliation timeout

* update CODEOWNERS

* update openapi client

* use path for jobmanager_url

* update requirements.txt

* update jobmanager

* update jobmanager_provider

* update multiplexer platform

* update jobmanager

* update jobmanager_provider

* update integration test

* add clouds.yaml to juju ignore

* WIP checkin

* specify platform in runner scaler

* WIP adaption for pre-spawned test in jobmanager

* clean pre-spawned test

* set oneshot request handler before normal handler

* add unit on top

* add mongodb endpoints

* add jubilant

* fix get_mongodb_uri for case where multiple secrets are there

* add test for reactive mode

* reuse model from ops_test for integration test

* fix jubilant scope

* don't mention JobManager yet

* remove todo

* wait that all agents are idle

* Refactor integration test

* remove unnecessary integration test

* handle mongodb relation broken

* fix wf file

* lint

* remove unused constants in charm.py

* split integration tests

* add back ending new line

* remove empty line comment

* move Platform enum to multiplexer

* add job_id validation

* add more validation

* fix job_path_prefix in validation

Author: cbartz

.github/workflows/integration_test.yaml

@@ -12,22 +12,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # test option values defined at test/conftest.py are passed on via repository secret
-  # INTEGRATION_TEST_ARGS to operator-workflows automatically.
-  integration-tests:
-    name: Integration test with juju 3.1
-    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    secrets: inherit
-    with:
-      juju-channel: 3.1/stable
-      pre-run-script: scripts/setup-integration-tests.sh
-      provider: lxd
-      test-tox-env: integration-juju3.1
-      modules: '["test_charm_scheduled_events", "test_debug_ssh", "test_charm_upgrade"]'
-      extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"'
-      self-hosted-runner: true
-      self-hosted-runner-label: stg-private-endpoint
-      test-timeout: 90
   openstack-interface-tests-private-endpoint:
     name: openstack interface test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
@@ -50,14 +34,13 @@ jobs:
       pre-run-script: scripts/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.6
-      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_reactive", "test_jobmanager"]'
+      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade",  "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive"]'
       extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"'
       self-hosted-runner: true
       self-hosted-runner-label: stg-private-endpoint
   allure-report:
     if: ${{ (success() || failure()) && github.event_name == 'schedule' }}
     needs:
-      - integration-tests
       - openstack-interface-tests-private-endpoint
       - openstack-integration-tests-private-endpoint
     uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main

.jujuignore

@@ -6,3 +6,4 @@ tests
 src-docs
 docs
 github-runner-operator
+clouds.yaml

docs/changelog.md

@@ -41,9 +41,9 @@ network/infrastructure specific things.
 
 ### 2025-05-06
 
-- The ssh health checks are removed and the platform providers (GitHub or the JobManager) are used instead to get the runners health
+- The ssh health checks are removed and GitHub is used instead to get the runners health
 information. This implies many changes in both the structure of the project and its functionality. Potentially, many race conditions should
-disappear for the GitHub case. 
+disappear.
 
 ### 2025-04-28
 

github-runner-manager/src/github_runner_manager/configuration/base.py

@@ -10,7 +10,7 @@
 import yaml
 from pydantic import AnyHttpUrl, BaseModel, Field, IPvAnyAddress, MongoDsn, root_validator
 
-from github_runner_manager.configuration import github
+from github_runner_manager.configuration import github, jobmanager
 from github_runner_manager.openstack_cloud.configuration import OpenStackConfiguration
 
 logger = logging.getLogger(__name__)
@@ -40,6 +40,7 @@ class ApplicationConfiguration(BaseModel):
     Attributes:
         name: Name to identify the manager. Used for metrics.
         extra_labels: Extra labels to add to the runner.
+        jobmanager_config: Configuration for the jobmanager platform.
         github_config: GitHub configuration.
         service_config: The configuration for supporting services.
         non_reactive_configuration: Configuration for non-reactive mode.
@@ -50,6 +51,7 @@ class ApplicationConfiguration(BaseModel):
 
     name: str
     extra_labels: list[str]
+    jobmanager_config: jobmanager.JobManagerConfiguration | None
     github_config: github.GitHubConfiguration | None
     service_config: "SupportServiceConfig"
     non_reactive_configuration: "NonReactiveConfiguration"

github-runner-manager/src/github_runner_manager/configuration/jobmanager.py

@@ -0,0 +1,15 @@
+#  Copyright 2025 Canonical Ltd.
+#  See LICENSE file for licensing details.
+
+"""Module containing JobManager Configuration."""
+from pydantic import BaseModel, HttpUrl
+
+
+class JobManagerConfiguration(BaseModel):
+    """JobManager configuration for the application.
+
+    Attributes:
+       url: Base url of the job manager API.
+    """
+
+    url: HttpUrl

github-runner-manager/src/github_runner_manager/manager/runner_manager.py

@@ -508,13 +508,13 @@ def _create_runner(args: _CreateRunnerArgs) -> InstanceID:
             RunnerError: On error creating OpenStack runner.
         """
         instance_id = InstanceID.build(args.cloud_runner_manager.name_prefix, args.reactive)
-        runner_context, github_runner = args.platform_provider.get_runner_context(
+        runner_context, runner_info = args.platform_provider.get_runner_context(
             instance_id=instance_id, metadata=args.metadata, labels=args.labels
         )
 
         # Update the runner id if necessary
         if not args.metadata.runner_id:
-            args.metadata.runner_id = str(github_runner.id)
+            args.metadata.runner_id = str(runner_info.id)
 
         runner_identity = RunnerIdentity(instance_id=instance_id, metadata=args.metadata)
         try:
@@ -533,7 +533,7 @@ def _create_runner(args: _CreateRunnerArgs) -> InstanceID:
 
         except RunnerError:
             logger.warning("Deleting runner %s from platform after creation failed", instance_id)
-            args.platform_provider.delete_runner(github_runner.identity)
+            args.platform_provider.delete_runner(runner_info.identity)
             raise
         return instance_id
 

github-runner-manager/src/github_runner_manager/manager/runner_scaler.py

@@ -33,7 +33,7 @@
     OpenStackRunnerManager,
     OpenStackRunnerManagerConfig,
 )
-from github_runner_manager.platform.multiplexer_provider import MultiplexerPlatform
+from github_runner_manager.platform.multiplexer_provider import MultiplexerPlatform, Platform
 from github_runner_manager.platform.platform_provider import PlatformRunnerState
 from github_runner_manager.reactive.types_ import ReactiveProcessConfig
 
@@ -141,6 +141,7 @@ def build(  # pylint: disable-msg=too-many-locals
         platform_provider = MultiplexerPlatform.build(
             prefix=application_configuration.openstack_configuration.vm_prefix,
             github_configuration=application_configuration.github_config,
+            jobmanager_configuration=application_configuration.jobmanager_config,
         )
 
         runner_manager = RunnerManager(
@@ -163,6 +164,7 @@ def build(  # pylint: disable-msg=too-many-locals
                 queue=reactive_config.queue,
                 manager_name=application_configuration.name,
                 github_configuration=application_configuration.github_config,
+                jobmanager_configuration=application_configuration.jobmanager_config,
                 cloud_runner_manager=openstack_runner_manager_config,
                 supported_labels=supported_labels,
                 labels=labels,
@@ -174,6 +176,9 @@ def build(  # pylint: disable-msg=too-many-locals
             user=user,
             base_quantity=base_quantity,
             max_quantity=max_quantity,
+            platform_name=(
+                Platform.GITHUB if application_configuration.github_config else Platform.JOBMANAGER
+            ),
             python_path=python_path,
         )
 
@@ -190,6 +195,7 @@ def __init__(  # pylint: disable=too-many-arguments, too-many-positional-argumen
         base_quantity: int,
         max_quantity: int,
         python_path: str | None = None,
+        platform_name: Platform = Platform.GITHUB,
     ):
         """Construct the object.
 
@@ -199,13 +205,15 @@ def __init__(  # pylint: disable=too-many-arguments, too-many-positional-argumen
             user: The user to run the reactive process.
             base_quantity: The number of intended non-reactive runners.
             max_quantity: The number of maximum runners for reactive.
+            platform_name: The name of the platform used for spawning runners.
             python_path: The PYTHONPATH to access the github-runner-manager library.
         """
         self._manager = runner_manager
         self._reactive_config = reactive_process_config
         self._user = user
         self._base_quantity = base_quantity
         self._max_quantity = max_quantity
+        self._platform_name = platform_name
         self._python_path = python_path
 
     def get_runner_info(self) -> RunnerInfo:
@@ -335,7 +343,9 @@ def _reconcile_non_reactive(self, expected_quantity: int) -> _ReconcileResult:
         runner_diff = expected_quantity - len(runners)
         if runner_diff > 0:
             try:
-                self._manager.create_runners(num=runner_diff, metadata=RunnerMetadata())
+                self._manager.create_runners(
+                    num=runner_diff, metadata=RunnerMetadata(platform_name=self._platform_name)
+                )
             except MissingServerConfigError:
                 logging.exception(
                     "Unable to spawn runner due to missing server configuration, "

github-runner-manager/src/github_runner_manager/platform/jobmanager_provider.py

@@ -7,12 +7,12 @@
 from enum import Enum
 
 import jobmanager_client
-from jobmanager_client.models.v1_jobs_job_id_token_post_request import V1JobsJobIdTokenPostRequest
 from jobmanager_client.rest import ApiException, NotFoundException
 from pydantic import HttpUrl
 from pydantic.error_wrappers import ValidationError
 from urllib3.exceptions import RequestError
 
+from github_runner_manager.configuration.jobmanager import JobManagerConfiguration
 from github_runner_manager.manager.models import (
     InstanceID,
     RunnerContext,
@@ -38,14 +38,25 @@
 class JobManagerPlatform(PlatformProvider):
     """Manage self-hosted runner on the JobManager."""
 
+    def __init__(self, url: str):
+        """Construct the object.
+
+        Args:
+            url: The jobmanager base URL.
+        """
+        self._url = url
+
     @classmethod
-    def build(cls) -> "JobManagerPlatform":
+    def build(cls, jobmanager_configuration: JobManagerConfiguration) -> "JobManagerPlatform":
         """Build a new instance of the JobManagerPlatform.
 
+        Args:
+            jobmanager_configuration: Configuration for the jobmanager.
+
         Returns:
             New JobManagerPlatform.
         """
-        return cls()
+        return cls(url=jobmanager_configuration.url)
 
     def get_runner_health(
         self,
@@ -62,11 +73,11 @@ def get_runner_health(
         Returns:
            The health of the runner in the jobmanager.
         """
-        configuration = jobmanager_client.Configuration(host=runner_identity.metadata.url)
+        configuration = jobmanager_client.Configuration(host=self._url)
         with jobmanager_client.ApiClient(configuration) as api_client:
-            api_instance = jobmanager_client.DefaultApi(api_client)
+            api_instance = jobmanager_client.RunnersApi(api_client)
             try:
-                response = api_instance.v1_jobs_job_id_health_get(
+                response = api_instance.get_runner_health_v1_runner_runner_id_health_get(
                     int(runner_identity.metadata.runner_id)
                 )
             except NotFoundException:
@@ -132,14 +143,14 @@ def get_runners_health(self, requested_runners: list[RunnerIdentity]) -> Runners
         )
 
     def delete_runner(self, runner_identity: RunnerIdentity) -> None:
-        """Delete a runner from jobmanager..
+        """Delete a runner from jobmanager.
 
         This method does nothing, as the jobmanager does not implement it.
 
         Args:
             runner_identity: The identity of the runner to delete.
         """
-        logger.debug("No need to delete jobs in the jobmanager.")
+        logger.debug("No need to delete runners in the jobmanager.")
 
     def get_runner_context(
         self, metadata: RunnerMetadata, instance_id: InstanceID, labels: list[str]
@@ -159,17 +170,29 @@ def get_runner_context(
         Returns:
             New runner token.
         """
-        configuration = jobmanager_client.Configuration(host=metadata.url)
+        configuration = jobmanager_client.Configuration(host=self._url)
         with jobmanager_client.ApiClient(configuration) as api_client:
-            api_instance = jobmanager_client.DefaultApi(api_client)
+            api_instance = jobmanager_client.RunnersApi(api_client)
             try:
-                # Retrieve jobs
-                jobrequest = V1JobsJobIdTokenPostRequest(job_id=int(metadata.runner_id))
-                response = api_instance.v1_jobs_job_id_token_post(
-                    int(metadata.runner_id), jobrequest
+                runner_register_request = (
+                    jobmanager_client.RegisterRunnerV1RunnerRegisterPostRequest(
+                        name=instance_id.name, labels=labels
+                    )
+                )
+
+                response = api_instance.register_runner_v1_runner_register_post(
+                    runner_register_request
                 )
+                if not response.id:
+                    raise PlatformApiError("No runner ID from jobmanager API")
+                updated_metadata = RunnerMetadata(
+                    platform_name=metadata.platform_name, url=self._url
+                )
+                updated_metadata.runner_id = str(response.id)
                 if token := response.token:
-                    jobmanager_endpoint = f"{metadata.url}/v1/jobs/{metadata.runner_id}/health"
+                    jobmanager_endpoint = (
+                        f"{self._url}/v1/runner/{updated_metadata.runner_id}/health"
+                    )
                     # For now, use the first label
                     label = "undefined"
                     if labels:
@@ -190,7 +213,7 @@ def get_runner_context(
                                 metadata=metadata,
                             ),
                             busy=False,
-                            id=int(metadata.runner_id),
+                            id=int(updated_metadata.runner_id),
                             labels=[SelfHostedRunnerLabel(name=label) for label in labels],
                             status=GitHubRunnerStatus.OFFLINE,
                         ),
@@ -209,17 +232,46 @@ def check_job_been_picked_up(self, metadata: RunnerMetadata, job_url: HttpUrl) -
 
         Raises:
             PlatformApiError: Problem with the underlying client.
+            ValueError: Raised when the job_url is malformed.
 
         Returns:
             True if the job has been picked up, False otherwise.
         """
-        configuration = jobmanager_client.Configuration(host=metadata.url)
+        configuration = jobmanager_client.Configuration(host=self._url)
+
+        # job_url has the path:
+        # "/v1/jobs/<job_id>"
+        job_path_prefix = "/v1/jobs/"
+
+        path = job_url.path
+        if not (path and path.startswith(job_path_prefix)):
+            logger.error(
+                "Job URL path does not start with '%s'. Received %s", job_path_prefix, path
+            )
+            raise ValueError(f'Job URL path does not start with "{job_path_prefix}"')
+        try:
+            job_id = int(path[len(job_path_prefix) :])  # Extract job_id from the path
+        except ValueError as exc:
+            logger.error(
+                "Job URL path %s does not contain a valid job_id after '%s'",
+                path,
+                job_path_prefix,
+            )
+            raise ValueError(
+                f"Job URL path does not contain a valid job_id after '{job_path_prefix}'"
+            ) from exc
+        logging.debug(
+            "Parsed job_id: %s from job_url path %s",
+            job_id,
+            path,
+        )
 
         with jobmanager_client.ApiClient(configuration) as api_client:
-            api_instance = jobmanager_client.DefaultApi(api_client)
+            api_instance = jobmanager_client.JobsApi(api_client)
             try:
-                job = api_instance.v1_jobs_job_id_get(int(metadata.runner_id))
-                if job.status != JobStatus.PENDING:
+                job = api_instance.get_job_v1_jobs_job_id_get(job_id)
+                # the api returns a generic object, ignore the type for status
+                if job.status != JobStatus.PENDING:  # type: ignore
                     return True
             except (ApiException, RequestError, ValidationError) as exc:
                 logger.exception("Error calling jobmanager api to get job information.")