[ISD-3751] Fix: rejecting without retry for reactive job not found. (#581)

Author: yhaliaw

docs/changelog.md

@@ -2,11 +2,13 @@
 
 This changelog documents user-relevant changes to the GitHub runner charm.
 
+## 2025-06-24
 
-### 2025-06-17
+- Fix a bug where deleted GitHub Actions Job would cause an endless loop of retries.
 
-- Fix bug where SSH connection error always appears in the logs.
+### 2025-06-17
 
+- Fix a bug where SSH connection error always appears in the logs.
 
 ### 2025-06-16
 

github-runner-manager/src/github_runner_manager/github_client.py

@@ -342,14 +342,22 @@ def get_job_info(self, path: GitHubRepo, job_id: str) -> JobInfo:
             path: GitHub repository path in the format '<owner>/<repo>'.
             job_id: The job id.
 
+        Raises:
+            JobNotFoundError: Cannot find job on GitHub.
+
         Returns:
             The JSON response from the API.
         """
-        job_raw = self._client.actions.get_job_for_workflow_run(
-            owner=path.owner,
-            repo=path.repo,
-            job_id=job_id,
-        )
+        try:
+            job_raw = self._client.actions.get_job_for_workflow_run(
+                owner=path.owner,
+                repo=path.repo,
+                job_id=job_id,
+            )
+        except HTTPError as exc:
+            if exc.code == 404:
+                raise JobNotFoundError(f"Could not find job for job id {job_id}.") from exc
+            raise
         return self._to_job_info(job_raw)
 
     @staticmethod

github-runner-manager/src/github_runner_manager/reactive/consumer.py

@@ -20,7 +20,7 @@
 
 from github_runner_manager.manager.models import RunnerMetadata
 from github_runner_manager.manager.runner_manager import RunnerManager
-from github_runner_manager.platform.platform_provider import PlatformProvider
+from github_runner_manager.platform.platform_provider import JobNotFoundError, PlatformProvider
 from github_runner_manager.reactive.types_ import QueueConfig
 
 logger = logging.getLogger(__name__)
@@ -90,7 +90,8 @@ def get_queue_size(queue_config: QueueConfig) -> int:
         raise QueueError("Error when communicating with the queue") from exc
 
 
-def consume(
+# Ignore `consume` too complex as it is pending re-design.
+def consume(  # noqa: C901
     queue_config: QueueConfig,
     runner_manager: RunnerManager,
     platform_provider: PlatformProvider,
@@ -145,12 +146,18 @@ def consume(
                 except ValueError:
                     msg.reject(requeue=False)
                     break
-                if platform_provider.check_job_been_picked_up(
-                    metadata=metadata, job_url=job_details.url
-                ):
-                    logger.info("reactive job: %s already picked up.", job_details)
-                    msg.ack()
-                    continue
+                try:
+                    if platform_provider.check_job_been_picked_up(
+                        metadata=metadata, job_url=job_details.url
+                    ):
+                        logger.info("reactive job: %s already picked up.", job_details)
+                        msg.ack()
+                        continue
+                except JobNotFoundError:
+                    logger.warning(
+                        "Unable to find the job %s. Not retrying this job.", job_details.url
+                    )
+                    msg.reject(requeue=False)
                 _spawn_runner(
                     runner_manager=runner_manager,
                     job_url=job_details.url,

tests/integration/test_reactive.py

@@ -10,6 +10,7 @@
 import pytest_asyncio
 from github import Branch, Repository
 from github_runner_manager.manager.cloud_runner_manager import PostJobStatus
+from github_runner_manager.reactive.consumer import JobDetails
 from juju.application import Application
 from pytest_operator.plugin import OpsTest
 
@@ -136,6 +137,37 @@ async def _runner_installed_in_metrics_log() -> bool:
     await _assert_metrics_are_logged(app, github_repository)
 
 
+@pytest.mark.abort_on_fail
+async def test_reactive_mode_with_not_found_job(
+    ops_test: OpsTest,
+    app: Application,
+):
+    """
+    arrange: Place a message in the queue with a non-existent job url.
+    act: Call reconcile.
+    assert: The message is removed from the queue.
+    """
+    mongodb_uri = await get_mongodb_uri(ops_test, app)
+
+    labels = {app.name, "x64"}  # The architecture label should be ignored in the
+    # label validation in the reactive consumer.
+    job = JobDetails(
+        labels=labels,
+        url="https://github.com/canonical/github-runner-operator/actions/runs/mock-run/job/mock-job",
+    )
+    add_to_queue(
+        json.dumps(json.loads(job.json()) | {"ignored_noise": "foobar"}),
+        mongodb_uri,
+        app.name,
+    )
+
+    # There may be a race condition between getting the token and spawning the machine.
+    await sleep(10)
+    await wait_for_reconcile(app)
+
+    assert_queue_is_empty(mongodb_uri, app.name)
+
+
 @pytest.mark.abort_on_fail
 async def test_reactive_mode_does_not_consume_jobs_with_unsupported_labels(
     ops_test: OpsTest,