Retry on Fedora CI Koji build submission failure (#3040)

Retry on Fedora CI Koji build submission failure

Fixes #3003
Assisted-by: Claude Opus 4.6 noreply@anthropic.com
RELEASE NOTES BEGIN
Packit now retries on build submission failure for Fedora CI Koji scratch builds.
RELEASE NOTES END

Reviewed-by: gemini-code-assist[bot]
Reviewed-by: Nikola Forró

Author: centosinfra-prod-github-app[bot]

packit_service/worker/handlers/distgit.py

@@ -32,6 +32,7 @@
 from packit_service import sentry_integration
 from packit_service.config import ServiceConfig
 from packit_service.constants import (
+    BASE_RETRY_INTERVAL_IN_MINUTES_FOR_OUTAGES,
     CONTACTS_URL,
     DEFAULT_RETRY_BACKOFF,
     MSG_DOWNSTREAM_JOB_ERROR_HEADER,
@@ -937,6 +938,29 @@ def _run(self) -> TaskResults:
             )
             self.pushgateway.fedora_ci_koji_builds_queued.inc()
         except Exception as ex:
+            if (
+                isinstance(ex, PackitCommandFailedError)
+                and self.celery_task
+                and not self.celery_task.is_last_try()
+            ):
+                koji_build.set_status("retry")
+                interval = BASE_RETRY_INTERVAL_IN_MINUTES_FOR_OUTAGES * 2**self.celery_task.retries
+                self.report(
+                    commit_status=BaseCommitStatus.pending,
+                    description="Failed to submit the build. The task will be"
+                    f" retried in {interval} {'minute' if interval == 1 else 'minutes'}.",
+                    url=get_koji_build_info_url(koji_build.id),
+                )
+                kargs = self.celery_task.task.request.kwargs.copy()
+                self.celery_task.retry(delay=interval * 60, kargs=kargs)
+                return TaskResults(
+                    success=True,
+                    details={
+                        "msg": "Task will be retried because of failure"
+                        f" when submitting the build: {ex}",
+                    },
+                )
+
             sentry_integration.send_to_sentry(ex)
             self.report(
                 commit_status=BaseCommitStatus.error,

tests/integration/test_koji_build_cancel.py

@@ -10,7 +10,7 @@
 from ogr.services.pagure import PagureProject
 from packit.api import PackitAPI
 from packit.config import Deployment, JobConfigTriggerType
-from packit.exceptions import PackitException
+from packit.exceptions import PackitCommandFailedError, PackitException
 from packit.local_project import LocalProjectBuilder
 from packit.utils import commands
 
@@ -244,6 +244,55 @@ def test_downstream_koji_scratch_build_cancel_running(mock_distgit_pr_functional
     assert first_dict_value(results["job"])["success"]
 
 
+def test_downstream_koji_scratch_build_retry_on_submission_failure(
+    mock_distgit_pr_functionality,
+):
+    """Test that DownstreamKojiScratchBuildHandler retries on build submission failure.
+
+    Simulates a Koji CLI failure (e.g. network issue, Koji outage) and verifies
+    the handler sets retry status and schedules a retry via Celery.
+    """
+    flexmock(PackitAPI).should_receive("init_kerberos_ticket")
+    koji_build_target = flexmock(
+        id=123,
+        target="main",
+        status="queued",
+    )
+    koji_build_target.should_receive("set_status").with_args("retry").once()
+    koji_build_target.should_receive("set_task_id")
+    koji_build_target.should_receive("set_web_url")
+    koji_build_target.should_receive("set_build_logs_urls")
+    koji_build_target.should_receive("set_data")
+    koji_build_target.should_receive("set_build_submission_stdout")
+    flexmock(KojiBuildTargetModel).should_receive("create").and_return(koji_build_target)
+    flexmock(KojiBuildGroupModel).should_receive("create").and_return(
+        flexmock(grouped_targets=[koji_build_target]),
+    )
+
+    # Simulate koji CLI failure
+    flexmock(commands).should_receive("run_command_remote").and_raise(
+        PackitCommandFailedError,
+        "Command failed",
+        stdout_output="",
+        stderr_output="koji: AuthError: unable to obtain a session",
+    )
+
+    # Mock the Celery task's retry method to prevent actual retry and verify it's called
+    flexmock(run_downstream_koji_scratch_build_handler).should_receive("retry").once()
+
+    processing_results = SteveJobs().process_message(mock_distgit_pr_functionality)
+    event_dict, _, job_config, package_config = get_parameters_from_results(
+        processing_results[:1],
+    )
+    results = run_downstream_koji_scratch_build_handler(
+        package_config=package_config,
+        event=event_dict,
+        job_config=job_config,
+    )
+
+    assert first_dict_value(results["job"])["success"]
+
+
 def test_downstream_koji_build_cancel_running(monkeypatch):
     """Test that DownstreamKojiBuildHandler calls cancel_running_builds.