✅ Is3569/flaky public api test (ITISFoundation#3634)

Author: pcrespov

tests/public-api/test_solvers_jobs_api.py

@@ -8,6 +8,7 @@
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
 
+import logging
 import time
 from operator import attrgetter
 from pathlib import Path
@@ -16,14 +17,23 @@
 from zipfile import ZipFile
 
 import osparc
+import osparc.exceptions
 import pytest
 from osparc import FilesApi, SolversApi
 from osparc.models import File, Job, JobInputs, JobOutputs, JobStatus, Solver
+from tenacity import Retrying, TryAgain
+from tenacity.after import after_log
+from tenacity.retry import retry_if_exception_type
+from tenacity.stop import stop_after_attempt
+from tenacity.wait import wait_fixed
 
 OSPARC_CLIENT_VERSION = tuple(map(int, osparc.__version__.split(".")))
 assert OSPARC_CLIENT_VERSION >= (0, 4, 3)
 
 
+logger = logging.getLogger(__name__)
+
+
 @pytest.fixture(scope="module")
 def sleeper_solver(
     solvers_api: SolversApi,
@@ -169,6 +179,17 @@ def test_create_job(
     assert job.id != job2.id
 
 
+_RETRY_POLICY_IF_LOGFILE_404_NOT_FOUND = dict(
+    # NOTE: Only 404s https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404
+    # are retried, the rest are failures
+    retry=retry_if_exception_type(TryAgain),
+    wait=wait_fixed(1),
+    stop=stop_after_attempt(3),
+    after=after_log(logger, logging.WARNING),
+    reraise=True,
+)
+
+
 @pytest.mark.parametrize("expected_outcome", ("SUCCESS", "FAILED"))
 def test_run_job(
     uploaded_input_file: File,
@@ -264,9 +285,22 @@ def test_run_job(
     # download log (Added in on API version 0.4.0 / client version 0.5.0 )
     if OSPARC_CLIENT_VERSION >= (0, 5, 0):
         print("Testing output logfile ...")
-        logfile: str = solvers_api.get_job_output_logfile(
-            solver.id, solver.version, job.id
-        )
+
+        # NOTE: https://github.com/itisfoundation/osparc-simcore/issues/3569 shows
+        # that this test might not have the logs ready in time and returns a 404 (not found)
+        # for that reason we do a few retries before giving up
+        for attempt in Retrying(_RETRY_POLICY_IF_LOGFILE_404_NOT_FOUND):
+            with attempt:
+                try:
+                    logfile: str = solvers_api.get_job_output_logfile(
+                        solver.id, solver.version, job.id
+                    )
+                except osparc.exceptions.ApiException as err:
+                    if err.status == 404:
+                        raise TryAgain(
+                            f"get_job_output_logfile failed with {err}"
+                        ) from err
+
         zip_path = Path(logfile)
         print(
             f"{zip_path=}",