RHOAIENG-17695: chore(ci): disable ryuk for gha runs and when it actually fails to start (most likely due to rootless podman) (#849)

* RHOAIENG-17695: chore(ci): disable ryuk for gha runs and when it actually fails to start (most likely due to rootless podman)

* fixup, change the logic to only suggest disabling Ryuk, not disabling automatically

Author: jiridanek

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -291,16 +291,12 @@ jobs:
       - name: Run container tests (in PyTest)
         run: |
           set -Eeuxo pipefail
-          # retry to increase CI reliability
-          for i in {1..5}; do
-            if podman pull --retry 10 "${RYUK_CONTAINER_IMAGE}"; then break; fi
-          done
-          # now run the tests
           poetry run pytest --capture=fd tests/containers --image="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
         env:
           DOCKER_HOST: "unix:///var/run/podman/podman.sock"
           TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/podman/podman.sock"
-          RYUK_CONTAINER_IMAGE: "testcontainers/ryuk:0.8.1"
+          # pulling the Ryuk container from docker.io introduces CI flakiness
+          TESTCONTAINERS_RYUK_DISABLED: "true"
 
       # endregion Pytest image tests
 

tests/containers/conftest.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import os
 from typing import Iterable, TYPE_CHECKING
 
@@ -65,7 +66,15 @@ def pytest_sessionstart(session: Session) -> None:
 
     # second preflight check: start the Reaper container
     if not testcontainers.core.config.testcontainers_config.ryuk_disabled:
-        assert testcontainers.core.container.Reaper.get_instance() is not None, "Failed to start Reaper container"
+        # when running on rootless podman, ryuk fails to start and may need to be disabled
+        # https://java.testcontainers.org/supported_docker_environment/#podman
+        logging.warning("Ryuk is enabled. This may not work with rootless podman.")
+        try:
+            assert testcontainers.core.container.Reaper.get_instance() is not None, "Failed to start Reaper container"
+        except Exception as e:
+            logging.exception("Failed to start the Ryuk Reaper container", exc_info=e)
+            logging.error(f"Set env variable 'export TESTCONTAINERS_RYUK_DISABLED=true' and try again.")
+            raise RuntimeError("Consider disabling Ryuk as per the log messages above.") from e
 
 
 # https://docs.pytest.org/en/latest/reference/reference.html#pytest.hookspec.pytest_sessionfinish