✅ maintenance: fixes random failures to deploy the stack upon integration tests setup (ITISFoundation#2630)

WARNING: this has a test disabled. New PR should re-enable it back

Author: pcrespov

.github/workflows/ci-testing-deploy.yml

@@ -1905,7 +1905,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   integration-test-director-v2-01:
-    timeout-minutes: 30 # if this timeout gets too small, then split the tests
+    timeout-minutes: 35 # FIXME: temporary up (PC). if this timeout gets too small, then split the tests
     name: "[int] director-v2 01"
     needs: [build-test-images]
     runs-on: ${{ matrix.os }}
@@ -2198,7 +2198,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   system-test-public-api:
-    timeout-minutes: 20 # if this timeout gets too small, then split the tests
+    timeout-minutes: 25 # if this timeout gets too small, then split the tests
     name: "[sys] public api"
     needs: [build-test-images]
     runs-on: ${{ matrix.os }}
@@ -2301,6 +2301,12 @@ jobs:
         run: ./ci/github/system-testing/swarm-deploy.bash install
       - name: test
         run: ./ci/github/system-testing/swarm-deploy.bash test
+      - name: upload failed tests logs
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./tests/swarm-deploy/test_failures
       - name: cleanup
         if: always()
         run: ./ci/github/system-testing/swarm-deploy.bash clean_up
@@ -2366,24 +2372,24 @@ jobs:
         run: ./ci/github/system-testing/e2e.bash install
       - name: test
         run: ./ci/github/system-testing/e2e.bash test
-      - name: recover docker logs
-        id: logs_recovery
-        if: always()
+      - name: dump docker logs
+        id: docker_logs_dump
+        if: failure()
         run: ./ci/github/system-testing/e2e.bash dump_docker_logs
       - name: upload docker logs
-        if: always()
+        if: failure()
         uses: actions/upload-artifact@v2
         with:
           name: ${{ github.job }}_docker_logs
-          path: simcore_logs
+          path: ./tests/e2e/test_failures
       - name: upload screenshots
         if: always()
         uses: actions/upload-artifact@v2
         with:
           name: ${{ github.job }}_screenshots
           path: tests/e2e/screenshots
       - name: upload e2e logs
-        if: always()
+        if: failure()
         uses: actions/upload-artifact@v2
         with:
           name: ${{ github.job }}_logs

api/tests/test_repo_data.py

@@ -23,7 +23,7 @@
 # TODO: find json files under services with the word project or similar wildcard??
 PROJECTS_PATHS = [
     "services/web/server/tests/data/fake-project.json",
-    "services/web/server/tests/integration/02/workbench_sleeper_payload.json",
+    "services/web/server/tests/data/workbench_sleeper_payload.json",
     "services/web/server/src/simcore_service_webserver/data/fake-template-projects.isan.json",
     "services/web/server/src/simcore_service_webserver/data/fake-template-projects.osparc.json",
     "services/web/server/src/simcore_service_webserver/data/fake-template-projects.json",

ci/github/system-testing/e2e.bash

@@ -48,17 +48,14 @@ clean_up() {
 dump_docker_logs() {
   # get docker logs
   # NOTE: Timeout avoids issue with dumping logs that hang!
-  mkdir --parents simcore_logs
-  (timeout 30 docker service logs --timestamps --tail=300 --details "${SWARM_STACK_NAME:-test}"_webserver >simcore_logs/webserver.log 2>&1) || true
-  # then the rest (alphabetically)
-  (timeout 30 docker service logs --timestamps --tail=100 --details "${SWARM_STACK_NAME:-test}"_api-server >simcore_logs/api-server.log 2>&1) || true
-  (timeout 30 docker service logs --timestamps --tail=200 --details "${SWARM_STACK_NAME:-test}"_catalog >simcore_logs/catalog.log 2>&1) || true
-  (timeout 30 docker service logs --timestamps --tail=200 --details "${SWARM_STACK_NAME:-test}"_director >simcore_logs/director.log 2>&1) || true
-  (timeout 30 docker service logs --timestamps --tail=200 --details "${SWARM_STACK_NAME:-test}"_director-v2 >simcore_logs/director-v2.log 2>&1) || true
-  (timeout 30 docker service logs --timestamps --tail=200 --details "${SWARM_STACK_NAME:-test}"_sidecar >simcore_logs/sidecar.log 2>&1) || true
-  (timeout 30 docker service logs --timestamps --tail=200 --details "${SWARM_STACK_NAME:-test}"_storage >simcore_logs/storage.log 2>&1) || true
-  (timeout 30 docker service logs --timestamps --tail=200 --details "${SWARM_STACK_NAME:-test}"_migration >simcore_logs/migration.log 2>&1) || true
-  (timeout 30 docker service logs --timestamps --tail=200 --details "${SWARM_STACK_NAME:-test}"_postgres >simcore_logs/postgres.log 2>&1) || true
+  out_dir=tests/e2e/test_failures
+  mkdir --parents "$out_dir"
+
+  for service_id in $(docker service ls -q); do
+    service_name=$(docker service inspect "$service_id" --format="{{.Spec.Name}}")
+    echo "Dumping logs for $service_name"
+    (timeout 30 docker service logs --timestamps --tail=400 --details "$service_id" >"$out_dir/$service_name.log" 2>&1) || true
+  done
 }
 
 # Check if the function exists (bash specific)

ci/github/system-testing/public-api.bash

@@ -28,10 +28,13 @@ install() {
 }
 
 test() {
+  # WARNING: this test is heavy. Due to limited CI machine power, please do not
+  # add too much overhead (e.g. low log-level etc)
   pytest \
     --color=yes \
     --cov-report=term-missing \
     --keep-docker-up \
+    --durations=5 \
     -v \
     tests/public-api
 }

ci/github/system-testing/swarm-deploy.bash

@@ -27,7 +27,15 @@ install() {
 }
 
 test() {
-  pytest --color=yes --cov-report=term-missing -v tests/swarm-deploy --log-level=DEBUG
+  # WARNING: this test is heavy. Due to limited CI machine power, please do not
+  # add too much overhead (e.g. low log-level etc)
+  pytest \
+    --color=yes \
+    --cov-report=term-missing \
+    -v \
+    --durations=5 \
+    --log-level=INFO \
+    tests/swarm-deploy
 }
 
 clean_up() {

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -2,17 +2,18 @@
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
 
-""" Fixtures to create docker-compose.yaml configururation files (as in Makefile)
+""" Fixtures to create docker-compose.yaml configuration files (as in Makefile)
 
-    Basically runs `docker-compose config
+    - Basically runs `docker-compose config
+    - Services in stack can be selected using 'core_services_selection', 'ops_services_selection' fixtures
 """
 
+import json
 import os
 import shutil
 import sys
 from copy import deepcopy
 from pathlib import Path
-from pprint import pformat
 from typing import Any, Dict, Iterator, List
 
 import pytest
@@ -24,6 +25,7 @@
     FIXTURE_CONFIG_CORE_SERVICES_SELECTION,
     FIXTURE_CONFIG_OPS_SERVICES_SELECTION,
 )
+from .helpers.constants import HEADER_STR
 from .helpers.utils_docker import get_ip, run_docker_compose_config, save_docker_infos
 from .helpers.utils_environs import EnvVarsDict
 
@@ -143,7 +145,12 @@ def simcore_docker_compose(
         env_file_path=env_file_for_testing,
         destination_path=temp_folder / "simcore_docker_compose.yml",
     )
-    print("simcore docker-compose:\n%s", pformat(config))
+    # NOTE: do not add indent. Copy&Paste log into editor instead
+    print(
+        HEADER_STR.format("simcore docker-compose"),
+        json.dumps(config),
+        HEADER_STR.format("-"),
+    )
     return config
 
 
@@ -170,7 +177,12 @@ def ops_docker_compose(
         env_file_path=env_file_for_testing,
         destination_path=temp_folder / "ops_docker_compose.yml",
     )
-    print("ops docker-compose:\n%s", pformat(config))
+    # NOTE: do not add indent. Copy&Paste log into editor instead
+    print(
+        HEADER_STR.format("ops docker-compose"),
+        json.dumps(config),
+        HEADER_STR.format("-"),
+    )
     return config
 
 
@@ -239,13 +251,14 @@ def pytest_exception_interact(node, call, report):
 @pytest.hookimpl()
 def pytest_sessionfinish(session: pytest.Session, exitstatus: ExitCode) -> None:
     if exitstatus == ExitCode.TESTS_FAILED:
-        # get the node root dir (guaranteed to exist)
         root_directory: Path = Path(session.fspath)
         failed_test_directory = root_directory / "test_failures" / session.name
         save_docker_infos(failed_test_directory)
 
 
 # HELPERS ---------------------------------------------
+
+
 def _minio_fix(service_environs: Dict) -> Dict:
     """this hack ensures that S3 is accessed from the host at all time, thus pre-signed links work."""
     if "S3_ENDPOINT" in service_environs:

packages/pytest-simcore/src/pytest_simcore/docker_registry.py

@@ -21,6 +21,7 @@
 
 @pytest.fixture(scope="session")
 def docker_registry(keep_docker_up: bool) -> str:
+    """sets up and runs a docker registry container locally and returns its URL"""
     # run the registry outside of the stack
     docker_client = docker.from_env()
     # try to login to private registry