✨Adding tracing in fastapi-based services (⚠️ devops) (ITISFoundation#2558)

Author: sanderegg

.env-devel

@@ -67,6 +67,7 @@ STORAGE_ENDPOINT=storage:8080
 
 TRACING_ENABLED=1
 TRACING_ZIPKIN_ENDPOINT=http://jaeger:9411
+TRACING_THRIFT_COMPACT_ENDPOINT=http://jaeger:5775
 
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
 

.github/workflows/ci-testing-deploy.yml

@@ -354,9 +354,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: install
-        run: ./ci/github/unit-testing/director_v2.bash install
+        run: ./ci/github/unit-testing/director-v2.bash install
       - name: test
-        run: ./ci/github/unit-testing/director_v2.bash test
+        run: ./ci/github/unit-testing/director-v2.bash test
       - name: upload failed tests logs
         if: failure()
         uses: actions/upload-artifact@v2
@@ -747,7 +747,6 @@ jobs:
           name: codeclimate-${{ github.job }}-coverage
           path: codeclimate.${{ github.job }}_coverage.json
 
-
   unit-test-service-library:
     timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] service-library"

ci/github/unit-testing/catalog.bash

@@ -1,25 +1,26 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -o errexit   # abort on nonzero exitstatus
-set -o nounset   # abort on unbound variable
-set -o pipefail  # don't hide errors within pipes
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-    bash ci/helpers/ensure_python_pip.bash
-    pushd services/catalog; pip3 install -r requirements/ci.txt; popd
-    pip list --verbose
+  bash ci/helpers/ensure_python_pip.bash
+  pushd services/catalog
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list --verbose
 }
 
 test() {
-    pytest --cov=simcore_service_catalog --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/catalog/tests/unit
+  pytest --cov=simcore_service_catalog --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" services/catalog/tests/unit
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

ci/github/unit-testing/director_v2.bash renamed to ci/github/unit-testing/director-v2.bash

@@ -19,7 +19,7 @@ test() {
     # these tests cannot be run in parallel
     pytest --cov=simcore_service_director_v2 --durations=10 --cov-append \
           --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/director-v2/tests/unit/with_dbs services/director-v2/tests/unit/with_swarm;
+          -v -m "not travis" services/director-v2/tests/unit/with_swarm services/director-v2/tests/unit/with_dbs;
 }
 
 # Check if the function exists (bash specific)

packages/models-library/src/models_library/settings/http_clients.py

@@ -4,11 +4,11 @@
 
 
 class ClientRequestSettings(BaseSettings):
-    # NOTE: when updating the defaults please make sure to search for the env vars
-    # in all the project, they also need to be updated inside the service-library
+    # NOTE: These entries are used in some old services as well. These need to be updated if these
+    # variable names or defaults are changed.
     total_timeout: Optional[int] = Field(
         default=20,
-        description="timeout used for outgoing http requests",
+        description="timeout in seconds used for outgoing http requests",
         env="HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT",
     )
 

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -17,6 +17,7 @@
 
 import pytest
 import yaml
+from _pytest.config import ExitCode
 from dotenv import dotenv_values
 
 from .helpers import (
@@ -234,6 +235,15 @@ def pytest_exception_interact(node, call, report):
     save_docker_infos(failed_test_directory)
 
 
+@pytest.hookimpl()
+def pytest_sessionfinish(session: pytest.Session, exitstatus: ExitCode) -> None:
+    if exitstatus == ExitCode.TESTS_FAILED:
+        # get the node root dir (guaranteed to exist)
+        root_directory: Path = Path(session.fspath)
+        failed_test_directory = root_directory / "test_failures" / session.name
+        save_docker_infos(failed_test_directory)
+
+
 # HELPERS ---------------------------------------------
 def _minio_fix(service_environs: Dict) -> Dict:
     """this hack ensures that S3 is accessed from the host at all time, thus pre-signed links work."""

packages/pytest-simcore/src/pytest_simcore/docker_swarm.py

@@ -15,6 +15,10 @@
 import pytest
 import tenacity
 import yaml
+from docker.errors import APIError
+from tenacity.before_sleep import before_sleep_log
+from tenacity.stop import stop_after_attempt, stop_after_delay
+from tenacity.wait import wait_exponential, wait_fixed
 
 from .helpers.utils_docker import get_ip
 
@@ -36,7 +40,7 @@ def _in_docker_swarm(
         docker_client.swarm.reload()
         inspect_result = docker_client.swarm.attrs
         assert type(inspect_result) == dict
-    except docker.errors.APIError as error:
+    except APIError as error:
         if raise_error:
             raise _NotInSwarmException() from error
         return False
@@ -45,8 +49,8 @@ def _in_docker_swarm(
 
 def _attempt_for(retry_error_cls: Type[Exception]) -> tenacity.Retrying:
     return tenacity.Retrying(
-        wait=tenacity.wait_exponential(),
-        stop=tenacity.stop_after_delay(15),
+        wait=wait_exponential(),
+        stop=stop_after_delay(15),
         retry_error_cls=retry_error_cls,
     )
 
@@ -77,17 +81,8 @@ def docker_swarm(
 
     yield
 
-    for attempt in _attempt_for(retry_error_cls=_StillInSwarmException):
-        with attempt:
-            if _in_docker_swarm(docker_client):
-                if not keep_docker_up:
-                    assert docker_client.swarm.leave(force=True)
-
-            if _in_docker_swarm(docker_client) and not keep_docker_up:
-                # if still in swarm, raise an error to try and leave again
-                raise _StillInSwarmException()
-            if keep_docker_up:
-                assert _in_docker_swarm(docker_client) is True
+    if not keep_docker_up:
+        assert docker_client.swarm.leave(force=True)
 
     assert _in_docker_swarm(docker_client) is keep_docker_up
 
@@ -109,9 +104,9 @@ def by_task_update(task: Dict) -> datetime:
 
 
 @tenacity.retry(
-    wait=tenacity.wait_fixed(5),
-    stop=tenacity.stop_after_attempt(20),
-    before_sleep=tenacity.before_sleep_log(log, logging.INFO),
+    wait=wait_fixed(5),
+    stop=stop_after_attempt(20),
+    before_sleep=before_sleep_log(log, logging.INFO),
     reraise=True,
 )
 def _wait_for_services(docker_client: docker.client.DockerClient) -> None:
@@ -237,4 +232,17 @@ def docker_stack(
         ):
             time.sleep(WAIT_BEFORE_RETRY_SECS)
 
+        while docker_client.containers.list(
+            filters={"label": f"com.docker.stack.namespace={stack}"}
+        ):
+            time.sleep(WAIT_BEFORE_RETRY_SECS)
+
+        for attempt in _attempt_for(retry_error_cls=APIError):
+            with attempt:
+                list_of_volumes = docker_client.volumes.list(
+                    filters={"label": f"com.docker.stack.namespace={stack}"}
+                )
+                for volume in list_of_volumes:
+                    volume.remove(force=True)
+
     _print_services(docker_client, "[AFTER REMOVED]")

packages/pytest-simcore/src/pytest_simcore/postgres_service.py

@@ -12,6 +12,9 @@
 import tenacity
 from simcore_postgres_database.models.base import metadata
 from sqlalchemy.orm import sessionmaker
+from tenacity.before_sleep import before_sleep_log
+from tenacity.stop import stop_after_attempt
+from tenacity.wait import wait_fixed
 
 from .helpers.utils_docker import get_service_published_port
 
@@ -168,7 +171,7 @@ def postgres_db(
     postgres_dsn: Dict[str, str],
     postgres_engine: sa.engine.Engine,
 ) -> Iterator[sa.engine.Engine]:
-    """ An postgres database init with empty tables and an sqlalchemy engine connected to it """
+    """An postgres database init with empty tables and an sqlalchemy engine connected to it"""
 
     # upgrades database from zero
     kwargs = postgres_dsn.copy()
@@ -196,7 +199,7 @@ def postgres_db(
 async def aiopg_engine(
     postgres_db: sa.engine.Engine, loop
 ) -> Iterator[aiopg.sa.engine.Engine]:
-    """ An aiopg engine connected to an initialized database """
+    """An aiopg engine connected to an initialized database"""
     from aiopg.sa import create_engine
 
     engine = await create_engine(str(postgres_db.url))
@@ -234,13 +237,13 @@ def postgres_session(postgres_db: sa.engine.Engine) -> sa.orm.session.Session:
 
 
 @tenacity.retry(
-    wait=tenacity.wait_fixed(5),
-    stop=tenacity.stop_after_attempt(60),
-    before_sleep=tenacity.before_sleep_log(log, logging.INFO),
+    wait=wait_fixed(5),
+    stop=stop_after_attempt(60),
+    before_sleep=before_sleep_log(log, logging.WARNING),
     reraise=True,
 )
 def wait_till_postgres_is_responsive(url: str) -> None:
-    print("Trying", url, "...")
     engine = sa.create_engine(url, isolation_level="AUTOCOMMIT")
     conn = engine.connect()
     conn.close()
+    log.info("Connected with %s", url)

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -0,0 +1,16 @@
+from fastapi import FastAPI
+from fastapi_contrib.conf import settings
+from fastapi_contrib.tracing.middlewares import OpentracingMiddleware
+from fastapi_contrib.tracing.utils import setup_opentracing
+from settings_library.tracing import TracingSettings
+
+
+def setup_tracing(app: FastAPI, tracing_settings: TracingSettings):
+    async def start_app() -> None:
+        settings.service_name = tracing_settings.TRACING_CLIENT_NAME
+        settings.jaeger_host = tracing_settings.TRACING_THRIFT_COMPACT_ENDPOINT.host
+        settings.jaeger_port = tracing_settings.TRACING_THRIFT_COMPACT_ENDPOINT.port
+        setup_opentracing(app)
+        app.add_middleware(OpentracingMiddleware)
+
+    app.add_event_handler("startup", start_app)

packages/settings-library/src/settings_library/http_client_request.py

@@ -0,0 +1,33 @@
+from typing import Optional
+
+from pydantic import Field
+
+from .base import BaseCustomSettings
+
+
+class ClientRequestSettings(BaseCustomSettings):
+    # NOTE: These entries are used in some old services as well. These need to be updated if these
+    # variable names or defaults are changed.
+    HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT: Optional[int] = Field(
+        default=20,
+        description="timeout in seconds used for outgoing http requests",
+    )
+
+    HTTP_CLIENT_REQUEST_AIOHTTP_CONNECT_TIMEOUT: Optional[int] = Field(
+        default=None,
+        description=(
+            "Maximal number of seconds for acquiring a connection"
+            " from pool. The time consists connection establishment"
+            " for a new connection or waiting for a free connection"
+            " from a pool if pool connection limits are exceeded. "
+            "For pure socket connection establishment time use sock_connect."
+        ),
+    )
+
+    HTTP_CLIENT_REQUEST_AIOHTTP_SOCK_CONNECT_TIMEOUT: Optional[int] = Field(
+        default=5,
+        description=(
+            "aiohttp specific field used in ClientTimeout, timeout for connecting to a "
+            "peer for a new connection not given a pool"
+        ),
+    )