♻️✨ maintenance new settings (final round) (ITISFoundation#2836)

SEE description in ITISFoundation#2836

Author: pcrespov

Makefile

@@ -355,11 +355,13 @@ push-version: tag-version
 .PHONY: devenv devenv-all node-env
 
 .venv:
+	@python3 --version
 	python3 -m venv $@
 	$@/bin/pip3 --quiet install --upgrade \
 		pip~=21.3 \
 		wheel \
 		setuptools
+	@$@/bin/pip3 list --verbose
 
 devenv: .venv ## create a python virtual environment with dev tools (e.g. linters, etc)
 	$</bin/pip3 --quiet install -r requirements/devenv.txt

packages/models-library/src/models_library/settings/rabbit.py

@@ -29,7 +29,11 @@
 )
 from .helpers.constants import HEADER_STR
 from .helpers.typing_env import EnvVarsDict
-from .helpers.utils_docker import get_ip, run_docker_compose_config, save_docker_infos
+from .helpers.utils_docker import (
+    get_localhost_ip,
+    run_docker_compose_config,
+    save_docker_infos,
+)
 
 
 @pytest.fixture(scope="session")
@@ -49,7 +53,7 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict:
     env_devel["LOG_LEVEL"] = "DEBUG"
 
     env_devel["REGISTRY_SSL"] = "False"
-    env_devel["REGISTRY_URL"] = "{}:5000".format(get_ip())
+    env_devel["REGISTRY_URL"] = "{}:5000".format(get_localhost_ip())
     env_devel["REGISTRY_PATH"] = "127.0.0.1:5000"
     env_devel["REGISTRY_USER"] = "simcore"
     env_devel["REGISTRY_PW"] = ""
@@ -301,7 +305,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: ExitCode) -> None:
 def _minio_fix(service_environs: Dict) -> Dict:
     """this hack ensures that S3 is accessed from the host at all time, thus pre-signed links work."""
     if "S3_ENDPOINT" in service_environs:
-        service_environs["S3_ENDPOINT"] = f"{get_ip()}:9001"
+        service_environs["S3_ENDPOINT"] = f"{get_localhost_ip()}:9001"
     return service_environs
 
 

packages/models-library/src/models_library/settings/redis.py

@@ -14,7 +14,7 @@
 import pytest
 import tenacity
 
-from .helpers.utils_docker import get_ip
+from .helpers.utils_docker import get_localhost_ip
 
 log = logging.getLogger(__name__)
 
@@ -71,7 +71,7 @@ def docker_registry(keep_docker_up: bool) -> str:
     os.environ["REGISTRY_SSL"] = "False"
     os.environ["REGISTRY_AUTH"] = "False"
     # the registry URL is how to access from the container (e.g. for accessing the API)
-    os.environ["REGISTRY_URL"] = f"{get_ip()}:5000"
+    os.environ["REGISTRY_URL"] = f"{get_localhost_ip()}:5000"
     # the registry PATH is how the docker engine shall access the images (usually same as REGISTRY_URL but for testing)
     os.environ["REGISTRY_PATH"] = "127.0.0.1:5000"
     os.environ["REGISTRY_USER"] = "simcore"

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -24,13 +24,13 @@
 from .helpers.constants import HEADER_STR, MINUTE
 from .helpers.typing_env import EnvVarsDict
 from .helpers.utils_dict import copy_from_dict
-from .helpers.utils_docker import get_ip
+from .helpers.utils_docker import get_localhost_ip
 
 log = logging.getLogger(__name__)
 
 
 #
-# NOTE this file must be PYTHON >=3.6 COMPATIBLE because it is used by the director service
+# NOTE: this file must be PYTHON >=3.6 COMPATIBLE because it is used by the director service
 #
 
 # HELPERS --------------------------------------------------------------------------------
@@ -170,7 +170,7 @@ def docker_swarm(
     ):
         with attempt:
             if not _in_docker_swarm(docker_client):
-                docker_client.swarm.init(advertise_addr=get_ip())
+                docker_client.swarm.init(advertise_addr=get_localhost_ip())
             # if still not in swarm, raise an error to try and initialize again
             _in_docker_swarm(docker_client, raise_error=True)
 
@@ -217,13 +217,30 @@ def docker_stack(
     # make up-version
     stacks_deployed: Dict[str, Dict] = {}
     for key, stack_name, compose_file in stacks:
-        subprocess.run(
-            f"docker stack deploy --with-registry-auth -c {compose_file.name} {stack_name}".split(
-                " "
-            ),
-            check=True,
-            cwd=compose_file.parent,
-        )
+        try:
+            subprocess.run(
+                [
+                    "docker",
+                    "stack",
+                    "deploy",
+                    "--with-registry-auth",
+                    "--compose-file",
+                    f"{compose_file.name}",
+                    f"{stack_name}",
+                ],
+                check=True,
+                cwd=compose_file.parent,
+            )
+        except subprocess.CalledProcessError as err:
+            print(
+                "docker_stack failed",
+                f"{' '.join(err.cmd)}",
+                f"returncode={err.returncode}",
+                f"stdout={err.stdout}",
+                f"stderr={err.stderr}",
+            )
+            raise
+
         stacks_deployed[key] = {
             "name": stack_name,
             "compose": yaml.safe_load(compose_file.read_text()),

packages/pytest-simcore/src/pytest_simcore/docker_registry.py

@@ -2,6 +2,8 @@
 
 from typing import Any, Dict, Mapping, Optional, Set, Union
 
+ConfigDict = Dict[str, Any]
+
 
 def get_from_dict(obj: Mapping[str, Any], dotted_key: str, default=None) -> Any:
     keys = dotted_key.split(".")

packages/pytest-simcore/src/pytest_simcore/docker_swarm.py

@@ -14,20 +14,26 @@
 from tenacity.stop import stop_after_attempt
 from tenacity.wait import wait_fixed
 
+COLOR_ENCODING_RE = re.compile(r"\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]")
+MAX_PATH_CHAR_LEN_ALLOWED = 260
+kFILENAME_TOO_LONG = 36
+_NORMPATH_COUNT = 0
+
+
 log = logging.getLogger(__name__)
 
 
-def get_ip() -> str:
+def get_localhost_ip(default="127.0.0.1") -> str:
+    """Return the IP address for localhost"""
+    local_ip = default
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     try:
         # doesn't even have to be reachable
         s.connect(("10.255.255.255", 1))
-        IP = s.getsockname()[0]
-    except Exception:  # pylint: disable=broad-except
-        IP = "127.0.0.1"
+        local_ip = s.getsockname()[0]
     finally:
         s.close()
-    return IP
+    return local_ip
 
 
 @retry(
@@ -170,7 +176,27 @@ def run_docker_compose_config(
     return compose_file
 
 
-COLOR_ENCODING_RE = re.compile(r"\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]")
+def shorten_path(filename: str) -> Path:
+    # These paths are composed using test name hierarchies
+    # when the test is parametrized, it uses the str of the
+    # object as id which could result in path that goes over
+    # allowed limit (260 characters).
+    # This helper function tries to normalize the path
+    # Another possibility would be that the path has some
+    # problematic characters but so far we did not find any case ...
+    global _NORMPATH_COUNT  # pylint: disable=global-statement
+
+    if len(filename) > MAX_PATH_CHAR_LEN_ALLOWED:
+        _NORMPATH_COUNT += 1
+        path = Path(filename)
+        if path.is_dir():
+            limit = MAX_PATH_CHAR_LEN_ALLOWED - 60
+            filename = filename[:limit] + f"{_NORMPATH_COUNT}"
+        elif path.is_file():
+            limit = MAX_PATH_CHAR_LEN_ALLOWED - 10
+            filename = filename[:limit] + f"{_NORMPATH_COUNT}{path.suffix}"
+
+    return Path(filename)
 
 
 def save_docker_infos(destination_path: Path):
@@ -180,21 +206,42 @@ def save_docker_infos(destination_path: Path):
     all_containers = client.containers.list(all=True)
 
     if all_containers:
-        destination_path.mkdir(parents=True, exist_ok=True)
+        try:
+            destination_path.mkdir(parents=True, exist_ok=True)
+
+        except OSError as err:
+            if err.errno == kFILENAME_TOO_LONG:
+                destination_path = shorten_path(err.filename)
+                destination_path.mkdir(parents=True, exist_ok=True)
 
         for container in all_containers:
 
             try:
                 # logs w/o coloring characters
                 logs: str = container.logs(timestamps=True, tail=1000).decode()
-                (destination_path / f"{container.name}.log").write_text(
-                    COLOR_ENCODING_RE.sub("", logs)
-                )
+
+                try:
+                    (destination_path / f"{container.name}.log").write_text(
+                        COLOR_ENCODING_RE.sub("", logs)
+                    )
+
+                except OSError as err:
+                    if err.errno == kFILENAME_TOO_LONG:
+                        shorten_path(err.filename).write_text(
+                            COLOR_ENCODING_RE.sub("", logs)
+                        )
 
                 # inspect attrs
-                (destination_path / f"{container.name}.json").write_text(
-                    json.dumps(container.attrs, indent=2)
-                )
+                try:
+                    (destination_path / f"{container.name}.json").write_text(
+                        json.dumps(container.attrs, indent=2)
+                    )
+                except OSError as err:
+                    if err.errno == kFILENAME_TOO_LONG:
+                        shorten_path(err.filename).write_text(
+                            json.dumps(container.attrs, indent=2)
+                        )
+
             except Exception as err:  # pylint: disable=broad-except
                 print(f"Unexpected failure while dumping {container}." f"Details {err}")