From 72d8e19fd9328cdeb3b88b48e1b5caa78939fe3b Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 8 Apr 2024 04:33:29 -0400 Subject: [PATCH 1/7] editable config from main config module --- core/testcontainers/core/config.py | 22 +++++++++++++++++++ core/testcontainers/core/container.py | 18 +++++---------- core/testcontainers/core/labels.py | 4 ++-- core/testcontainers/core/waiting_utils.py | 10 ++++----- .../testcontainers/arangodb/__init__.py | 4 ++-- modules/k3s/testcontainers/k3s/__init__.py | 4 ++-- .../neo4j/testcontainers/neo4j/__init__.py | 4 ++-- .../testcontainers/postgres/__init__.py | 8 +++---- .../qdrant/testcontainers/qdrant/__init__.py | 4 ++-- 9 files changed, 47 insertions(+), 31 deletions(-) diff --git a/core/testcontainers/core/config.py b/core/testcontainers/core/config.py index 0c1b5e0c2..bd6c8215a 100644 --- a/core/testcontainers/core/config.py +++ b/core/testcontainers/core/config.py @@ -1,3 +1,5 @@ +from dataclasses import dataclass +from functools import cached_property from os import environ MAX_TRIES = int(environ.get("TC_MAX_TRIES", 120)) @@ -9,3 +11,23 @@ RYUK_DISABLED: bool = environ.get("TESTCONTAINERS_RYUK_DISABLED", "false") == "true" RYUK_DOCKER_SOCKET: str = environ.get("TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE", "/var/run/docker.sock") RYUK_RECONNECTION_TIMEOUT: str = environ.get("RYUK_RECONNECTION_TIMEOUT", "10s") + + +@dataclass +class TestcontainersConfiguration: + max_tries: int = MAX_TRIES + sleep_time: int = SLEEP_TIME + ryuk_image: str = RYUK_IMAGE + ryuk_privileged: bool = RYUK_PRIVILEGED + ryuk_disabled: bool = RYUK_DISABLED + ryuk_docker_socket: str = RYUK_DOCKER_SOCKET + ryuk_reconnection_timeout: str = RYUK_RECONNECTION_TIMEOUT + + @cached_property + def timeout(self): + return self.max_tries * self.sleep_time + + +testcontainers_config = TestcontainersConfiguration() + +__all__ = ["testcontainers_config"] diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 559a4ffe7..efa06734d 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -6,13 +6,7 @@ import docker.errors from typing_extensions import Self -from testcontainers.core.config import ( - RYUK_DISABLED, - RYUK_DOCKER_SOCKET, - RYUK_IMAGE, - RYUK_PRIVILEGED, - RYUK_RECONNECTION_TIMEOUT, -) +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.docker_client import DockerClient from testcontainers.core.exceptions import ContainerStartException from testcontainers.core.labels import LABEL_SESSION_ID, SESSION_ID @@ -77,7 +71,7 @@ def maybe_emulate_amd64(self) -> Self: return self def start(self) -> Self: - if not RYUK_DISABLED and self.image != RYUK_IMAGE: + if not c.ryuk_disabled and self.image != c.ryuk_image: logger.debug("Creating Ryuk container") Reaper.get_instance() logger.info("Pulling image %s", self.image) @@ -201,12 +195,12 @@ def _create_instance(cls) -> "Reaper": logger.debug(f"Creating new Reaper for session: {SESSION_ID}") Reaper._container = ( - DockerContainer(RYUK_IMAGE) + DockerContainer(c.ryuk_image) .with_name(f"testcontainers-ryuk-{SESSION_ID}") .with_exposed_ports(8080) - .with_volume_mapping(RYUK_DOCKER_SOCKET, "/var/run/docker.sock", "rw") - .with_kwargs(privileged=RYUK_PRIVILEGED, auto_remove=True) - .with_env("RYUK_RECONNECTION_TIMEOUT", RYUK_RECONNECTION_TIMEOUT) + .with_volume_mapping(c.ryuk_docker_socket, "/var/run/docker.sock", "rw") + .with_kwargs(privileged=c.ryuk_privileged, auto_remove=True) + .with_env("RYUK_RECONNECTION_TIMEOUT", c.ryuk_reconnection_timeout) .start() ) wait_for_logs(Reaper._container, r".* Started!") diff --git a/core/testcontainers/core/labels.py b/core/testcontainers/core/labels.py index 13937a5e8..144e4365e 100644 --- a/core/testcontainers/core/labels.py +++ b/core/testcontainers/core/labels.py @@ -1,7 +1,7 @@ from typing import Optional from uuid import uuid4 -from testcontainers.core.config import RYUK_IMAGE +from testcontainers.core.config import testcontainers_config as c SESSION_ID: str = str(uuid4()) LABEL_SESSION_ID = "org.testcontainers.session-id" @@ -13,7 +13,7 @@ def create_labels(image: str, labels: Optional[dict[str, str]]) -> dict[str, str labels = {} labels[LABEL_LANG] = "python" - if image == RYUK_IMAGE: + if image == c.ryuk_image: return labels labels[LABEL_SESSION_ID] = SESSION_ID diff --git a/core/testcontainers/core/waiting_utils.py b/core/testcontainers/core/waiting_utils.py index ea52683d5..4db9bc162 100644 --- a/core/testcontainers/core/waiting_utils.py +++ b/core/testcontainers/core/waiting_utils.py @@ -19,7 +19,7 @@ import wrapt -from testcontainers.core import config +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.utils import setup_logger if TYPE_CHECKING: @@ -54,18 +54,18 @@ def wrapper(wrapped: Callable, instance: Any, args: list, kwargs: dict) -> Any: logger.info("Waiting for %s to be ready ...", instance) exception = None - for attempt_no in range(config.MAX_TRIES): + for attempt_no in range(c.max_tries): try: return wrapped(*args, **kwargs) except transient_exceptions as e: logger.debug( - f"Connection attempt '{attempt_no + 1}' of '{config.MAX_TRIES + 1}' " + f"Connection attempt '{attempt_no + 1}' of '{c.max_tries + 1}' " f"failed: {traceback.format_exc()}" ) - time.sleep(config.SLEEP_TIME) + time.sleep(c.sleep_time) exception = e raise TimeoutError( - f"Wait time ({config.TIMEOUT}s) exceeded for {wrapped.__name__}(args: {args}, kwargs: " + f"Wait time ({c.timeout}s) exceeded for {wrapped.__name__}(args: {args}, kwargs: " f"{kwargs}). Exception: {exception}" ) diff --git a/modules/arangodb/testcontainers/arangodb/__init__.py b/modules/arangodb/testcontainers/arangodb/__init__.py index a7c954652..9ea36f6ea 100644 --- a/modules/arangodb/testcontainers/arangodb/__init__.py +++ b/modules/arangodb/testcontainers/arangodb/__init__.py @@ -5,7 +5,7 @@ import typing from os import environ -from testcontainers.core.config import TIMEOUT +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.generic import DbContainer from testcontainers.core.utils import raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_for_logs @@ -90,4 +90,4 @@ def get_connection_url(self) -> str: return f"http://{self.get_container_host_ip()}:{port}" def _connect(self) -> None: - wait_for_logs(self, predicate="is ready for business", timeout=TIMEOUT) + wait_for_logs(self, predicate="is ready for business", timeout=c.timeout) diff --git a/modules/k3s/testcontainers/k3s/__init__.py b/modules/k3s/testcontainers/k3s/__init__.py index 045e2eb5d..2682df356 100644 --- a/modules/k3s/testcontainers/k3s/__init__.py +++ b/modules/k3s/testcontainers/k3s/__init__.py @@ -11,7 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. -from testcontainers.core.config import MAX_TRIES +from testcontainers.core.config import testcontainers_config from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -46,7 +46,7 @@ def __init__(self, image="rancher/k3s:latest", **kwargs) -> None: self.with_volume_mapping("/sys/fs/cgroup", "/sys/fs/cgroup", "rw") def _connect(self) -> None: - wait_for_logs(self, predicate="Node controller sync successful", timeout=MAX_TRIES) + wait_for_logs(self, predicate="Node controller sync successful", timeout=testcontainers_config.timeout) def start(self) -> "K3SContainer": super().start() diff --git a/modules/neo4j/testcontainers/neo4j/__init__.py b/modules/neo4j/testcontainers/neo4j/__init__.py index 26f46dc61..7939c013f 100644 --- a/modules/neo4j/testcontainers/neo4j/__init__.py +++ b/modules/neo4j/testcontainers/neo4j/__init__.py @@ -15,7 +15,7 @@ from typing import Optional from neo4j import Driver, GraphDatabase -from testcontainers.core.config import TIMEOUT +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.generic import DbContainer from testcontainers.core.utils import raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs @@ -62,7 +62,7 @@ def get_connection_url(self) -> str: @wait_container_is_ready() def _connect(self) -> None: - wait_for_logs(self, "Remote interface available at", TIMEOUT) + wait_for_logs(self, "Remote interface available at", c.timeout) # Then we actually check that the container really is listening with self.get_driver() as driver: diff --git a/modules/postgres/testcontainers/postgres/__init__.py b/modules/postgres/testcontainers/postgres/__init__.py index 3810ea0f2..9b347aa61 100644 --- a/modules/postgres/testcontainers/postgres/__init__.py +++ b/modules/postgres/testcontainers/postgres/__init__.py @@ -14,7 +14,7 @@ from time import sleep from typing import Optional -from testcontainers.core.config import MAX_TRIES, SLEEP_TIME +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.generic import DbContainer from testcontainers.core.utils import raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs @@ -91,15 +91,15 @@ def get_connection_url(self, host: Optional[str] = None, driver: Optional[str] = @wait_container_is_ready() def _connect(self) -> None: - wait_for_logs(self, ".*database system is ready to accept connections.*", MAX_TRIES, SLEEP_TIME) + wait_for_logs(self, ".*database system is ready to accept connections.*", c.max_tries, c.sleep_time) count = 0 - while count < MAX_TRIES: + while count < c.max_tries: status, _ = self.exec(f"pg_isready -hlocalhost -p{self.port} -U{self.username}") if status == 0: return - sleep(SLEEP_TIME) + sleep(c.sleep_time) count += 1 raise RuntimeError("Postgres could not get into a ready state") diff --git a/modules/qdrant/testcontainers/qdrant/__init__.py b/modules/qdrant/testcontainers/qdrant/__init__.py index ac9279955..d36fe62ee 100644 --- a/modules/qdrant/testcontainers/qdrant/__init__.py +++ b/modules/qdrant/testcontainers/qdrant/__init__.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Optional -from testcontainers.core.config import TIMEOUT +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.generic import DbContainer from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs @@ -61,7 +61,7 @@ def _configure(self) -> None: @wait_container_is_ready() def _connect(self) -> None: - wait_for_logs(self, ".*Actix runtime found; starting in Actix runtime.*", TIMEOUT) + wait_for_logs(self, ".*Actix runtime found; starting in Actix runtime.*", c.timeout) def get_client(self, **kwargs): """ From 7bde15bc338e54f4d1d10800b2800d14836b2c10 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 8 Apr 2024 04:46:17 -0400 Subject: [PATCH 2/7] actually it should be property not cached property --- core/testcontainers/core/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/testcontainers/core/config.py b/core/testcontainers/core/config.py index bd6c8215a..319dc2974 100644 --- a/core/testcontainers/core/config.py +++ b/core/testcontainers/core/config.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from functools import cached_property from os import environ MAX_TRIES = int(environ.get("TC_MAX_TRIES", 120)) @@ -23,7 +22,7 @@ class TestcontainersConfiguration: ryuk_docker_socket: str = RYUK_DOCKER_SOCKET ryuk_reconnection_timeout: str = RYUK_RECONNECTION_TIMEOUT - @cached_property + @property def timeout(self): return self.max_tries * self.sleep_time From a79eeffbb0f7e52fe0c8c9f32675a22318bdc434 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 8 Apr 2024 04:48:07 -0400 Subject: [PATCH 3/7] take opportunity to consolidate tc_host code --- core/testcontainers/core/config.py | 27 +++++++++++++++++++++- core/testcontainers/core/docker_client.py | 28 ++--------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/core/testcontainers/core/config.py b/core/testcontainers/core/config.py index 319dc2974..58f66fc24 100644 --- a/core/testcontainers/core/config.py +++ b/core/testcontainers/core/config.py @@ -1,5 +1,7 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from os import environ +from os.path import exists +from pathlib import Path MAX_TRIES = int(environ.get("TC_MAX_TRIES", 120)) SLEEP_TIME = int(environ.get("TC_POOLING_INTERVAL", 1)) @@ -11,6 +13,28 @@ RYUK_DOCKER_SOCKET: str = environ.get("TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE", "/var/run/docker.sock") RYUK_RECONNECTION_TIMEOUT: str = environ.get("RYUK_RECONNECTION_TIMEOUT", "10s") +TC_FILE = ".testcontainers.properties" +TC_GLOBAL = Path.home() / TC_FILE + + +def read_tc_properties() -> dict[str, str]: + """ + Read the .testcontainers.properties for settings. (see the Java implementation for details) + Currently we only support the ~/.testcontainers.properties but may extend to per-project variables later. + + :return: the merged properties from the sources. + """ + tc_files = [item for item in [TC_GLOBAL] if exists(item)] + if not tc_files: + return {} + settings = {} + + for file in tc_files: + with open(file) as contents: + tuples = [line.split("=") for line in contents.readlines() if "=" in line] + settings = {**settings, **{item[0].strip(): item[1].strip() for item in tuples}} + return settings + @dataclass class TestcontainersConfiguration: @@ -21,6 +45,7 @@ class TestcontainersConfiguration: ryuk_disabled: bool = RYUK_DISABLED ryuk_docker_socket: str = RYUK_DOCKER_SOCKET ryuk_reconnection_timeout: str = RYUK_RECONNECTION_TIMEOUT + tc_properties: dict[str, str] = field(default_factory=read_tc_properties) @property def timeout(self): diff --git a/core/testcontainers/core/docker_client.py b/core/testcontainers/core/docker_client.py index 89db0fbfc..5cf008c2b 100644 --- a/core/testcontainers/core/docker_client.py +++ b/core/testcontainers/core/docker_client.py @@ -16,20 +16,17 @@ import os import urllib import urllib.parse -from os.path import exists -from pathlib import Path from typing import Callable, Optional, TypeVar, Union import docker from docker.models.containers import Container, ContainerCollection from typing_extensions import ParamSpec +from testcontainers.core.config import testcontainers_config as c from testcontainers.core.labels import SESSION_ID, create_labels from testcontainers.core.utils import default_gateway_ip, inside_container, setup_logger LOGGER = setup_logger(__name__) -TC_FILE = ".testcontainers.properties" -TC_GLOBAL = Path.home() / TC_FILE _P = ParamSpec("_P") _T = TypeVar("_T") @@ -185,26 +182,5 @@ def host(self) -> str: return "localhost" -@ft.cache -def read_tc_properties() -> dict[str, str]: - """ - Read the .testcontainers.properties for settings. (see the Java implementation for details) - Currently we only support the ~/.testcontainers.properties but may extend to per-project variables later. - - :return: the merged properties from the sources. - """ - tc_files = [item for item in [TC_GLOBAL] if exists(item)] - if not tc_files: - return {} - settings = {} - - for file in tc_files: - tuples = [] - with open(file) as contents: - tuples = [line.split("=") for line in contents.readlines() if "=" in line] - settings = {**settings, **{item[0].strip(): item[1].strip() for item in tuples}} - return settings - - def get_docker_host() -> Optional[str]: - return read_tc_properties().get("tc.host") or os.getenv("DOCKER_HOST") + return c.tc_properties.get("tc.host") or os.getenv("DOCKER_HOST") From 64c253468df9da9d6e1ace9af8245a853645f682 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 8 Apr 2024 04:52:15 -0400 Subject: [PATCH 4/7] encapsulate get_tc_host for #494 --- core/testcontainers/core/config.py | 3 +++ core/testcontainers/core/docker_client.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/core/config.py b/core/testcontainers/core/config.py index 58f66fc24..b522a4966 100644 --- a/core/testcontainers/core/config.py +++ b/core/testcontainers/core/config.py @@ -47,6 +47,9 @@ class TestcontainersConfiguration: ryuk_reconnection_timeout: str = RYUK_RECONNECTION_TIMEOUT tc_properties: dict[str, str] = field(default_factory=read_tc_properties) + def tc_properties_get_tc_host(self): + return self.tc_properties.get("tc.host") + @property def timeout(self): return self.max_tries * self.sleep_time diff --git a/core/testcontainers/core/docker_client.py b/core/testcontainers/core/docker_client.py index 5cf008c2b..9ff6170e6 100644 --- a/core/testcontainers/core/docker_client.py +++ b/core/testcontainers/core/docker_client.py @@ -183,4 +183,4 @@ def host(self) -> str: def get_docker_host() -> Optional[str]: - return c.tc_properties.get("tc.host") or os.getenv("DOCKER_HOST") + return c.tc_properties_get_tc_host() or os.getenv("DOCKER_HOST") From 3438a4abc3d0b2692a1f9f39475b21f46ee1ed76 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 8 Apr 2024 04:55:41 -0400 Subject: [PATCH 5/7] backwards compatibility with note to remove --- core/testcontainers/core/config.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/core/config.py b/core/testcontainers/core/config.py index b522a4966..391c88bfa 100644 --- a/core/testcontainers/core/config.py +++ b/core/testcontainers/core/config.py @@ -57,4 +57,16 @@ def timeout(self): testcontainers_config = TestcontainersConfiguration() -__all__ = ["testcontainers_config"] +__all__ = [ + # the public API of this module + "testcontainers_config", + # and all the legacy things that are deprecated: + "MAX_TRIES", + "SLEEP_TIME", + "TIMEOUT", + "RYUK_IMAGE", + "RYUK_PRIVILEGED", + "RYUK_DISABLED", + "RYUK_DOCKER_SOCKET", + "RYUK_RECONNECTION_TIMEOUT", +] From 820acdf2da88187a1da6881e50891228d4044e39 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 8 Apr 2024 04:57:44 -0400 Subject: [PATCH 6/7] redo line wrapping for cleaner diff --- core/testcontainers/core/waiting_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/testcontainers/core/waiting_utils.py b/core/testcontainers/core/waiting_utils.py index 4db9bc162..4eb7ad890 100644 --- a/core/testcontainers/core/waiting_utils.py +++ b/core/testcontainers/core/waiting_utils.py @@ -19,7 +19,7 @@ import wrapt -from testcontainers.core.config import testcontainers_config as c +from testcontainers.core.config import testcontainers_config as config from testcontainers.core.utils import setup_logger if TYPE_CHECKING: @@ -54,18 +54,18 @@ def wrapper(wrapped: Callable, instance: Any, args: list, kwargs: dict) -> Any: logger.info("Waiting for %s to be ready ...", instance) exception = None - for attempt_no in range(c.max_tries): + for attempt_no in range(config.max_tries): try: return wrapped(*args, **kwargs) except transient_exceptions as e: logger.debug( - f"Connection attempt '{attempt_no + 1}' of '{c.max_tries + 1}' " + f"Connection attempt '{attempt_no + 1}' of '{config.max_tries + 1}' " f"failed: {traceback.format_exc()}" ) - time.sleep(c.sleep_time) + time.sleep(config.sleep_time) exception = e raise TimeoutError( - f"Wait time ({c.timeout}s) exceeded for {wrapped.__name__}(args: {args}, kwargs: " + f"Wait time ({config.timeout}s) exceeded for {wrapped.__name__}(args: {args}, kwargs: " f"{kwargs}). Exception: {exception}" ) From 47bcfa7c4a31a7ce98e9b2927396100c5743ffc9 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 8 Apr 2024 05:07:35 -0400 Subject: [PATCH 7/7] fix ryuk test --- core/tests/test_ryuk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/tests/test_ryuk.py b/core/tests/test_ryuk.py index e21b045ba..e081d2c07 100644 --- a/core/tests/test_ryuk.py +++ b/core/tests/test_ryuk.py @@ -5,7 +5,7 @@ from docker import DockerClient from docker.errors import NotFound -from testcontainers.core import container as container_module +from testcontainers.core.config import testcontainers_config from testcontainers.core.container import Reaper from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -13,7 +13,7 @@ def test_wait_for_reaper(monkeypatch: MonkeyPatch): Reaper.delete_instance() - monkeypatch.setattr(container_module, "RYUK_RECONNECTION_TIMEOUT", "0.1s") + monkeypatch.setattr(testcontainers_config, "ryuk_reconnection_timeout", "0.1s") docker_client = DockerClient() container = DockerContainer("hello-world").start() @@ -40,7 +40,7 @@ def test_wait_for_reaper(monkeypatch: MonkeyPatch): def test_container_without_ryuk(monkeypatch: MonkeyPatch): Reaper.delete_instance() - monkeypatch.setattr(container_module, "RYUK_DISABLED", True) + monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True) with DockerContainer("hello-world") as container: wait_for_logs(container, "Hello from Docker!") assert Reaper._instance is None