diff --git a/python/arcticdb/storage_fixtures/azure.py b/python/arcticdb/storage_fixtures/azure.py index 15f2d1c646..fcbcb80912 100644 --- a/python/arcticdb/storage_fixtures/azure.py +++ b/python/arcticdb/storage_fixtures/azure.py @@ -13,6 +13,7 @@ import stat import tempfile import uuid +import requests from typing import TYPE_CHECKING, Optional, Union from tempfile import mkdtemp @@ -214,8 +215,25 @@ def _safe_enter(self): process_start_cmd=args, cwd=self.working_dir, ) + if not self.ssl_test_support: + # Do not verify when ssl test support because of need of proper certs + assert self.is_azurite(self.endpoint_root), f"Azurite not started at: {self.endpoint_root}" return self + def is_azurite(self, url: str, timeout: int = 60): + try: + response = requests.get(url, timeout=timeout) + headers = response.headers + + # Check for Azurite-specific headers + server_header = headers.get("Server", "").lower() + has_azurite_headers = "x-ms-request-id" in str(headers) and "azurite" in server_header + + return has_azurite_headers + + except requests.RequestException: + return False + def __exit__(self, exc_type, exc_value, traceback): with handle_cleanup_exception(self, "process", consequence="Subsequent file deletion may also fail. "): GracefulProcessUtils.terminate(self._p) diff --git a/python/arcticdb/storage_fixtures/mongo.py b/python/arcticdb/storage_fixtures/mongo.py index 39349fcfa0..730e1031b9 100644 --- a/python/arcticdb/storage_fixtures/mongo.py +++ b/python/arcticdb/storage_fixtures/mongo.py @@ -117,7 +117,7 @@ class ManagedMongoDBServer(StorageFixtureFactory): def __init__(self, data_dir: Optional[str] = None, port=0, executable="mongod"): self._data_dir = data_dir or tempfile.mkdtemp("ManagedMongoDBServer") - self._port = port or get_ephemeral_port(5) + self._port = port or get_ephemeral_port(7) self._executable = executable self._client = None diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 1822c52234..cbba5cb754 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -13,6 +13,7 @@ import re import sys import platform +import pprint from tempfile import mkdtemp from urllib.parse import urlparse import boto3 @@ -688,6 +689,11 @@ def __call__(self, environ, start_response): else: self._reqs_till_rate_limit -= 1 + # Lets add ability to identify type as S3 + if "/whoami" in path_info: + start_response("200 OK", [("Content-Type", "text/plain")]) + return [b"Moto AWS S3"] + return super().__call__(environ, start_response) @@ -695,6 +701,8 @@ class GcpHostDispatcherApplication(HostDispatcherApplication): """GCP's S3 implementation does not have batch delete.""" def __call__(self, environ, start_response): + path_info: bytes = environ.get("PATH_INFO", "") + if environ["REQUEST_METHOD"] == "POST" and environ["QUERY_STRING"] == "delete": response_body = ( b'' @@ -708,6 +716,12 @@ def __call__(self, environ, start_response): "501 Not Implemented", [("Content-Type", "text/xml"), ("Content-Length", str(len(response_body)))] ) return [response_body] + + # Lets add ability to identify type as GCP + if "/whoami" in path_info: + start_response("200 OK", [("Content-Type", "text/plain")]) + return [b"Moto GCP"] + return super().__call__(environ, start_response) @@ -731,6 +745,20 @@ def run_gcp_server(port, key_file, cert_file): ) +def is_server_type(url: str, server_type: str): + """Check if a server is of certain type. + + /whoami url is added to Moto* objects to identify GCP or S3""" + try: + response = requests.get(url, verify=False) + if response.status_code == 200 and server_type in response.text: + return True + except Exception as e: + logger.error(f"Error during server type check: {e}") + logger.error(f"Was not of expected type: status code {response.status_code}, text: {response.text}") + return False + + def create_bucket(s3_client, bucket_name, max_retries=15): for i in range(max_retries): try: @@ -741,6 +769,9 @@ def create_bucket(s3_client, bucket_name, max_retries=15): raise logger.warning(f"S3 create bucket failed. Retry {1}/{max_retries}") time.sleep(1) + except Exception as e: + logger.error(f"create_bucket - Error: {e.response['Error']['Message']}") + pprint.pprint(e.response) class MotoS3StorageFixtureFactory(BaseS3StorageFixtureFactory): @@ -791,10 +822,19 @@ def bucket_name(self, bucket_type="s3"): # We need the unique_id because we have tests that are creating the factory directly # and not using the fixtures # so this guarantees a unique bucket name - return f"test_{bucket_type}_bucket_{self.unique_id}_{self._bucket_id}" + return f"test-{bucket_type}-bucket-{self.unique_id}-{self._bucket_id}" - def _start_server(self): - port = self.port = get_ephemeral_port(2) + def is_server_type(url: str, server_type: str): + try: + response = requests.get(url, verify=False) + if response.status_code == 200 and server_type in response.text: + return True + except Exception: + pass + return False + + def _start_server(self, seed=2): + port = self.port = get_ephemeral_port(seed) self.endpoint = f"{self.http_protocol}://{self.host}:{port}" self.working_dir = mkdtemp(suffix="MotoS3StorageFixtureFactory") self._iam_endpoint = f"{self.http_protocol}://localhost:{port}" @@ -826,17 +866,22 @@ def _start_server(self): # There is a problem with the performance of the socket module in the MacOS 15 GH runners - https://github.com/actions/runner-images/issues/12162 # Due to this, we need to wait for the server to come up for a longer time wait_for_server_to_come_up(self.endpoint, "moto", self._p, timeout=240) + assert is_server_type(self.endpoint + "/whoami", "S3"), "The server has not identified as S3" def _safe_enter(self): - for _ in range(3): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start + for i in range(5): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start try: - self._start_server() + logger.info(f"Attempt to start server - {i}") + self._start_server(2 + i) + self._s3_admin = self._boto(service="s3", key=self.default_key) + logger.info(f"Moto S3 STARTED!!! on port {self.port}") break except AssertionError as e: # Thrown by wait_for_server_to_come_up sys.stderr.write(repr(e)) GracefulProcessUtils.terminate(self._p) + except Exception as e: + logger.error(f"Error during startup of Moto S3. Trying again. Error: {e}") - self._s3_admin = self._boto(service="s3", key=self.default_key) return self def __exit__(self, exc_type, exc_value, traceback): @@ -928,8 +973,8 @@ def create_fixture(self) -> NfsS3Bucket: class MotoGcpS3StorageFixtureFactory(MotoS3StorageFixtureFactory): - def _start_server(self): - port = self.port = get_ephemeral_port(3) + def _start_server(self, seed=20): + port = self.port = get_ephemeral_port(seed) self.endpoint = f"{self.http_protocol}://{self.host}:{port}" self.working_dir = mkdtemp(suffix="MotoGcpS3StorageFixtureFactory") self._iam_endpoint = f"{self.http_protocol}://localhost:{port}" @@ -961,6 +1006,7 @@ def _start_server(self): # There is a problem with the performance of the socket module in the MacOS 15 GH runners - https://github.com/actions/runner-images/issues/12162 # Due to this, we need to wait for the server to come up for a longer time wait_for_server_to_come_up(self.endpoint, "moto", self._p, timeout=240) + assert is_server_type(self.endpoint + "/whoami", "GCP"), "The server has not identified as GCP" def create_fixture(self) -> GcpS3Bucket: bucket = self.bucket_name("gcp") diff --git a/python/tests/integration/arcticdb/test_s3.py b/python/tests/integration/arcticdb/test_s3.py index cf795752ad..490a79bd95 100644 --- a/python/tests/integration/arcticdb/test_s3.py +++ b/python/tests/integration/arcticdb/test_s3.py @@ -196,7 +196,6 @@ def test_wrapped_s3_storage(lib_name, wrapped_s3_storage_bucket): lib.write("s", data=create_df()) -@SKIP_CONDA_MARK # issue with fixture init will be fixed in https://github.com/man-group/ArcticDB/issues/2640 def test_library_get_key_path(lib_name, s3_and_nfs_storage_bucket, test_prefix): lib = s3_and_nfs_storage_bucket.create_version_store_factory(lib_name)() lib.write("s", data=create_df())