♻️Maintenance/remove minio-sdk (#5120)

Author: sanderegg

.github/dependabot.yml

@@ -29,9 +29,6 @@ updates:
       - dependency-name: httpx
         versions:
           - 0.17.0
-      - dependency-name: minio
-        versions:
-          - 7.0.0
   - package-ecosystem: pip
     directory: "/packages/service-library"
     schedule:

Makefile

@@ -269,23 +269,15 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 
 
 .stack-ops.yml: .env $(docker-compose-configs)
-	# Compiling config file for filestash
-	$(eval TMP_PATH_TO_FILESTASH_CONFIG=$(shell set -o allexport && \
-	source $(CURDIR)/.env && \
-	set +o allexport && \
-	python3 scripts/filestash/create_config.py))
 	# Creating config for ops stack to $@
-	# -> filestash config at $(TMP_PATH_TO_FILESTASH_CONFIG)
 ifdef ops_ci
 	@$(shell \
-		export TMP_PATH_TO_FILESTASH_CONFIG="${TMP_PATH_TO_FILESTASH_CONFIG}" && \
 		scripts/docker/docker-compose-config.bash -e .env \
 		services/docker-compose-ops-ci.yml \
 		> $@ \
 	)
 else
 	@$(shell \
-		export TMP_PATH_TO_FILESTASH_CONFIG="${TMP_PATH_TO_FILESTASH_CONFIG}" && \
 		scripts/docker/docker-compose-config.bash -e .env \
 		services/docker-compose-ops.yml \
 		> $@ \
@@ -299,7 +291,6 @@ endif
 .deploy-ops: .stack-ops.yml
 	# Deploy stack 'ops'
 ifndef ops_disabled
-	# -> filestash config at $(TMP_PATH_TO_FILESTASH_CONFIG)
 	docker stack deploy --with-registry-auth -c $< ops
 else
 	@echo "Explicitly disabled with ops_disabled flag in CLI"
@@ -328,7 +319,6 @@ printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVIT
 printf "$$rows" "Payments" "http://$(get_my_ip).nip.io:8011/dev/doc" $${PAYMENTS_USERNAME} $${PAYMENTS_PASSWORD};\
 printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\
 printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\
-printf "$$rows" "Storage S3 Filestash" "http://$(get_my_ip).nip.io:9002" 12345678 12345678;\
 printf "$$rows" "Storage S3 Minio" "http://$(get_my_ip).nip.io:9001" 12345678 12345678;\
 
 printf "\n%s\n" "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar";\

packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py

@@ -0,0 +1,102 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-import
+
+
+import contextlib
+import typing
+
+import aioboto3
+import pytest
+from aiobotocore.session import ClientCreatorContext
+from botocore.client import Config
+from settings_library.s3 import S3Settings
+from types_aiobotocore_s3 import S3Client
+
+
+@pytest.fixture
+def s3_settings() -> S3Settings:
+    return S3Settings.create_from_envs()
+
+
+@pytest.fixture
+async def s3_client(s3_settings: S3Settings) -> typing.AsyncIterator[S3Client]:
+    session = aioboto3.Session()
+    exit_stack = contextlib.AsyncExitStack()
+    session_client = session.client(
+        "s3",
+        endpoint_url=s3_settings.S3_ENDPOINT,
+        aws_access_key_id=s3_settings.S3_ACCESS_KEY,
+        aws_secret_access_key=s3_settings.S3_SECRET_KEY,
+        aws_session_token=s3_settings.S3_ACCESS_TOKEN,
+        region_name=s3_settings.S3_REGION,
+        config=Config(signature_version="s3v4"),
+    )
+    assert isinstance(session_client, ClientCreatorContext)
+    client = typing.cast(S3Client, await exit_stack.enter_async_context(session_client))
+
+    yield client
+
+    await exit_stack.aclose()
+
+
+async def _empty_bucket(s3_client: S3Client, bucket_name: str) -> None:
+    # List object versions
+    response = await s3_client.list_object_versions(Bucket=bucket_name)
+
+    # Delete all object versions
+    for version in response.get("Versions", []):
+        assert "Key" in version
+        assert "VersionId" in version
+        await s3_client.delete_object(
+            Bucket=bucket_name, Key=version["Key"], VersionId=version["VersionId"]
+        )
+
+    # Delete all delete markers
+    for marker in response.get("DeleteMarkers", []):
+        assert "Key" in marker
+        assert "VersionId" in marker
+        await s3_client.delete_object(
+            Bucket=bucket_name, Key=marker["Key"], VersionId=marker["VersionId"]
+        )
+
+    # Delete remaining objects in the bucket
+    response = await s3_client.list_objects(Bucket=bucket_name)
+    for obj in response.get("Contents", []):
+        assert "Key" in obj
+        await s3_client.delete_object(Bucket=bucket_name, Key=obj["Key"])
+
+
+@pytest.fixture
+async def s3_bucket(
+    s3_settings: S3Settings, s3_client: S3Client
+) -> typing.AsyncIterator[str]:
+    bucket_name = s3_settings.S3_BUCKET_NAME
+
+    response = await s3_client.list_buckets()
+    bucket_exists = bucket_name in [
+        bucket_struct.get("Name") for bucket_struct in response["Buckets"]
+    ]
+    if bucket_exists:
+        await _empty_bucket(s3_client, bucket_name)
+
+    if not bucket_exists:
+        await s3_client.create_bucket(Bucket=bucket_name)
+    response = await s3_client.list_buckets()
+    assert response["Buckets"]
+    assert bucket_name in [
+        bucket_struct.get("Name") for bucket_struct in response["Buckets"]
+    ], f"failed creating {bucket_name}"
+
+    yield bucket_name
+
+    await _empty_bucket(s3_client, bucket_name)
+
+
+@pytest.fixture
+async def with_bucket_versioning_enabled(s3_client: S3Client, s3_bucket: str) -> str:
+    await s3_client.put_bucket_versioning(
+        Bucket=s3_bucket,
+        VersioningConfiguration={"MFADelete": "Disabled", "Status": "Enabled"},
+    )
+    return s3_bucket

packages/pytest-simcore/src/pytest_simcore/aws_server.py

@@ -7,8 +7,10 @@
 import pytest
 import requests
 from aiohttp.test_utils import unused_port
+from faker import Faker
 from moto.server import ThreadedMotoServer
 from settings_library.ec2 import EC2Settings
+from settings_library.s3 import S3Settings
 
 from .helpers.utils_envs import EnvVarsDict, setenvs_from_dict
 from .helpers.utils_host import get_localhost_ip
@@ -71,16 +73,22 @@ def mocked_ec2_server_envs(
 
 
 @pytest.fixture
-async def mocked_s3_server_envs(
-    mocked_aws_server: ThreadedMotoServer,
-    reset_aws_server_state: None,
+def mocked_s3_server_settings(
+    mocked_aws_server: ThreadedMotoServer, reset_aws_server_state: None, faker: Faker
+) -> S3Settings:
+    return S3Settings(
+        S3_ACCESS_KEY="xxx",
+        S3_ENDPOINT=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}",  # pylint: disable=protected-access # noqa: SLF001
+        S3_SECRET_KEY="xxx",  # noqa: S106
+        S3_BUCKET_NAME=f"pytest{faker.pystr().lower()}",
+        S3_SECURE=False,
+    )
+
+
+@pytest.fixture
+def mocked_s3_server_envs(
+    mocked_s3_server_settings: S3Settings,
     monkeypatch: pytest.MonkeyPatch,
 ) -> EnvVarsDict:
-    changed_envs = {
-        "S3_SECURE": "false",
-        "S3_ENDPOINT": f"{mocked_aws_server._ip_address}:{mocked_aws_server._port}",  # pylint: disable=protected-access  # noqa: SLF001
-        "S3_ACCESS_KEY": "xxx",
-        "S3_SECRET_KEY": "xxx",
-        "S3_BUCKET_NAME": "pytestbucket",
-    }
+    changed_envs: EnvVarsDict = mocked_s3_server_settings.dict(exclude_unset=True)
     return setenvs_from_dict(monkeypatch, changed_envs)

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -13,7 +13,6 @@
 import os
 import re
 import shutil
-import subprocess
 import sys
 from collections.abc import Iterator
 from copy import deepcopy
@@ -82,6 +81,9 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict:
     if "DOCKER_IMAGE_TAG" not in os.environ:
         env_devel["DOCKER_IMAGE_TAG"] = "production"
 
+    # ensure we do not use the bucket of simcore or so
+    env_devel["S3_BUCKET_NAME"] = "pytestbucket"
+
     return {key: value for key, value in env_devel.items() if value is not None}
 
 
@@ -166,39 +168,12 @@ def simcore_docker_compose(
     return compose_specs
 
 
-@pytest.fixture(scope="module")
-def inject_filestash_config_path_env(
-    osparc_simcore_scripts_dir: Path,
-    env_file_for_testing: Path,
-) -> EnvVarsDict:
-    create_filestash_config_py = (
-        osparc_simcore_scripts_dir / "filestash" / "create_config.py"
-    )
-
-    # ensures .env at git_root_dir, which will be used as current directory
-    assert env_file_for_testing.exists()
-    env_values = dotenv_values(env_file_for_testing)
-
-    process = subprocess.run(
-        ["python3", f"{create_filestash_config_py}"],
-        shell=False,
-        check=True,
-        stdout=subprocess.PIPE,
-        env=env_values,
-    )
-    filestash_config_json_path = Path(process.stdout.decode("utf-8").strip())
-    assert filestash_config_json_path.exists()
-
-    return {"TMP_PATH_TO_FILESTASH_CONFIG": f"{filestash_config_json_path}"}
-
-
 @pytest.fixture(scope="module")
 def ops_docker_compose(
     osparc_simcore_root_dir: Path,
     osparc_simcore_scripts_dir: Path,
     env_file_for_testing: Path,
     temp_folder: Path,
-    inject_filestash_config_path_env: dict[str, str],
 ) -> dict[str, Any]:
     """Filters only services in docker-compose-ops.yml and returns yaml data
 
@@ -219,7 +194,6 @@ def ops_docker_compose(
         docker_compose_paths=docker_compose_path,
         env_file_path=env_file_for_testing,
         destination_path=temp_folder / "ops_docker_compose.yml",
-        additional_envs=inject_filestash_config_path_env,
     )
     # NOTE: do not add indent. Copy&Paste log into editor instead
     print(
@@ -277,7 +251,7 @@ def ops_docker_compose_file(
     docker_compose_path = Path(temp_folder / "ops_docker_compose.filtered.yml")
 
     # these services are useless when running in the CI
-    ops_view_only_services = ["adminer", "redis-commander", "portainer", "filestash"]
+    ops_view_only_services = ["adminer", "redis-commander", "portainer"]
     if "CI" in os.environ:
         print(
             f"WARNING: Services such as {ops_view_only_services!r} are removed from the stack when running in the CI"

packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py

@@ -29,12 +29,17 @@ def setenvs_from_dict(
 ) -> EnvVarsDict:
     for key, value in envs.items():
         assert isinstance(key, str)
-        assert value is not None  # None keys cannot be is defined w/o value
+        assert (
+            value is not None
+        ), f"{key=},{value=}"  # None keys cannot be is defined w/o value
+        converted_value = value
+        if isinstance(value, bool):
+            converted_value = f"{'true' if value else 'false'}"
         assert isinstance(
-            value, str
-        ), "client MUST explicitly stringify values since some cannot be done automatically e.g. json-like values"
+            converted_value, str
+        ), f"client MUST explicitly stringify values since some cannot be done automatically e.g. json-like values. problematic {key=},{value=}"
 
-        monkeypatch.setenv(key, value)
+        monkeypatch.setenv(key, converted_value)
     return deepcopy(envs)
 
 
@@ -72,7 +77,7 @@ def delenvs_from_envfile(
     monkeypatch: pytest.MonkeyPatch,
     content_or_path: str | Path,
     raising: bool,
-    **dotenv_kwags
+    **dotenv_kwags,
 ) -> EnvVarsDict:
     """Batch monkeypatch.delenv(...) on all env vars in an envfile"""
     envs = load_dotenv(content_or_path, **dotenv_kwags)