✨CEPH/S3 additions (⚠️ devops) (ITISFoundation#2818)

* added filestash as s3 visualizer/editor

* trying to use proper bucket

* correctly reading env vars

* fixed protocol

* pinning versions

* setting vfs to minimal

* @pcrespov review changes

* fixed makefile

* random secret for filestash

* reverting changes

* remove usless file

* warning if no volumes found

* adding settings for vfs cache mode

* trying to fix test flakyness

* warns when volumes still need to be removed

* fixtures now process docker-compose ops

* fixed  catalog test

* reverting makefile changes

* name refactor & reverting changes

* trying to make multiuple deplyment safe

* makeing command safer

* debug test case

* revert change back

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

Makefile

@@ -195,16 +195,27 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	@docker-compose --env-file .env --file services/docker-compose.yml --file services/docker-compose.local.yml --log-level=ERROR config > $@
 
 .stack-ops.yml: .env $(docker-compose-configs)
+	# Compiling config file for filestash
+	$(eval TMP_PATH_TO_FILESTASH_CONFIG=$(shell set -o allexport; \
+	source $(CURDIR)/.env; \
+	set +o allexport; \
+	python3 scripts/filestash/create_config.py))
 	# Creating config for ops stack to $@
-	@docker-compose --env-file .env --file services/docker-compose-ops.yml --log-level=ERROR config > $@
+	# -> filestash config at $(TMP_PATH_TO_FILESTASH_CONFIG)
+	@$(shell \
+		export TMP_PATH_TO_FILESTASH_CONFIG="${TMP_PATH_TO_FILESTASH_CONFIG}" && \
+		docker-compose --env-file .env --file services/docker-compose-ops.yml --log-level=DEBUG config > $@ \
+	)
+
 
 
 .PHONY: up-devel up-prod up-version up-latest .deploy-ops
 
 .deploy-ops: .stack-ops.yml
 	# Deploy stack 'ops'
 ifndef ops_disabled
-	@docker stack deploy --with-registry-auth -c $< ops
+	# -> filestash config at $(TMP_PATH_TO_FILESTASH_CONFIG)
+	docker stack deploy --with-registry-auth -c $< ops
 else
 	@echo "Explicitly disabled with ops_disabled flag in CLI"
 endif

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -11,6 +11,7 @@
 import json
 import os
 import shutil
+import subprocess
 import sys
 from copy import deepcopy
 from pathlib import Path
@@ -19,7 +20,8 @@
 import pytest
 import yaml
 from _pytest.config import ExitCode
-from dotenv import dotenv_values
+from _pytest.monkeypatch import MonkeyPatch
+from dotenv import dotenv_values, set_key
 
 from .helpers import (
     FIXTURE_CONFIG_CORE_SERVICES_SELECTION,
@@ -154,9 +156,46 @@ def simcore_docker_compose(
     return config
 
 
+@pytest.fixture(scope="module")
+def inject_filestash_config_path(
+    osparc_simcore_scripts_dir: Path,
+    monkeypatch_module: MonkeyPatch,
+    env_file_for_testing: Path,
+) -> None:
+    create_filestash_config_py = (
+        osparc_simcore_scripts_dir / "filestash" / "create_config.py"
+    )
+
+    # ensures .env at git_root_dir, which will be used as current directory
+    assert env_file_for_testing.exists()
+    env_values = dotenv_values(env_file_for_testing)
+
+    process = subprocess.run(
+        ["python3", f"{create_filestash_config_py}"],
+        shell=False,
+        check=True,
+        stdout=subprocess.PIPE,
+        env=env_values,
+    )
+    filestash_config_json_path = Path(process.stdout.decode("utf-8").strip())
+    assert filestash_config_json_path.exists()
+
+    set_key(
+        env_file_for_testing,
+        "TMP_PATH_TO_FILESTASH_CONFIG",
+        f"{filestash_config_json_path}",
+    )
+    monkeypatch_module.setenv(
+        "TMP_PATH_TO_FILESTASH_CONFIG", f"{filestash_config_json_path}"
+    )
+
+
 @pytest.fixture(scope="module")
 def ops_docker_compose(
-    osparc_simcore_root_dir: Path, env_file_for_testing: Path, temp_folder: Path
+    osparc_simcore_root_dir: Path,
+    env_file_for_testing: Path,
+    temp_folder: Path,
+    inject_filestash_config_path: None,
 ) -> Dict[str, Any]:
     """Filters only services in docker-compose-ops.yml and returns yaml data
 

scripts/filestash/create_config.py

@@ -0,0 +1,54 @@
+"""
+Parses the configuration template and injects it where the platform expects it
+Notes:
+- Admin credentials for filestash are admin:adminadmin
+- $ must be escaped with $$ in the template file
+"""
+
+import os
+import random
+import string
+import tempfile
+
+from distutils.util import strtobool
+from pathlib import Path
+from string import Template
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+TEMPLATE_PATH = SCRIPT_DIR / "filestash_config.json.template"
+CONFIG_JSON = Path(tempfile.mkdtemp()) / "filestash_config.json"
+
+
+def random_secret_key(length: int = 16) -> str:
+    return "".join(random.choice(string.ascii_letters) for _ in range(length))
+
+
+def patch_env_vars() -> None:
+    endpoint = os.environ["S3_ENDPOINT"]
+    if not endpoint.startswith("http"):
+        protocol = "https" if strtobool(os.environ["S3_SECURE"].lower()) else "http"
+        endpoint = f"{protocol}://{endpoint}"
+
+    os.environ["S3_ENDPOINT"] = endpoint
+
+    os.environ["REPLACE_SECRET_KEY"] = random_secret_key()
+
+
+def main() -> None:
+    patch_env_vars()
+
+    assert TEMPLATE_PATH.exists()
+
+    template_content = TEMPLATE_PATH.read_text()
+
+    config_json = Template(template_content).substitute(os.environ)
+
+    assert CONFIG_JSON.parent.exists()
+    CONFIG_JSON.write_text(config_json)
+
+    # path of configuration file is exported as env var
+    print(f"{CONFIG_JSON}")
+
+
+if __name__ == "__main__":
+    main()

scripts/filestash/filestash_config.json.template

@@ -0,0 +1,90 @@
+{
+  "general": {
+      "name": null,
+      "port": null,
+      "host": null,
+      "secret_key": "$REPLACE_SECRET_KEY",
+      "force_ssl": null,
+      "editor": null,
+      "fork_button": null,
+      "logout": null,
+      "display_hidden": null,
+      "refresh_after_upload": null,
+      "auto_connect": null,
+      "upload_button": null,
+      "upload_pool_size": null,
+      "filepage_default_view": "list",
+      "filepage_default_sort": "date",
+      "cookie_timeout": null,
+      "custom_css": null
+  },
+  "features": {
+      "share": {
+          "enable": null,
+          "default_access": null,
+          "redirect": null
+      },
+      "protection": {
+          "zip_timeout": null,
+          "enable": null,
+          "disable_svg": null
+      },
+      "office": {
+          "enable": null,
+          "onlyoffice_server": null
+      },
+      "server": {
+          "console_enable": null,
+          "tor_enable": null,
+          "tor_url": null
+      },
+      "syncthing": {
+          "enable": null,
+          "server_url": null
+      },
+      "image": {
+          "enable_image": null,
+          "thumbnail_size": null,
+          "thumbnail_quality": null,
+          "thumbnail_caching": null,
+          "image_quality": null,
+          "image_caching": null
+      },
+      "search": {
+          "explore_timeout": null
+      },
+      "video": {
+          "blacklist_format": null,
+          "enable_transcoder": null
+      }
+  },
+  "log": {
+      "enable": null,
+      "level": null,
+      "telemetry": null
+  },
+  "email": {
+      "server": null,
+      "port": null,
+      "username": null,
+      "password": null,
+      "from": null
+  },
+  "auth": {
+      "admin": "$$2a$$10$$viR16hXd35bAaEJFEgpd9OqIzNBb/VoIsgQ8P3SjKxolpEQEHltrW"
+  },
+  "constant": {
+      "user": "filestash",
+      "emacs": true,
+      "pdftotext": true
+  },
+  "connections": [
+      {
+          "label": "S3",
+          "type": "s3",
+          "advanced": true,
+          "path": "$S3_BUCKET_NAME",
+          "endpoint": "$S3_ENDPOINT"
+      }
+  ]
+}

services/catalog/tests/unit/conftest.py

@@ -14,6 +14,7 @@
     "pytest_simcore.docker_compose",
     "pytest_simcore.docker_registry",
     "pytest_simcore.docker_swarm",
+    "pytest_simcore.monkeypatch_extra",
     "pytest_simcore.postgres_service",
     "pytest_simcore.pydantic_models",
     "pytest_simcore.repository_paths",

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -49,6 +49,13 @@ class S3Provider(str, Enum):
     MINIO = "MINIO"
 
 
+class VFSCacheMode(str, Enum):
+    OFF = "off"
+    MINIMAL = "minimal"
+    WRITES = "writes"
+    FILL = "full"
+
+
 class RCloneSettings(S3Settings):
     R_CLONE_S3_PROVIDER: S3Provider
 
@@ -60,6 +67,10 @@ class RCloneSettings(S3Settings):
         9,
         description="time to wait between polling for changes",
     )
+    R_CLONE_VFS_CACHE_MODE: VFSCacheMode = Field(
+        VFSCacheMode.MINIMAL,
+        description="used primarly for easy testing without requiring requiring code changes",
+    )
 
     @validator("R_CLONE_POLL_INTERVAL_SECONDS")
     @classmethod

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/_namepsace.py

@@ -0,0 +1,10 @@
+from models_library.projects_nodes import NodeID
+
+from ...models.schemas.constants import DYNAMIC_SIDECAR_SERVICE_PREFIX
+
+
+def get_compose_namespace(node_uuid: NodeID) -> str:
+    # To avoid collisions for started docker resources a unique identifier is computed:
+    # - avoids container level collisions on same node
+    # - avoids volume level collisions on same node
+    return f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{node_uuid}"

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api.py

@@ -370,18 +370,24 @@ async def remove_dynamic_sidecar_network(network_name: str) -> bool:
         return False
 
 
-async def remove_dynamic_sidecar_volumes(node_uuid: NodeID) -> bool:
+async def remove_dynamic_sidecar_volumes(node_uuid: NodeID) -> Set[str]:
     async with docker_client() as client:
         volumes_response = await client.volumes.list(
             filters={"label": f"uuid={node_uuid}"}
         )
         volumes = volumes_response["Volumes"]
+        log.debug("Removing volumes: %s", [v["Name"] for v in volumes])
+        if len(volumes) == 0:
+            log.warning("Expected to find at least 1 volume to remove, 0 were found")
+
+        removed_volumes: Set[str] = set()
+
         for volume_data in volumes:
             volume = await client.volumes.get(volume_data["Name"])
             await volume.delete()
+            removed_volumes.add(volume_data["Name"])
 
-        log.debug("Remove volumes: %s", [v["Name"] for v in volumes])
-        return True
+        return removed_volumes
 
 
 async def list_dynamic_sidecar_services(

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -5,8 +5,8 @@
 from servicelib.json_serialization import json_dumps
 
 from ....core.settings import AppSettings, DynamicSidecarSettings
-from ....models.schemas.constants import DYNAMIC_SIDECAR_SERVICE_PREFIX
 from ....models.schemas.dynamic_services import SchedulerData, ServiceType
+from .._namepsace import get_compose_namespace
 from ..volumes_resolver import DynamicSidecarVolumesPathsResolver
 from .settings import inject_settings_to_create_service_params
 
@@ -75,10 +75,7 @@ def get_dynamic_sidecar_spec(
     of the dynamic service. The director-v2 directly coordinates with
     the dynamic-sidecar for this purpose.
     """
-    # To avoid collisions for started docker resources a unique identifier is computed:
-    # - avoids container level collisions on same node
-    # - avoids volume level collisions on same node
-    compose_namespace = f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_{scheduler_data.node_uuid}"
+    compose_namespace = get_compose_namespace(scheduler_data.node_uuid)
 
     mounts = [
         # docker socket needed to use the docker api