✨♻️ (⚠️ devops) Remove 5gb limit when uploading data via nodeports (ITISFoundation#2993)

* added r_clone and r_clone_utils

* added node_ports_v2_r_clone

* refactor settings

* refactor settings

* refactor tests

* nodeports supports optional r_clone_settings

* pylint

* pylint

* moved endpoint where it makes sense

* moved rclone endpoint and settings

* new way of importing

* added tests and fixed existing tests for nodeports

* extended stprage service

* fix tests

* pylint

* dynamic-sidecar now requires nodeports

* fix simcore-sdk requiremetns

* properly formatting URL

* using http in urls

* moved requirements

* injecting mandatory settings

* test refactoring

* fix env vars for rlcone

* adding r_clone

* fix unit test simore-sdk

* fix directr-v2 integration tests

* fixed first barch of directorv-2 unittest fixes

* pylint

* second btach of fixes director-v2

* fixed other tests

* moved tests

* refactor to use storage_client module

* replacing caching

* removed r_clone exra settings

* moved region to settings

* rename

* more refactoring

* revert

* file metadata update check permissions

* refactor command exception

* command is now a list not a string

* moving from list to *args

* removed test dir which no longer exists

* fix issues with encoding invalid characters

* removing defaults

* refactor to use exising upload_link fetching

* removed unused from specs

* pylint

* removed old comment

* pylint

* dynamic-sidecar work also gets transferred via rclone

* add missing file

* directory watcher disabled during docker-compse up

* removed log message

* addind middleware to log HTTP errors serialized to client

* removing logs since they get automatically captured

* returning AnyUrl

* more changes

* rename and quote shell

* added quotes and optimized checks

* fixing tests

* renaming

* removed delete file metedata
metada is updated when uploading file

* removes file and metadata in case of upload error

* added nosec

* fix pylint

* new dict format migration

* update requirements

* refacror installation

* @sanderegg always upload instead of raising error

* fix tests after merge

* using full parameter names

* repalced helper with validator

* using full parameter names

* fix issue with minio S3 endpoint format

* using full parameter names

* changing endpoint formatting

* moved s3_region

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

.env-devel

@@ -60,7 +60,7 @@ S3_BUCKET_NAME=simcore
 S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
-R_CLONE_S3_PROVIDER=MINIO
+R_CLONE_PROVIDER=MINIO
 
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key

.github/workflows/ci-testing-deploy.yml

@@ -1036,6 +1036,8 @@ jobs:
           driver: docker
       - name: setup docker-compose
         run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: install rclone in CI
+        run: sudo ./ci/github/helpers/install_rclone.bash
       - name: setup python environment
         uses: actions/setup-python@v3
         with:
@@ -1343,7 +1345,7 @@ jobs:
       - name: setup docker-compose
         run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
       - name: setup rclone docker volume plugin
-        run: sudo ./ci/github/helpers/install_rclone.bash
+        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
       - name: setup python environment
         uses: actions/setup-python@v3
         with:
@@ -1410,6 +1412,8 @@ jobs:
           driver: docker
       - name: setup docker-compose
         run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: install rclone in CI
+        run: sudo ./ci/github/helpers/install_rclone.bash
       - name: setup python environment
         uses: actions/setup-python@v3
         with:

api/specs/storage/openapi.yaml

@@ -257,11 +257,11 @@ paths:
           required: true
           schema:
             type: string
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/FileMetaData"
+        - name: user_id
+          in: query
+          required: true
+          schema:
+            type: string
       responses:
         "200":
           description: "Returns file metadata"

ci/github/helpers/install_rclone.bash

@@ -10,11 +10,8 @@ set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 
-# Installation instructions from https://rclone.org/docker/
-
-apt-get -y install fuse=2.9.9-3
-mkdir -p /var/lib/docker-plugins/rclone/config
-mkdir -p /var/lib/docker-plugins/rclone/cache
-docker plugin install rclone/docker-volume-rclone:amd64-1.57.0 args="-v" --alias rclone --grant-all-permissions
-docker plugin list
-docker plugin inspect rclone
+R_CLONE_VERSION="1.58.0"
+curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
+dpkg --install "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
+rm "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
+rclone --version

ci/github/helpers/install_rclone_docker_volume_plugin.bash

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+#  Installs the latest version of rclone plugin
+#
+
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+IFS=$'\n\t'
+
+
+# Installation instructions from https://rclone.org/docker/
+
+apt-get -y install fuse=2.9.9-3
+mkdir --parents /var/lib/docker-plugins/rclone/config
+mkdir --parents /var/lib/docker-plugins/rclone/cache
+docker plugin install rclone/docker-volume-rclone:amd64-1.57.0 args="-v" --alias rclone --grant-all-permissions
+docker plugin list
+docker plugin inspect rclone

ci/github/unit-testing/director-v2.bash

@@ -18,14 +18,13 @@ test() {
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     -v -m "not travis" services/director-v2/tests/unit --ignore=services/director-v2/tests/unit/with_dbs \
     --asyncio-mode=auto \
-    --ignore=services/director-v2/tests/unit/with_swarm
   # these tests cannot be run in parallel
   pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
     --log-date-format="%Y-%m-%d %H:%M:%S" \
     --cov=simcore_service_director_v2 --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     --asyncio-mode=auto \
-    -v -m "not travis" services/director-v2/tests/unit/with_swarm services/director-v2/tests/unit/with_dbs
+    -v -m "not travis" services/director-v2/tests/unit/with_dbs
 }
 
 # Check if the function exists (bash specific)

packages/pytest-simcore/src/pytest_simcore/minio_service.py

@@ -3,7 +3,7 @@
 # pylint: disable=unused-variable
 
 import logging
-from typing import Dict, Iterator
+from typing import Any, Iterator
 
 import pytest
 from _pytest.monkeypatch import MonkeyPatch
@@ -43,8 +43,8 @@ def _ensure_remove_bucket(client: Minio, bucket_name: str):
 
 @pytest.fixture(scope="module")
 def minio_config(
-    docker_stack: Dict, testing_environ_vars: Dict, monkeypatch_module: MonkeyPatch
-) -> Dict[str, str]:
+    docker_stack: dict, testing_environ_vars: dict, monkeypatch_module: MonkeyPatch
+) -> dict[str, Any]:
     assert "pytest-ops_minio" in docker_stack["services"]
 
     config = {
@@ -68,7 +68,7 @@ def minio_config(
 
 
 @pytest.fixture(scope="module")
-def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]:
+def minio_service(minio_config: dict[str, str]) -> Iterator[Minio]:
 
     client = Minio(**minio_config["client"])
 
@@ -99,7 +99,7 @@ def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]:
 
 
 @pytest.fixture(scope="module")
-def bucket(minio_config: Dict[str, str], minio_service: Minio) -> Iterator[str]:
+def bucket(minio_config: dict[str, str], minio_service: Minio) -> Iterator[str]:
     bucket_name = minio_config["bucket_name"]
 
     _ensure_remove_bucket(minio_service, bucket_name)

packages/service-library/src/servicelib/aiohttp/dev_error_logger.py

@@ -0,0 +1,36 @@
+
+from aiohttp.web import Application, middleware, Request, HTTPError
+from servicelib.aiohttp.typing_extension import Handler, Middleware
+import logging
+import traceback
+
+logger = logging.getLogger(__name__)
+
+_SEP = "|||"
+
+
+def _middleware_factory() -> Middleware:
+    @middleware
+    async def middleware_handler(request: Request, handler: Handler):
+        try:
+            return await handler(request)
+        except HTTPError as err:
+            fields = {
+                "Body": err.body,
+                "Status": err.status,
+                "Reason": err.reason,
+                "Headers": err.headers,
+                "Traceback": "\n".join(traceback.format_tb(err.__traceback__)),
+            }
+            formatted_error = "".join(
+                [f"\n{_SEP}{k}{_SEP}\n{v}" for k, v in fields.items()]
+            )
+            logger.debug("Error serialized to client:%s", formatted_error)
+            raise err
+
+    return middleware_handler
+
+
+def setup_dev_error_logger(app: Application) -> None:
+    logger.info("Setting up dev_error_logger")
+    app.middlewares.append(_middleware_factory())

packages/settings-library/src/settings_library/r_clone.py

@@ -0,0 +1,16 @@
+from enum import Enum
+
+from pydantic import Field
+from .base import BaseCustomSettings
+from .s3 import S3Settings
+
+
+class S3Provider(str, Enum):
+    AWS = "AWS"
+    CEPH = "CEPH"
+    MINIO = "MINIO"
+
+
+class RCloneSettings(BaseCustomSettings):
+    R_CLONE_S3: S3Settings = Field(auto_default_from_env=True)
+    R_CLONE_PROVIDER: S3Provider

packages/settings-library/src/settings_library/s3.py

@@ -1,12 +1,22 @@
 from typing import Optional
 
 from .base import BaseCustomSettings
+from pydantic import validator
 
 
 class S3Settings(BaseCustomSettings):
-    S3_ENDPOINT: str = "minio:9000"
-    S3_ACCESS_KEY: str = "12345678"
-    S3_SECRET_KEY: str = "12345678"
+    S3_ENDPOINT: str
+    S3_ACCESS_KEY: str
+    S3_SECRET_KEY: str
     S3_ACCESS_TOKEN: Optional[str] = None
-    S3_BUCKET_NAME: str = "simcore"
+    S3_BUCKET_NAME: str
     S3_SECURE: bool = False
+    S3_REGION: str = "us-east-1"
+
+    @validator("S3_ENDPOINT", pre=True)
+    @classmethod
+    def ensure_scheme(cls, v: str, values) -> str:
+        if not v.startswith("http"):
+            scheme = "https" if values.get("S3_SECURE") else "http"
+            return f"{scheme}://{v}"
+        return v