✨ Improve logging when un/archiving files and 2x faster compression times (ITISFoundation#3175)

Author: sanderegg

.env-devel

@@ -55,12 +55,15 @@ REGISTRY_URL=registry.osparc-master.speag.com
 REGISTRY_USER=admin
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
+R_CLONE_ENABLED=false
+R_CLONE_PROVIDER=MINIO
 S3_ACCESS_KEY=12345678
 S3_BUCKET_NAME=simcore
 S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
-R_CLONE_PROVIDER=MINIO
+
+
 
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key

ci/github/unit-testing/service-library.bash

@@ -27,6 +27,7 @@ test_all() {
     --durations=10 \
     --log-date-format="%Y-%m-%d %H:%M:%S" \
     --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --numprocesses=auto \
     --verbose \
     -m "not heavy_load" \
     packages/service-library/tests

packages/models-library/src/models_library/service_settings_labels.py

@@ -146,7 +146,7 @@ class PathMappingsLabel(BaseModel):
         description="optional list of paths which contents need to be persisted",
     )
 
-    state_exclude: Optional[list[str]] = Field(
+    state_exclude: Optional[set[str]] = Field(
         None,
         description="optional list unix shell rules used to exclude files from the state",
     )

packages/models-library/tests/test_service_settings_labels.py

@@ -2,11 +2,10 @@
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
 
-import json
 from collections import namedtuple
 from copy import deepcopy
 from pprint import pformat
-from typing import Any, Type
+from typing import Any
 
 import pytest
 from models_library.service_settings_labels import (
@@ -84,7 +83,7 @@ def test_service_settings() -> None:
     ),
 )
 def test_service_settings_model_examples(
-    model_cls: Type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
+    model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
 ) -> None:
     for name, example in model_cls_examples.items():
         print(name, ":", pformat(example))
@@ -97,7 +96,7 @@ def test_service_settings_model_examples(
     (SimcoreServiceLabels,),
 )
 def test_correctly_detect_dynamic_sidecar_boot(
-    model_cls: Type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
+    model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
 ) -> None:
     for name, example in model_cls_examples.items():
         print(name, ":", pformat(example))
@@ -128,7 +127,7 @@ def test_path_mappings_json_encoding() -> None:
     example = PathMappingsLabel.Config.schema_extra["example"]
     path_mappings = PathMappingsLabel.parse_obj(example)
     print(path_mappings)
-    assert json.loads(path_mappings.json()) == example
+    assert PathMappingsLabel.parse_raw(path_mappings.json()) == path_mappings
 
 
 def test_simcore_services_labels_compose_spec_null_container_http_entry_provided() -> None:

packages/service-library/requirements/_base.in

@@ -10,3 +10,4 @@ pydantic
 pyinstrument
 pyyaml
 tenacity
+tqdm

packages/service-library/requirements/_base.txt

@@ -21,6 +21,8 @@ pyyaml==5.4.1
     #   -r requirements/_base.in
 tenacity==8.0.1
     # via -r requirements/_base.in
+tqdm==4.64.0
+    # via -r requirements/_base.in
 typing-extensions==4.2.0
     # via
     #   aiodebug

packages/service-library/requirements/_test.in

@@ -25,3 +25,4 @@ pytest-instafail
 pytest-mock
 pytest-runner
 pytest-sugar
+pytest-xdist

packages/service-library/requirements/_test.txt

@@ -70,6 +70,8 @@ docopt==0.6.2
     # via
     #   coveralls
     #   docker-compose
+execnet==1.9.0
+    # via pytest-xdist
 faker==13.7.0
     # via -r requirements/_test.in
 flaky==3.7.0
@@ -128,7 +130,9 @@ platformdirs==2.5.2
 pluggy==1.0.0
     # via pytest
 py==1.11.0
-    # via pytest
+    # via
+    #   pytest
+    #   pytest-forked
 pycparser==2.21
     # via cffi
 pylint==2.13.8
@@ -148,9 +152,11 @@ pytest==7.1.2
     #   pytest-asyncio
     #   pytest-cov
     #   pytest-docker
+    #   pytest-forked
     #   pytest-instafail
     #   pytest-mock
     #   pytest-sugar
+    #   pytest-xdist
 pytest-aiohttp==1.0.4
     # via -r requirements/_test.in
 pytest-asyncio==0.18.3
@@ -159,6 +165,8 @@ pytest-cov==3.0.0
     # via -r requirements/_test.in
 pytest-docker==0.12.0
     # via -r requirements/_test.in
+pytest-forked==1.4.0
+    # via pytest-xdist
 pytest-instafail==0.4.2
     # via -r requirements/_test.in
 pytest-mock==3.7.0
@@ -167,6 +175,8 @@ pytest-runner==6.0.0
     # via -r requirements/_test.in
 pytest-sugar==0.9.4
     # via -r requirements/_test.in
+pytest-xdist==2.5.0
+    # via -r requirements/_test.in
 python-dateutil==2.8.2
     # via faker
 python-dotenv==0.20.0