moved utils to folder

Author: sanderegg

services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/core.py

@@ -25,9 +25,9 @@
 from settings_library.s3 import S3Settings
 from yarl import URL
 
-from ..dask_utils import TaskPublisher
-from ..file_utils import pull_file_from_remote, push_file_to_remote
 from ..settings import ApplicationSettings
+from ..utils.dask import TaskPublisher
+from ..utils.files import pull_file_from_remote, push_file_to_remote
 from .docker_utils import (
     create_container_config,
     get_computational_shared_data_mount_point,

services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/docker_utils.py

@@ -38,9 +38,9 @@
 from servicelib.progress_bar import ProgressBarData
 from settings_library.s3 import S3Settings
 
-from ..dask_utils import TaskPublisher
-from ..file_utils import push_file_to_remote
 from ..settings import ApplicationSettings
+from ..utils.dask import TaskPublisher
+from ..utils.files import push_file_to_remote
 from .constants import LEGACY_SERVICE_LOG_FILE_NAME
 from .models import (
     LEGACY_INTEGRATION_VERSION,

services/dask-sidecar/src/simcore_service_dask_sidecar/dask_utils.py renamed to services/dask-sidecar/src/simcore_service_dask_sidecar/utils/dask.py

@@ -19,7 +19,7 @@
 from models_library.rabbitmq_messages import LoggerRabbitMessage
 from servicelib.logging_utils import LogLevelInt, LogMessageStr, log_catch
 
-from .rabbitmq_plugin import get_rabbitmq_client
+from ..rabbitmq_plugin import get_rabbitmq_client
 
 _logger = logging.getLogger(__name__)
 

services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py renamed to services/dask-sidecar/src/simcore_service_dask_sidecar/utils/files.py

@@ -16,9 +16,13 @@
 
 from ._meta import print_dask_sidecar_banner
 from .computational_sidecar.core import ComputationalSidecar
-from .dask_utils import TaskPublisher, get_current_task_resources, monitor_task_abortion
 from .rabbitmq_plugin import RabbitMQPlugin
 from .settings import ApplicationSettings
+from .utils.dask import (
+    TaskPublisher,
+    get_current_task_resources,
+    monitor_task_abortion,
+)
 from .utils.logs import setup_app_logging
 
 _logger = logging.getLogger(__name__)

services/dask-sidecar/src/simcore_service_dask_sidecar/worker.py

@@ -29,7 +29,9 @@
 from pytest_simcore.helpers.typing_env import EnvVarsDict
 from settings_library.rabbit import RabbitSettings
 from settings_library.s3 import S3Settings
-from simcore_service_dask_sidecar.file_utils import _s3fs_settings_from_s3_settings
+from simcore_service_dask_sidecar.utils.files import (
+    _s3fs_settings_from_s3_settings,
+)
 from yarl import URL
 
 pytest_plugins = [

services/dask-sidecar/tests/unit/conftest.py

@@ -16,7 +16,7 @@
 from dask_task_models_library.container_tasks.events import TaskProgressEvent
 from dask_task_models_library.container_tasks.io import TaskCancelEventName
 from dask_task_models_library.container_tasks.protocol import TaskOwner
-from simcore_service_dask_sidecar.dask_utils import (
+from simcore_service_dask_sidecar.utils.dask import (
     _DEFAULT_MAX_RESOURCES,
     TaskPublisher,
     get_current_task_resources,

services/dask-sidecar/tests/unit/test_dask_utils.py

@@ -57,8 +57,10 @@
     LEGACY_INTEGRATION_VERSION,
     ImageLabels,
 )
-from simcore_service_dask_sidecar.dask_utils import _DEFAULT_MAX_RESOURCES
-from simcore_service_dask_sidecar.file_utils import _s3fs_settings_from_s3_settings
+from simcore_service_dask_sidecar.utils.dask import _DEFAULT_MAX_RESOURCES
+from simcore_service_dask_sidecar.utils.files import (
+    _s3fs_settings_from_s3_settings,
+)
 from simcore_service_dask_sidecar.worker import run_computational_sidecar
 
 _logger = logging.getLogger(__name__)
@@ -77,10 +79,10 @@ def dask_subsystem_mock(
 
     # mock tasks get worker and state
     dask_distributed_worker_mock = mocker.patch(
-        "simcore_service_dask_sidecar.dask_utils.get_worker", autospec=True
+        "simcore_service_dask_sidecar.utils.dask.get_worker", autospec=True
     )
     dask_task_mock = mocker.patch(
-        "simcore_service_dask_sidecar.dask_utils.TaskState", autospec=True
+        "simcore_service_dask_sidecar.utils.dask.TaskState", autospec=True
     )
     dask_task_mock.resource_restrictions = {}
     dask_distributed_worker_mock.return_value.state.tasks.get.return_value = (
@@ -99,22 +101,22 @@ def dask_subsystem_mock(
     )
     # mock dask event publishing
     dask_utils_publish_event_mock = mocker.patch(
-        "simcore_service_dask_sidecar.dask_utils.distributed.Pub",
+        "simcore_service_dask_sidecar.utils.dask.distributed.Pub",
         autospec=True,
     )
     mocker.patch(
-        "simcore_service_dask_sidecar.dask_utils.distributed.Sub",
+        "simcore_service_dask_sidecar.utils.dask.distributed.Sub",
         autospec=True,
     )
     mocker.patch(
-        "simcore_service_dask_sidecar.dask_utils.is_current_task_aborted",
+        "simcore_service_dask_sidecar.utils.dask.is_current_task_aborted",
         autospec=True,
         return_value=False,
     )
     # mock dask rabbitmq plugin
     mock_rabbitmq_client = create_rabbitmq_client("pytest_dask_sidecar_logs_publisher")
     mocker.patch(
-        "simcore_service_dask_sidecar.dask_utils.get_rabbitmq_client",
+        "simcore_service_dask_sidecar.utils.dask.get_rabbitmq_client",
         autospec=True,
         return_value=mock_rabbitmq_client,
     )

services/dask-sidecar/tests/unit/test_tasks.py

@@ -19,7 +19,7 @@
 from pytest_localftpserver.servers import ProcessFTPServer
 from pytest_mock.plugin import MockerFixture
 from settings_library.s3 import S3Settings
-from simcore_service_dask_sidecar.file_utils import (
+from simcore_service_dask_sidecar.utils.files import (
     _s3fs_settings_from_s3_settings,
     pull_file_from_remote,
     push_file_to_remote,
@@ -310,14 +310,17 @@ async def test_pull_compressed_zip_file_from_remote(
     if remote_parameters.s3_settings:
         storage_kwargs = _s3fs_settings_from_s3_settings(remote_parameters.s3_settings)
 
-    with cast(
-        fsspec.core.OpenFile,
-        fsspec.open(
-            f"{destination_url}",
-            mode="wb",
-            **storage_kwargs,
-        ),
-    ) as dest_fp, local_zip_file_path.open("rb") as src_fp:
+    with (
+        cast(
+            fsspec.core.OpenFile,
+            fsspec.open(
+                f"{destination_url}",
+                mode="wb",
+                **storage_kwargs,
+            ),
+        ) as dest_fp,
+        local_zip_file_path.open("rb") as src_fp,
+    ):
         dest_fp.write(src_fp.read())
 
     # now we want to download that file so it becomes the source