refactor

Author: sanderegg

services/storage/src/simcore_service_storage/cli.py

@@ -9,11 +9,11 @@
 
 LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 main = typer.Typer(name="simcore-service-storage service")
 
-main.command()(create_settings_command(settings_cls=Settings, logger=log))
+main.command()(create_settings_command(settings_cls=Settings, logger=_logger))
 
 
 @main.command()

services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py

@@ -21,7 +21,7 @@
     DatcoreAdapterTimeoutError,
 )
 
-log = logging.getLogger(__file__)
+_logger = logging.getLogger(__file__)
 
 
 class _DatcoreAdapterResponseError(DatcoreAdapterError):
@@ -102,7 +102,7 @@ async def _retrieve_all_pages(
             ),
         )
     ) and response.get("items"):
-        log.debug(
+        _logger.debug(
             "called %s [%d/%d], received %d objects",
             path,
             page,
@@ -155,7 +155,7 @@ async def list_all_datasets_files_metadatas(
             )
             for d in all_datasets
         ),
-        log=log,
+        _logger=_logger,
         max_concurrency=MAX_CONCURRENT_REST_CALLS,
     )
     all_files_of_all_datasets: list[FileMetaData] = []

services/storage/src/simcore_service_storage/db_tokens.py

@@ -9,7 +9,7 @@
 
 from .constants import APP_AIOPG_ENGINE_KEY, APP_CONFIG_KEY
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 async def _get_tokens_from_db(engine: Engine, user_id: UserID) -> dict[str, Any]:

services/storage/src/simcore_service_storage/handlers_datasets.py

@@ -21,7 +21,7 @@
     StorageQueryParamsBase,
 )
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 routes = RouteTableDef()
 
@@ -36,7 +36,7 @@ async def list_datasets_metadata(request: web.Request) -> web.Response:
         StorageQueryParamsBase, request
     )
     path_params = parse_request_path_parameters_as(LocationPathParams, request)
-    log.debug(
+    _logger.debug(
         "received call to list_datasets_metadata with %s",
         f"{path_params=}, {query_params=}",
     )
@@ -58,7 +58,7 @@ async def list_dataset_files_metadata(request: web.Request) -> web.Response:
     path_params = parse_request_path_parameters_as(
         FilesMetadataDatasetPathParams, request
     )
-    log.debug(
+    _logger.debug(
         "received call to list_dataset_files_metadata with %s",
         f"{path_params=}, {query_params=}",
     )

services/storage/src/simcore_service_storage/handlers_health.py

@@ -1,6 +1,6 @@
 """
 
-    - Checks connectivity with other services in the backend
+- Checks connectivity with other services in the backend
 
 """
 
@@ -21,7 +21,7 @@
 from .s3 import get_s3_client
 from .settings import Settings
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 routes = web.RouteTableDef()
 

services/storage/src/simcore_service_storage/handlers_locations.py

@@ -24,7 +24,7 @@
 from .settings import Settings
 from .simcore_s3_dsm import SimcoreS3DataManager
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 routes = RouteTableDef()
 
@@ -35,7 +35,7 @@ async def list_storage_locations(request: web.Request) -> web.Response:
     query_params: StorageQueryParamsBase = parse_request_query_parameters_as(
         StorageQueryParamsBase, request
     )
-    log.debug(
+    _logger.debug(
         "received call to list_storage_locations with %s",
         f"{query_params=}",
     )
@@ -58,7 +58,7 @@ async def synchronise_meta_data_table(request: web.Request) -> web.Response:
         SyncMetadataQueryParams, request
     )
     path_params = parse_request_path_parameters_as(LocationPathParams, request)
-    log.debug(
+    _logger.debug(
         "received call to synchronise_meta_data_table with %s",
         f"{path_params=}, {query_params=}",
     )
@@ -77,12 +77,12 @@ async def _go():
             timeout = settings.STORAGE_SYNC_METADATA_TIMEOUT
             try:
                 result = await asyncio.wait_for(sync_coro, timeout=timeout)
-                log.info(
+                _logger.info(
                     "Sync metadata table completed: %d entries removed",
                     len(result),
                 )
             except asyncio.TimeoutError:
-                log.exception("Sync metadata table timed out (%s seconds)", timeout)
+                _logger.exception("Sync metadata table timed out (%s seconds)", timeout)
 
         fire_and_forget_task(
             _go(),

services/storage/src/simcore_service_storage/s3.py

@@ -1,6 +1,4 @@
-""" Module to access s3 service
-
-"""
+"""Module to access s3 service"""
 
 import logging
 from collections.abc import AsyncGenerator
@@ -17,28 +15,28 @@
 from .constants import APP_CONFIG_KEY, APP_S3_KEY, RETRY_WAIT_SECS
 from .settings import Settings
 
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 async def setup_s3_client(app) -> AsyncGenerator[None, None]:
     client = None
 
-    with log_context(log, logging.DEBUG, msg="setup.s3_client.cleanup_ctx"):
+    with log_context(_logger, logging.DEBUG, msg="setup.s3_client.cleanup_ctx"):
         storage_settings: Settings = app[APP_CONFIG_KEY]
         storage_s3_settings = storage_settings.STORAGE_S3
         assert storage_s3_settings  # nosec
 
         async for attempt in AsyncRetrying(
             wait=wait_fixed(RETRY_WAIT_SECS),
-            before_sleep=before_sleep_log(log, logging.WARNING),
+            before_sleep=before_sleep_log(_logger, logging.WARNING),
             reraise=True,
         ):
             with attempt:
                 client = await SimcoreS3API.create(
                     storage_s3_settings,
                     storage_settings.STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY,
                 )
-                log.info(
+                _logger.info(
                     "S3 client %s successfully created [%s]",
                     f"{client=}",
                     json_dumps(attempt.retry_state.retry_object.statistics),
@@ -48,13 +46,13 @@ async def setup_s3_client(app) -> AsyncGenerator[None, None]:
 
     yield
 
-    with log_context(log, logging.DEBUG, msg="teardown.s3_client.cleanup_ctx"):
+    with log_context(_logger, logging.DEBUG, msg="teardown.s3_client.cleanup_ctx"):
         if client:
             await client.close()
 
 
 async def setup_s3_bucket(app: web.Application):
-    with log_context(log, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"):
+    with log_context(_logger, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"):
         storage_s3_settings = app[APP_CONFIG_KEY].STORAGE_S3
         client = get_s3_client(app)
         await client.create_bucket(

services/storage/src/simcore_service_storage/settings.py

@@ -88,14 +88,14 @@ class Settings(BaseCustomSettings, MixinLoggingSettings):
             "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED",
             "LOG_FORMAT_LOCAL_DEV_ENABLED",
         ),
-        description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
+        description="Enables local development _logger format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
     STORAGE_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field(
         default_factory=dict,
         validation_alias=AliasChoices(
             "STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"
         ),
-        description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.",
+        description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.",
     )
 
     @field_validator("LOG_LEVEL", mode="before")

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -789,7 +789,6 @@ async def create_soft_link(
     async def synchronise_meta_data_table(
         self, *, dry_run: bool
     ) -> list[StorageFileID]:
-
         async with self.engine.acquire() as conn:
             _logger.warning(
                 "Total number of entries to check %d",
@@ -855,7 +854,7 @@ async def _clean_expired_uploads(self) -> None:
                 for fmd in list_of_expired_uploads
             ),
             reraise=False,
-            log=_logger,
+            _logger=_logger,
             limit=_NO_CONCURRENCY,
         )
 
@@ -886,7 +885,7 @@ async def _revert_file(fmd: FileMetaDataAtDB) -> FileMetaDataAtDB:
         reverted_fmds = await limited_gather(
             *(_revert_file(fmd) for fmd in list_of_fmds_to_delete),
             reraise=False,
-            log=_logger,
+            _logger=_logger,
             limit=_NO_CONCURRENCY,
         )
         list_of_fmds_to_delete = [