removed locations stuff

Author: sanderegg

services/storage/src/simcore_service_storage/api/rest/_locations.py

@@ -1,27 +1,13 @@
-import asyncio
 import logging
-from typing import Annotated, cast
+from typing import Annotated
 
-from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi import APIRouter, Depends, Request, status
 from models_library.api_schemas_storage import FileLocation
 from models_library.generics import Envelope
-from models_library.projects_nodes_io import LocationID, StorageFileID
-from servicelib.aiohttp import status
-from servicelib.aiohttp.application_keys import (
-    APP_CONFIG_KEY,
-    APP_FIRE_AND_FORGET_TASKS_KEY,
-)
-from servicelib.utils import fire_and_forget_task
 
 # Exclusive for simcore-s3 storage -----------------------
-from ...core.settings import ApplicationSettings
 from ...dsm import get_dsm_provider
-from ...models import (
-    StorageQueryParamsBase,
-    SyncMetadataQueryParams,
-    SyncMetadataResponse,
-)
-from ...simcore_s3_dsm import SimcoreS3DataManager
+from ...models import StorageQueryParamsBase
 
 _logger = logging.getLogger(__name__)
 
@@ -47,52 +33,3 @@ async def list_storage_locations(
         if await dsm.authorized(query_params.user_id):
             locs.append(FileLocation(name=dsm.location_name, id=dsm.location_id))
     return Envelope[list[FileLocation]](data=locs)
-
-
-@router.post(
-    "/locations/{location_id}:sync", response_model=Envelope[SyncMetadataResponse]
-)
-async def synchronise_meta_data_table(
-    query_params: Annotated[SyncMetadataQueryParams, Depends()],
-    location_id: LocationID,
-    request: Request,
-):
-    if not location_id == SimcoreS3DataManager.get_location_id():
-        raise HTTPException(
-            status.HTTP_400_BAD_REQUEST,
-            detail="invalid call: cannot be called for other than simcore",
-        )
-    dsm = cast(
-        SimcoreS3DataManager,
-        get_dsm_provider(request.app).get(SimcoreS3DataManager.get_location_id()),
-    )
-    sync_results: list[StorageFileID] = []
-    sync_coro = dsm.synchronise_meta_data_table(dry_run=query_params.dry_run)
-
-    if query_params.fire_and_forget:
-        settings: ApplicationSettings = request.app[APP_CONFIG_KEY]
-
-        async def _go():
-            timeout = settings.STORAGE_SYNC_METADATA_TIMEOUT
-            try:
-                result = await asyncio.wait_for(sync_coro, timeout=timeout)
-                _logger.info(
-                    "Sync metadata table completed: %d entries removed",
-                    len(result),
-                )
-            except TimeoutError:
-                _logger.exception("Sync metadata table timed out (%s seconds)", timeout)
-
-        fire_and_forget_task(
-            _go(),
-            task_suffix_name="synchronise_meta_data_table",
-            fire_and_forget_tasks_collection=request.app[APP_FIRE_AND_FORGET_TASKS_KEY],
-        )
-    else:
-        sync_results = await sync_coro
-    response = SyncMetadataResponse(
-        removed=sync_results,
-        fire_and_forget=query_params.fire_and_forget,
-        dry_run=query_params.dry_run,
-    )
-    return Envelope[SyncMetadataResponse](data=response)

services/storage/src/simcore_service_storage/modules/db/db_tokens.py

@@ -38,7 +38,4 @@ async def get_api_token_and_secret(app: FastAPI, user_id: UserID) -> tuple[str,
     api_token = data.get("token_key", api_token)
     api_secret = data.get("token_secret", api_secret)
 
-    assert api_token is not None  # nosec
-    assert api_secret is not None  # nosec
-
     return api_token, api_secret

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -784,34 +784,6 @@ async def create_soft_link(
         async with self.engine.connect() as conn:
             return convert_db_to_model(await db_file_meta_data.insert(conn, target))
 
-    async def synchronise_meta_data_table(
-        self, *, dry_run: bool
-    ) -> list[StorageFileID]:
-        async with self.engine.connect() as conn:
-            _logger.warning(
-                "Total number of entries to check %d",
-                await db_file_meta_data.total(conn),
-            )
-            # iterate over all entries to check if there is a file in the S3 backend
-            file_ids_to_remove = [
-                fmd.file_id
-                async for fmd in db_file_meta_data.list_valid_uploads(conn)
-                if not await get_s3_client(self.app).object_exists(
-                    bucket=self.simcore_bucket_name, object_key=fmd.object_name
-                )
-            ]
-
-            if not dry_run:
-                await db_file_meta_data.delete(conn, file_ids_to_remove)
-
-            _logger.info(
-                "%s %d entries ",
-                "Would delete" if dry_run else "Deleted",
-                len(file_ids_to_remove),
-            )
-
-        return cast(list[StorageFileID], file_ids_to_remove)
-
     async def _clean_pending_upload(
         self, conn: AsyncConnection, file_id: SimcoreS3FileID
     ) -> None:

services/storage/tests/helpers/utils.py

@@ -10,13 +10,11 @@
 
 
 def has_datcore_tokens() -> bool:
-    # TODO: activate tests against BF services in the CI.
-    #
-    # CI shall add BF_API_KEY, BF_API_SECRET environs as secrets
-    #
-    if not os.environ.get("BF_API_KEY") or not os.environ.get("BF_API_SECRET"):
+    api_key = os.environ.get("BF_API_KEY")
+    api_secret = os.environ.get("BF_API_SECRET")
+    if not api_key or not api_secret:
         return False
-    return True
+    return not (api_key == "none" or api_secret == "none")  # noqa: S105
 
 
 async def get_updated_project(

services/storage/tests/unit/test_handlers_locations.py

@@ -4,70 +4,30 @@
 # pylint:disable=too-many-arguments
 # pylint:disable=no-name-in-module
 
-from typing import Any
 
 import httpx
-import pytest
-from fastapi import FastAPI
+from fastapi import FastAPI, status
 from models_library.users import UserID
-from pytest_simcore.helpers.assert_checks import assert_status
-from servicelib.aiohttp import status
+from pytest_simcore.helpers.httpx_assert_checks import url_from_operation_id
 from tests.helpers.utils import has_datcore_tokens
 
 pytest_simcore_core_services_selection = ["postgres"]
 pytest_simcore_ops_services_selection = ["adminer"]
 
 
-async def test_locations(client: httpx.AsyncClient, user_id: UserID):
-    resp = await client.get(f"/v0/locations?user_id={user_id}")
+async def test_locations(
+    initialized_app: FastAPI, client: httpx.AsyncClient, user_id: UserID
+):
+    url = url_from_operation_id(
+        client, initialized_app, "list_storage_locations"
+    ).with_query(user_id=user_id)
+    resp = await client.get(f"{url}")
 
-    payload = await resp.json()
-    assert resp.status == 200, str(payload)
+    payload = resp.json()
+    assert resp.status_code == status.HTTP_200_OK, str(payload)
 
     data, error = tuple(payload.get(k) for k in ("data", "error"))
 
     _locs = 2 if has_datcore_tokens() else 1
     assert len(data) == _locs
     assert not error
-
-
-@pytest.mark.parametrize(
-    "dry_run, fire_and_forget, expected_removed",
-    [
-        (None, None, []),
-        (True, False, []),
-        (True, True, []),
-        (False, True, []),
-        (False, False, []),
-    ],
-)
-async def test_synchronise_meta_data_table(
-    initialized_app: FastAPI,
-    client: httpx.AsyncClient,
-    location_id: int,
-    user_id: UserID,
-    dry_run: bool | None,
-    fire_and_forget: bool | None,
-    expected_removed: list,
-):
-    query_params: dict[str, Any] = {"user_id": user_id}
-    if dry_run:
-        query_params["dry_run"] = f"{dry_run}"
-    if fire_and_forget:
-        query_params["fire_and_forget"] = f"{fire_and_forget}"
-    url = (
-        client.app.router["synchronise_meta_data_table"]
-        .url_for(location_id=f"{location_id}")
-        .with_query(**query_params)
-    )
-    resp = await client.post(
-        f"{url}",
-    )
-    data, error = await assert_status(resp, status.HTTP_200_OK)
-    assert not error
-    assert data
-    assert data["dry_run"] == (False if dry_run is None else dry_run)
-    assert data["fire_and_forget"] == (
-        False if fire_and_forget is None else fire_and_forget
-    )
-    assert data["removed"] == expected_removed