Skip to content

Commit 5fe0a1c

Browse files
bisgaard-itisbisgaard-itis
authored
🐛 Only allow to search *owned* files in storage search endpoint (#5772)
1 parent 30a295d commit 5fe0a1c

File tree

16 files changed

+66
-96
lines changed

16 files changed

+66
-96
lines changed

‎api/specs/storage/openapi.yaml‎

Lines changed: 25 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -652,14 +652,14 @@ paths:
652652
responses:
653653
'204':
654654
description: Successful Response
655-
/v0/simcore-s3/files/metadata:search:
655+
/v0/simcore-s3/files/metadata:search_owned:
656656
post:
657657
tags:
658658
- simcore-s3
659659
summary: search for files starting with
660660
description: search for files starting with `startswith` and/or matching a sha256_checksum
661661
in the file_meta_data table
662-
operationId: search_files
662+
operationId: search_owned_files
663663
parameters:
664664
- required: true
665665
schema:
@@ -683,16 +683,6 @@ paths:
683683
title: Sha256 Checksum
684684
name: sha256_checksum
685685
in: query
686-
- required: false
687-
schema:
688-
type: string
689-
enum:
690-
- read
691-
- write
692-
title: Access Right
693-
default: read
694-
name: access_right
695-
in: query
696686
responses:
697687
'200':
698688
description: Successful Response
@@ -1292,36 +1282,40 @@ components:
12921282
title: PresignedLink
12931283
S3Settings:
12941284
properties:
1295-
S3_SECURE:
1296-
type: boolean
1297-
title: S3 Secure
1298-
default: false
1299-
S3_ENDPOINT:
1300-
type: string
1301-
title: S3 Endpoint
13021285
S3_ACCESS_KEY:
13031286
type: string
1287+
maxLength: 50
1288+
minLength: 1
13041289
title: S3 Access Key
1305-
S3_SECRET_KEY:
1306-
type: string
1307-
title: S3 Secret Key
1308-
S3_ACCESS_TOKEN:
1309-
type: string
1310-
title: S3 Access Token
13111290
S3_BUCKET_NAME:
13121291
type: string
1292+
maxLength: 50
1293+
minLength: 1
13131294
title: S3 Bucket Name
1295+
S3_ENDPOINT:
1296+
type: string
1297+
maxLength: 65536
1298+
minLength: 1
1299+
format: uri
1300+
title: S3 Endpoint
1301+
description: do not define if using standard AWS
13141302
S3_REGION:
13151303
type: string
1304+
maxLength: 50
1305+
minLength: 1
13161306
title: S3 Region
1317-
default: us-east-1
1307+
S3_SECRET_KEY:
1308+
type: string
1309+
maxLength: 50
1310+
minLength: 1
1311+
title: S3 Secret Key
13181312
additionalProperties: false
13191313
type: object
13201314
required:
1321-
- S3_ENDPOINT
13221315
- S3_ACCESS_KEY
1323-
- S3_SECRET_KEY
13241316
- S3_BUCKET_NAME
1317+
- S3_REGION
1318+
- S3_SECRET_KEY
13251319
title: S3Settings
13261320
description: "- Customized configuration for all settings\n- If a field is a\
13271321
\ BaseCustomSettings subclass, it allows creating a default from env vars\
@@ -1381,6 +1375,9 @@ components:
13811375
title: TaskGet
13821376
TaskProgress:
13831377
properties:
1378+
task_id:
1379+
type: string
1380+
title: Task Id
13841381
message:
13851382
type: string
13861383
title: Message

‎api/specs/storage/scripts/storage.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -343,14 +343,14 @@ async def delete_folders_of_project(
343343
f"/{API_VTAG}/simcore-s3/files/metadata:search",
344344
response_model=Envelope[FileMetaDataGet],
345345
tags=TAGS_SIMCORE_S3,
346-
summary="search for files starting with",
346+
summary="search for owned files",
347347
operation_id="search_files",
348348
)
349349
async def search_files(
350350
user_id: UserID,
351+
kind: Literal["owned"],
351352
startswith: str = "",
352353
sha256_checksum: SHA256Str | None = None,
353-
access_right: Literal["read", "write"] = "read",
354354
):
355355
"""search for files starting with `startswith` and/or matching a sha256_checksum in the file_meta_data table"""
356356

‎services/api-server/src/simcore_service_api_server/api/routes/files.py‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,7 @@
4040
UploadLinks,
4141
)
4242
from ...services.service_exception_handling import DEFAULT_BACKEND_SERVICE_STATUS_CODES
43-
from ...services.storage import (
44-
AccessRight,
45-
StorageApi,
46-
StorageFileMetaData,
47-
to_file_api_model,
48-
)
43+
from ...services.storage import StorageApi, StorageFileMetaData, to_file_api_model
4944
from ..dependencies.authentication import get_current_user_id
5045
from ..dependencies.services import get_api_client
5146
from ._common import API_SERVER_DEV_FEATURES_ENABLED
@@ -73,16 +68,16 @@ async def _get_file(
7368
file_id: UUID,
7469
storage_client: StorageApi,
7570
user_id: int,
76-
access_right: AccessRight,
7771
):
7872
"""Gets metadata for a given file resource"""
7973

8074
try:
81-
stored_files: list[StorageFileMetaData] = await storage_client.search_files(
75+
stored_files: list[
76+
StorageFileMetaData
77+
] = await storage_client.search_owned_files(
8278
user_id=user_id,
8379
file_id=file_id,
8480
sha256_checksum=None,
85-
access_right=access_right,
8681
)
8782
if not stored_files:
8883
msg = "Not found in storage"
@@ -289,7 +284,6 @@ async def get_file(
289284
file_id=file_id,
290285
storage_client=storage_client,
291286
user_id=user_id,
292-
access_right="read",
293287
)
294288

295289

@@ -307,11 +301,10 @@ async def search_files_page(
307301
file_id: UUID | None = None,
308302
):
309303
"""Search files"""
310-
stored_files: list[StorageFileMetaData] = await storage_client.search_files(
304+
stored_files: list[StorageFileMetaData] = await storage_client.search_owned_files(
311305
user_id=user_id,
312306
file_id=file_id,
313307
sha256_checksum=sha256_checksum,
314-
access_right="read",
315308
)
316309
if page_params.offset > len(stored_files):
317310
_logger.debug("File with sha256_checksum=%d not found.", sha256_checksum)
@@ -342,7 +335,6 @@ async def delete_file(
342335
file_id=file_id,
343336
storage_client=storage_client,
344337
user_id=user_id,
345-
access_right="write",
346338
)
347339
await storage_client.delete_file(
348340
user_id=user_id, quoted_storage_file_id=file.quoted_storage_file_id

‎services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -251,11 +251,10 @@ async def get_job_outputs(
251251
if isinstance(value, BaseFileLink):
252252
file_id: UUID = File.create_id(*value.path.split("/"))
253253

254-
found = await storage_client.search_files(
254+
found = await storage_client.search_owned_files(
255255
user_id=user_id,
256256
file_id=file_id,
257257
sha256_checksum=None,
258-
access_right="read",
259258
)
260259
if found:
261260
assert len(found) == 1 # nosec

‎services/api-server/src/simcore_service_api_server/services/storage.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ async def list_files(self, user_id: int) -> list[StorageFileMetaData]:
6161
response = await self.client.post(
6262
"/simcore-s3/files/metadata:search",
6363
params={
64+
"kind": "owned",
6465
"user_id": str(user_id),
6566
"startswith": "api/",
6667
},
@@ -74,24 +75,23 @@ async def list_files(self, user_id: int) -> list[StorageFileMetaData]:
7475
return files
7576

7677
@_exception_mapper({})
77-
async def search_files(
78+
async def search_owned_files(
7879
self,
7980
*,
8081
user_id: int,
8182
file_id: UUID | None,
8283
sha256_checksum: SHA256Str | None,
83-
access_right: AccessRight,
8484
) -> list[StorageFileMetaData]:
8585
# NOTE: can NOT use /locations/0/files/metadata with uuid_filter=api/ because
8686
# logic in storage 'wrongly' assumes that all data is associated to a project and
8787
# here there is no project, so it would always returns an empty
8888
params: dict = {
89+
"kind": "owned",
8990
"user_id": f"{user_id}",
9091
"startswith": None if file_id is None else f"api/{file_id}",
9192
"sha256_checksum": None
9293
if sha256_checksum is None
9394
else f"{sha256_checksum}",
94-
"access_right": access_right,
9595
}
9696

9797
response = await self.client.post(

‎services/api-server/src/simcore_service_api_server/services/study_job_models_converters.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,11 +111,10 @@ async def create_job_outputs_from_project_outputs(
111111
path = value["path"]
112112
file_id: UUID = File.create_id(*path.split("/"))
113113

114-
if found := await storage_client.search_files(
114+
if found := await storage_client.search_owned_files(
115115
user_id=user_id,
116116
file_id=file_id,
117117
sha256_checksum=None,
118-
access_right="read",
119118
):
120119
assert len(found) == 1 # nosec
121120
results[name] = to_file_api_model(found[0])

‎services/api-server/tests/mocks/delete_file.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
[
22
{
33
"name": "POST /simcore-s3/files/metadata:search",
4-
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search?user_id=1&startswith=api/cc3dd190-8c87-3686-b581-d9f7809d312e')>",
4+
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search?kind=owned&user_id=1&startswith=api/cc3dd190-8c87-3686-b581-d9f7809d312e')>",
55
"method": "POST",
66
"host": "storage",
77
"path": {
88
"path": "/v0/simcore-s3/files/metadata:search",
99
"path_parameters": []
1010
},
11-
"query": "user_id=1&startswith=api/cc3dd190-8c87-3686-b581-d9f7809d312e",
11+
"query": "kind=owned&user_id=1&startswith=api/cc3dd190-8c87-3686-b581-d9f7809d312e",
1212
"request_payload": null,
1313
"response_body": {
1414
"data": [

‎services/api-server/tests/mocks/get_solver_outputs.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,14 +219,14 @@
219219
},
220220
{
221221
"name": "POST /simcore-s3/files/metadata:search",
222-
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search?user_id=1&startswith=api/4ea24645-fd8c-339b-9621-ae045d45d94d&access_right=read')>",
222+
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search?kind=owned&user_id=1&startswith=api/4ea24645-fd8c-339b-9621-ae045d45d94d')>",
223223
"method": "POST",
224224
"host": "storage",
225225
"path": {
226226
"path": "/v0/simcore-s3/files/metadata:search",
227227
"path_parameters": []
228228
},
229-
"query": "user_id=1&startswith=api/4ea24645-fd8c-339b-9621-ae045d45d94d&access_right=read",
229+
"query": "kind=owned&user_id=1&startswith=api/4ea24645-fd8c-339b-9621-ae045d45d94d",
230230
"request_payload": null,
231231
"response_body": {
232232
"data": [

‎services/api-server/tests/mocks/run_study_workflow.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3238,15 +3238,15 @@
32383238
}
32393239
},
32403240
{
3241-
"name": "POST /simcore-s3/files/metadata:search",
3242-
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search?user_id=1&startswith=api%2F5b0cd3cd-5ceb-3d74-9961-246840c1e1d4&access_right=read')>",
3241+
"name": "POST /simcore-s3/files/metadata:search_owned",
3242+
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search_owned?user_id=1&startswith=api%2F5b0cd3cd-5ceb-3d74-9961-246840c1e1d4')>",
32433243
"method": "POST",
32443244
"host": "storage",
32453245
"path": {
3246-
"path": "/v0/simcore-s3/files/metadata:search",
3246+
"path": "/v0/simcore-s3/files/metadata:search_owned",
32473247
"path_parameters": []
32483248
},
3249-
"query": "user_id=1&startswith=api%2F5b0cd3cd-5ceb-3d74-9961-246840c1e1d4&access_right=read",
3249+
"query": "user_id=1&startswith=api%2F5b0cd3cd-5ceb-3d74-9961-246840c1e1d4",
32503250
"response_body": {
32513251
"data": []
32523252
}

‎services/api-server/tests/mocks/search_file_checksum.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
[
22
{
33
"name": "POST /simcore-s3/files/metadata:search",
4-
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search?user_id=2&sha256_checksum=92c7a39ce451ee57edd6da0b9c734ca9e6423a20410f73ce55e0d07cfd603b9d')>",
4+
"description": "<Request('POST', 'http://storage:8080/v0/simcore-s3/files/metadata:search?kind=owned&user_id=2&sha256_checksum=92c7a39ce451ee57edd6da0b9c734ca9e6423a20410f73ce55e0d07cfd603b9d')>",
55
"method": "POST",
66
"host": "storage",
77
"path": {
88
"path": "/v0/simcore-s3/files/metadata:search",
99
"path_parameters": []
1010
},
11-
"query": "user_id=2&sha256_checksum=92c7a39ce451ee57edd6da0b9c734ca9e6423a20410f73ce55e0d07cfd603b9d",
11+
"query": "kind=owned&user_id=2&sha256_checksum=92c7a39ce451ee57edd6da0b9c734ca9e6423a20410f73ce55e0d07cfd603b9d",
1212
"request_payload": null,
1313
"response_body": {
1414
"data": [

0 commit comments

Comments
 (0)