Skip to content

Commit 59d9232

Browse files
sandereggsanderegg
authored
⚗️🐛 Add a test for reproducing potential 400 issue with upload to AWS (ITISFoundation#3538)
1 parent 6a85e40 commit 59d9232

File tree

11 files changed

+415
-10
lines changed

11 files changed

+415
-10
lines changed

‎packages/pytest-simcore/src/pytest_simcore/aws_services.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# pylint: disable=protected-access
2+
# pylint: disable=redefined-outer-name
3+
# pylint: disable=unused-argument
4+
# pylint: disable=unused-variable
5+
6+
import asyncio
7+
from typing import AsyncIterator, Iterator
8+
9+
import pytest
10+
from aiobotocore.session import get_session
11+
from aiohttp.test_utils import unused_port
12+
from moto.server import ThreadedMotoServer
13+
from pytest_simcore.helpers.utils_docker import get_localhost_ip
14+
15+
16+
@pytest.fixture(scope="module")
17+
def mocked_s3_server() -> Iterator[ThreadedMotoServer]:
18+
"""creates a moto-server that emulates AWS services in place
19+
NOTE: Never use a bucket with underscores it fails!!
20+
"""
21+
server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port())
22+
# pylint: disable=protected-access
23+
print(f"--> started mock S3 server on {server._ip_address}:{server._port}")
24+
print(
25+
f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]"
26+
)
27+
server.start()
28+
yield server
29+
server.stop()
30+
print(f"<-- stopped mock S3 server on {server._ip_address}:{server._port}")
31+
32+
33+
async def _clean_bucket_content(aiobotore_s3_client, bucket: str):
34+
response = await aiobotore_s3_client.list_objects_v2(Bucket=bucket)
35+
while response["KeyCount"] > 0:
36+
await aiobotore_s3_client.delete_objects(
37+
Bucket=bucket,
38+
Delete={
39+
"Objects": [
40+
{"Key": obj["Key"]} for obj in response["Contents"] if "Key" in obj
41+
]
42+
},
43+
)
44+
response = await aiobotore_s3_client.list_objects_v2(Bucket=bucket)
45+
46+
47+
async def _remove_all_buckets(aiobotore_s3_client):
48+
response = await aiobotore_s3_client.list_buckets()
49+
bucket_names = [
50+
bucket["Name"] for bucket in response["Buckets"] if "Name" in bucket
51+
]
52+
await asyncio.gather(
53+
*(_clean_bucket_content(aiobotore_s3_client, bucket) for bucket in bucket_names)
54+
)
55+
await asyncio.gather(
56+
*(aiobotore_s3_client.delete_bucket(Bucket=bucket) for bucket in bucket_names)
57+
)
58+
59+
60+
@pytest.fixture
61+
async def mocked_s3_server_envs(
62+
mocked_s3_server: ThreadedMotoServer, monkeypatch: pytest.MonkeyPatch
63+
) -> AsyncIterator[None]:
64+
monkeypatch.setenv("S3_SECURE", "false")
65+
monkeypatch.setenv(
66+
"S3_ENDPOINT",
67+
f"{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access
68+
)
69+
monkeypatch.setenv("S3_ACCESS_KEY", "xxx")
70+
monkeypatch.setenv("S3_SECRET_KEY", "xxx")
71+
monkeypatch.setenv("S3_BUCKET_NAME", "pytestbucket")
72+
73+
yield
74+
75+
# cleanup the buckets
76+
session = get_session()
77+
async with session.create_client(
78+
"s3",
79+
endpoint_url=f"http://{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access
80+
aws_secret_access_key="xxx",
81+
aws_access_key_id="xxx",
82+
) as client:
83+
await _remove_all_buckets(client)

‎packages/simcore-sdk/requirements/_base.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# Specifies third-party dependencies for 'simcore-sdk'
33
#
44
--constraint ../../../requirements/constraints.txt
5+
--constraint ./constraints.txt
56
--requirement ../../../packages/postgres-database/requirements/_base.in
67
--requirement ../../../packages/service-library/requirements/_base.in
78
--requirement ../../../packages/settings-library/requirements/_base.in

‎packages/simcore-sdk/requirements/_test.in‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# Specifies dependencies required to run 'simcore-sdk'
33
#
44
--constraint ../../../requirements/constraints.txt
5+
--constraint ./constraints.txt
56
# Adds base AS CONSTRAINT specs, not requirement.
67
# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt
78
#
@@ -17,6 +18,7 @@ docker
1718
faker
1819
flaky
1920
minio
21+
moto[server]
2022
pytest
2123
pytest-aiohttp
2224
pytest-cov

‎packages/simcore-sdk/requirements/_test.txt‎

Lines changed: 145 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,18 +35,34 @@ attrs==21.4.0
3535
# via
3636
# -c requirements/_base.txt
3737
# aiohttp
38+
# jschema-to-python
39+
# jsonschema
3840
# pytest
41+
# sarif-om
42+
aws-sam-translator==1.53.0
43+
# via cfn-lint
44+
aws-xray-sdk==2.10.0
45+
# via moto
3946
boto3==1.24.59
40-
# via aiobotocore
47+
# via
48+
# aiobotocore
49+
# aws-sam-translator
50+
# moto
4151
botocore==1.27.59
4252
# via
4353
# aiobotocore
54+
# aws-xray-sdk
4455
# boto3
56+
# moto
4557
# s3transfer
4658
certifi==2022.9.24
4759
# via
4860
# minio
4961
# requests
62+
cffi==1.15.1
63+
# via cryptography
64+
cfn-lint==0.71.0
65+
# via moto
5066
charset-normalizer==2.1.1
5167
# via
5268
# -c requirements/_base.txt
@@ -56,17 +72,31 @@ click==8.1.3
5672
# via
5773
# -c requirements/_base.txt
5874
# -r requirements/_test.in
75+
# flask
5976
coverage==6.5.0
6077
# via
6178
# -r requirements/_test.in
6279
# coveralls
6380
# pytest-cov
6481
coveralls==3.3.1
6582
# via -r requirements/_test.in
83+
cryptography==38.0.3
84+
# via
85+
# -c requirements/../../../requirements/constraints.txt
86+
# moto
87+
# python-jose
88+
# sshpubkeys
6689
docker==6.0.1
67-
# via -r requirements/_test.in
90+
# via
91+
# -r requirements/_test.in
92+
# moto
6893
docopt==0.6.2
6994
# via coveralls
95+
ecdsa==0.18.0
96+
# via
97+
# moto
98+
# python-jose
99+
# sshpubkeys
70100
exceptiongroup==1.0.1
71101
# via pytest
72102
execnet==1.9.0
@@ -75,11 +105,19 @@ faker==15.2.0
75105
# via -r requirements/_test.in
76106
flaky==3.7.0
77107
# via -r requirements/_test.in
108+
flask==2.1.3
109+
# via
110+
# flask-cors
111+
# moto
112+
flask-cors==3.0.10
113+
# via moto
78114
frozenlist==1.3.1
79115
# via
80116
# -c requirements/_base.txt
81117
# aiohttp
82118
# aiosignal
119+
graphql-core==3.2.3
120+
# via moto
83121
greenlet==2.0.1
84122
# via
85123
# -c requirements/_base.txt
@@ -89,14 +127,43 @@ icdiff==2.0.5
89127
idna==3.4
90128
# via
91129
# -c requirements/_base.txt
130+
# moto
92131
# requests
93132
# yarl
133+
importlib-metadata==5.0.0
134+
# via flask
94135
iniconfig==1.1.1
95136
# via pytest
137+
itsdangerous==2.1.2
138+
# via flask
139+
jinja2==3.1.2
140+
# via
141+
# -c requirements/../../../requirements/constraints.txt
142+
# flask
143+
# moto
96144
jmespath==1.0.1
97145
# via
98146
# boto3
99147
# botocore
148+
jschema-to-python==1.2.3
149+
# via cfn-lint
150+
jsondiff==2.0.0
151+
# via moto
152+
jsonpatch==1.32
153+
# via cfn-lint
154+
jsonpickle==2.2.0
155+
# via jschema-to-python
156+
jsonpointer==2.3
157+
# via jsonpatch
158+
jsonschema==3.2.0
159+
# via
160+
# -c requirements/_base.txt
161+
# aws-sam-translator
162+
# cfn-lint
163+
# openapi-schema-validator
164+
# openapi-spec-validator
165+
junit-xml==1.9
166+
# via cfn-lint
100167
mako==1.2.3
101168
# via
102169
# -c requirements/../../../requirements/constraints.txt
@@ -105,22 +172,40 @@ mako==1.2.3
105172
markupsafe==2.1.1
106173
# via
107174
# -c requirements/_base.txt
175+
# jinja2
108176
# mako
177+
# moto
109178
minio==7.0.4
110179
# via
111180
# -c requirements/../../../requirements/constraints.txt
112181
# -r requirements/_test.in
182+
moto==4.0.1
183+
# via
184+
# -c requirements/./constraints.txt
185+
# -r requirements/_test.in
113186
multidict==6.0.2
114187
# via
115188
# -c requirements/_base.txt
116189
# aiohttp
117190
# yarl
191+
networkx==2.8.8
192+
# via cfn-lint
193+
openapi-schema-validator==0.2.3
194+
# via openapi-spec-validator
195+
openapi-spec-validator==0.4.0
196+
# via
197+
# -c requirements/./constraints.txt
198+
# moto
118199
packaging==21.3
119200
# via
120201
# -c requirements/_base.txt
121202
# docker
122203
# pytest
123204
# pytest-sugar
205+
pbr==5.11.0
206+
# via
207+
# jschema-to-python
208+
# sarif-om
124209
pluggy==1.0.0
125210
# via pytest
126211
pprintpp==0.4.0
@@ -129,10 +214,21 @@ psycopg2-binary==2.9.5
129214
# via
130215
# -c requirements/_base.txt
131216
# sqlalchemy
217+
pyasn1==0.4.8
218+
# via
219+
# python-jose
220+
# rsa
221+
pycparser==2.21
222+
# via cffi
132223
pyparsing==3.0.9
133224
# via
134225
# -c requirements/_base.txt
226+
# moto
135227
# packaging
228+
pyrsistent==0.19.2
229+
# via
230+
# -c requirements/_base.txt
231+
# jsonschema
136232
pytest==7.2.0
137233
# via
138234
# -r requirements/_test.in
@@ -169,30 +265,62 @@ python-dateutil==2.8.2
169265
# via
170266
# botocore
171267
# faker
268+
# moto
172269
python-dotenv==0.21.0
173270
# via -r requirements/_test.in
271+
python-jose==3.3.0
272+
# via moto
273+
pytz==2022.6
274+
# via moto
275+
pyyaml==5.4.1
276+
# via
277+
# -c requirements/../../../requirements/constraints.txt
278+
# -c requirements/_base.txt
279+
# cfn-lint
280+
# moto
281+
# openapi-spec-validator
174282
requests==2.28.1
175283
# via
176284
# -r requirements/_test.in
177285
# coveralls
178286
# docker
287+
# moto
288+
# responses
289+
responses==0.22.0
290+
# via moto
291+
rsa==4.9
292+
# via
293+
# -c requirements/../../../requirements/constraints.txt
294+
# python-jose
179295
s3transfer==0.6.0
180296
# via boto3
297+
sarif-om==1.0.4
298+
# via cfn-lint
181299
six==1.16.0
182300
# via
183301
# -c requirements/_base.txt
302+
# ecdsa
303+
# flask-cors
304+
# jsonschema
305+
# junit-xml
184306
# python-dateutil
185307
sqlalchemy==1.4.43
186308
# via
187309
# -c requirements/../../../requirements/constraints.txt
188310
# -c requirements/_base.txt
189311
# alembic
312+
sshpubkeys==3.3.1
313+
# via moto
190314
termcolor==2.1.0
191315
# via pytest-sugar
316+
toml==0.10.2
317+
# via responses
192318
tomli==2.0.1
193319
# via
194320
# coverage
195321
# pytest
322+
types-toml==0.10.8
323+
# via responses
196324
typing-extensions==4.4.0
197325
# via
198326
# -c requirements/_base.txt
@@ -204,11 +332,25 @@ urllib3==1.26.12
204332
# docker
205333
# minio
206334
# requests
335+
# responses
207336
websocket-client==1.4.2
208337
# via docker
338+
werkzeug==2.1.2
339+
# via
340+
# flask
341+
# moto
209342
wrapt==1.14.1
210-
# via aiobotocore
343+
# via
344+
# aiobotocore
345+
# aws-xray-sdk
346+
xmltodict==0.13.0
347+
# via moto
211348
yarl==1.8.1
212349
# via
213350
# -c requirements/_base.txt
214351
# aiohttp
352+
zipp==3.10.0
353+
# via importlib-metadata
354+
355+
# The following packages are considered to be unsafe in a requirements file:
356+
# setuptools

0 commit comments

Comments
 (0)