Skip to content

Commit 34c95a1

Browse files
valeriupredoivaleriupredoi
committed
format tests
1 parent c34c2f3 commit 34c95a1

21 files changed

+529
-421
lines changed

‎tests/conftest.py‎

Lines changed: 31 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
1+
import json
12
import os
2-
import s3fs
33
import pathlib
4-
import json
4+
55
import moto
66
import pytest
7-
7+
import s3fs
88
from moto.moto_server.threaded_moto_server import ThreadedMotoServer
99

10-
1110
# some spoofy server parameters
1211
# test parameters; don't modify these
1312
port = 5555
@@ -16,6 +15,7 @@
1615
versioned_bucket_name = "test-versioned"
1716
secure_bucket_name = "test-secure"
1817

18+
1919
def get_boto3_client():
2020
from botocore.session import Session
2121

@@ -71,38 +71,40 @@ def s3fs_s3(s3_base):
7171
client.create_bucket(Bucket=test_bucket_name, ACL="public-read")
7272

7373
client.create_bucket(Bucket=versioned_bucket_name, ACL="public-read")
74-
client.put_bucket_versioning(
75-
Bucket=versioned_bucket_name, VersioningConfiguration={"Status": "Enabled"}
76-
)
74+
client.put_bucket_versioning(Bucket=versioned_bucket_name,
75+
VersioningConfiguration={"Status": "Enabled"})
7776

7877
# initialize secure bucket
7978
client.create_bucket(Bucket=secure_bucket_name, ACL="public-read")
80-
policy = json.dumps(
81-
{
82-
"Version": "2012-10-17",
83-
"Id": "PutObjPolicy",
84-
"Statement": [
85-
{
86-
"Sid": "DenyUnEncryptedObjectUploads",
87-
"Effect": "Deny",
88-
"Principal": "*",
89-
"Action": "s3:PutObject",
90-
"Resource": "arn:aws:s3:::{bucket_name}/*".format(
91-
bucket_name=secure_bucket_name
92-
),
93-
"Condition": {
94-
"StringNotEquals": {
95-
"s3:x-amz-server-side-encryption": "aws:kms"
96-
}
97-
},
79+
policy = json.dumps({
80+
"Version":
81+
"2012-10-17",
82+
"Id":
83+
"PutObjPolicy",
84+
"Statement": [{
85+
"Sid":
86+
"DenyUnEncryptedObjectUploads",
87+
"Effect":
88+
"Deny",
89+
"Principal":
90+
"*",
91+
"Action":
92+
"s3:PutObject",
93+
"Resource":
94+
"arn:aws:s3:::{bucket_name}/*".format(
95+
bucket_name=secure_bucket_name),
96+
"Condition": {
97+
"StringNotEquals": {
98+
"s3:x-amz-server-side-encryption": "aws:kms"
9899
}
99-
],
100-
}
101-
)
100+
},
101+
}],
102+
})
102103

103104
client.put_bucket_policy(Bucket=secure_bucket_name, Policy=policy)
104105
s3fs.S3FileSystem.clear_instance_cache()
105-
s3 = s3fs.S3FileSystem(anon=False, client_kwargs={"endpoint_url": endpoint_uri})
106+
s3 = s3fs.S3FileSystem(anon=False,
107+
client_kwargs={"endpoint_url": endpoint_uri})
106108
s3.invalidate_cache()
107109

108110
yield s3

‎tests/s3_exploratory/test_s3_arrange_files.py‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
11
import os
2+
import tempfile
3+
from pathlib import Path
24

35
import fsspec
46
import numpy as np
57
import pytest
68
import s3fs
7-
import tempfile
9+
from config_minio import *
10+
from numpy.testing import assert_allclose, assert_array_equal
811

912
from activestorage.active import Active
1013
from activestorage.dummy_data import make_vanilla_ncdata
1114

12-
from numpy.testing import assert_allclose, assert_array_equal
13-
from pathlib import Path
14-
15-
from config_minio import *
1615

1716
# HDF5 chunking is paramount for performance
1817
# many small chunks slow down the process by factors of hundreds
@@ -40,33 +39,31 @@ def make_s3_file(test_spec):
4039
"""Make dummy data."""
4140
temp_folder = tempfile.mkdtemp()
4241
CHUNKS, NSIZE = test_spec
43-
s3_testfile = os.path.join(temp_folder,
44-
's3_test_bizarre_large.nc')
42+
s3_testfile = os.path.join(temp_folder, 's3_test_bizarre_large.nc')
4543
print(f"S3 Test file is {s3_testfile}")
4644
if not os.path.exists(s3_testfile):
47-
make_vanilla_ncdata(filename=s3_testfile,
48-
chunksize=CHUNKS, n=NSIZE)
45+
make_vanilla_ncdata(filename=s3_testfile, chunksize=CHUNKS, n=NSIZE)
4946

5047
return s3_testfile
5148

5249

5350
def make_local_file(test_data_path, test_spec):
5451
"""Create a vanilla nc file and store in test_data dir here."""
55-
local_testfile = os.path.join(test_data_path,
56-
'test_bizarre.nc')
52+
local_testfile = os.path.join(test_data_path, 'test_bizarre.nc')
5753
CHUNKS, NSIZE = test_spec
5854

5955
print(f"Local Test file is {local_testfile}")
6056
if not os.path.exists(local_testfile):
61-
make_vanilla_ncdata(filename=local_testfile,
62-
chunksize=CHUNKS, n=NSIZE)
57+
make_vanilla_ncdata(filename=local_testfile, chunksize=CHUNKS, n=NSIZE)
6358

6459
return local_testfile
6560

6661

6762
def upload_to_s3(server, username, password, bucket, object, rfile):
6863
"""Upload a file to an S3 object store."""
69-
s3_fs = s3fs.S3FileSystem(key=username, secret=password, client_kwargs={'endpoint_url': server})
64+
s3_fs = s3fs.S3FileSystem(key=username,
65+
secret=password,
66+
client_kwargs={'endpoint_url': server})
7067
# Make sure s3 bucket exists
7168
try:
7269
s3_fs.mkdir(bucket)
@@ -83,13 +80,13 @@ def test_create_files(test_data_path, test_spec):
8380
# this runs as a simple pytest test ahead of the performance test
8481

8582
# make dummy data
86-
s3_testfile = make_s3_file(test_spec)
83+
s3_testfile = make_s3_file(test_spec)
8784
local_testfile = make_local_file(test_data_path, test_spec)
8885

8986
# put s3 dummy data onto S3
9087
object = os.path.basename(s3_testfile)
91-
bucket_file = upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY,
92-
S3_BUCKET, object, s3_testfile)
88+
bucket_file = upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET,
89+
object, s3_testfile)
9390

9491
s3_testfile_uri = os.path.join("s3://", bucket_file)
9592

‎tests/s3_exploratory/test_s3_performance.py‎

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1+
from pathlib import Path
2+
13
import fsspec
24
import pytest
35
import s3fs
46
import ujson
5-
6-
from pathlib import Path
7+
from config_minio import *
78

89
from activestorage.active import Active
9-
from config_minio import *
1010

1111

1212
@pytest.fixture
@@ -25,11 +25,9 @@ def test_s3_SingleHdf5ToZarr():
2525
secret=S3_SECRET_KEY,
2626
client_kwargs={'endpoint_url': S3_URL},
2727
default_fill_cache=False,
28-
default_cache_type="none"
29-
)
28+
default_cache_type="none")
3029
with fs.open(s3_file, 'rb') as s3file:
31-
h5chunks = SingleHdf5ToZarr(s3file, s3_file,
32-
inline_threshold=0)
30+
h5chunks = SingleHdf5ToZarr(s3file, s3_file, inline_threshold=0)
3331

3432

3533
@pytest.mark.xfail(reason="Pyfive don't use Kerchunk")
@@ -40,8 +38,7 @@ def test_local_SingleHdf5ToZarr(test_data_path):
4038
local_file = str(test_data_path / "test_bizarre.nc")
4139
fs = fsspec.filesystem('')
4240
with fs.open(local_file, 'rb') as localfile:
43-
h5chunks = SingleHdf5ToZarr(localfile, local_file,
44-
inline_threshold=0)
41+
h5chunks = SingleHdf5ToZarr(localfile, local_file, inline_threshold=0)
4542

4643

4744
@pytest.mark.xfail(reason="Pyfive don't use Kerchunk")
@@ -53,12 +50,10 @@ def test_s3_kerchunk_to_json():
5350
secret=S3_SECRET_KEY,
5451
client_kwargs={'endpoint_url': S3_URL},
5552
default_fill_cache=False,
56-
default_cache_type="none"
57-
)
53+
default_cache_type="none")
5854
fs2 = fsspec.filesystem('')
5955
with fs.open(s3_file, 'rb') as s3file:
60-
h5chunks = SingleHdf5ToZarr(s3file, s3_file,
61-
inline_threshold=0)
56+
h5chunks = SingleHdf5ToZarr(s3file, s3_file, inline_threshold=0)
6257
# to here, SingleHdf5ToZarr is VERY quick and MEM-light
6358
# eg 0.21s and 65M RES for a 50M file
6459
with fs2.open(outf, 'wb') as f:
@@ -72,8 +67,7 @@ def test_local_kerchunk_to_json(test_data_path):
7267
outf = "loocal_dump.json"
7368
fs = fsspec.filesystem('')
7469
with fs.open(local_file, 'rb') as localfile:
75-
h5chunks = SingleHdf5ToZarr(localfile, local_file,
76-
inline_threshold=0)
70+
h5chunks = SingleHdf5ToZarr(localfile, local_file, inline_threshold=0)
7771
# to here, SingleHdf5ToZarr is VERY quick and MEM-light
7872
# eg 0.07s and 65M RES for a 50M file
7973
with fs.open(outf, 'wb') as f:
@@ -89,12 +83,10 @@ def test_s3_kerchunk_openZarrGroup():
8983
secret=S3_SECRET_KEY,
9084
client_kwargs={'endpoint_url': S3_URL},
9185
default_fill_cache=False,
92-
default_cache_type="none"
93-
)
86+
default_cache_type="none")
9487
fs2 = fsspec.filesystem('')
9588
with fs.open(s3_file, 'rb') as s3file:
96-
h5chunks = SingleHdf5ToZarr(s3file, s3_file,
97-
inline_threshold=0)
89+
h5chunks = SingleHdf5ToZarr(s3file, s3_file, inline_threshold=0)
9890
# to here, SingleHdf5ToZarr is VERY quick and MEM-light
9991
# eg 0.21s and 65M RES for a 50M file
10092
with fs2.open(outf, 'wb') as f:
@@ -109,8 +101,7 @@ def test_local_kerchunk_openZarrGroup(test_data_path):
109101
outf = "loocal_dump.json"
110102
fs = fsspec.filesystem('')
111103
with fs.open(local_file, 'rb') as localfile:
112-
h5chunks = SingleHdf5ToZarr(localfile, local_file,
113-
inline_threshold=0)
104+
h5chunks = SingleHdf5ToZarr(localfile, local_file, inline_threshold=0)
114105
# to here, SingleHdf5ToZarr is VERY quick and MEM-light
115106
# eg 0.07s and 65M RES for a 50M file
116107
with fs.open(outf, 'wb') as f:

‎tests/s3_exploratory/test_s3_reduction.py‎

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,30 @@
11
import os
2+
import tempfile
3+
from pathlib import Path
4+
25
import numpy as np
36
import pytest
47
import s3fs
5-
import tempfile
8+
from config_minio import *
9+
from numpy.testing import assert_allclose, assert_array_equal
610

11+
import activestorage.storage as st
712
from activestorage.active import Active
813
from activestorage.dummy_data import make_vanilla_ncdata
9-
import activestorage.storage as st
10-
from activestorage.reductionist import reduce_chunk as reductionist_reduce_chunk
1114
from activestorage.reductionist import get_session
12-
from numpy.testing import assert_allclose, assert_array_equal
13-
from pathlib import Path
14-
15-
from config_minio import *
15+
from activestorage.reductionist import \
16+
reduce_chunk as reductionist_reduce_chunk
1617

1718

1819
def make_tempfile():
1920
"""Make dummy data."""
2021
temp_folder = tempfile.mkdtemp()
21-
s3_testfile = os.path.join(temp_folder,
22-
's3_test_bizarre.nc')
22+
s3_testfile = os.path.join(temp_folder, 's3_test_bizarre.nc')
2323
print(f"S3 Test file is {s3_testfile}")
2424
if not os.path.exists(s3_testfile):
2525
make_vanilla_ncdata(filename=s3_testfile)
2626

27-
local_testfile = os.path.join(temp_folder,
28-
'local_test_bizarre.nc')
27+
local_testfile = os.path.join(temp_folder, 'local_test_bizarre.nc')
2928
print(f"Local Test file is {local_testfile}")
3029
if not os.path.exists(local_testfile):
3130
make_vanilla_ncdata(filename=local_testfile)
@@ -35,7 +34,9 @@ def make_tempfile():
3534

3635
def upload_to_s3(server, username, password, bucket, object, rfile):
3736
"""Upload a file to an S3 object store."""
38-
s3_fs = s3fs.S3FileSystem(key=username, secret=password, client_kwargs={'endpoint_url': server})
37+
s3_fs = s3fs.S3FileSystem(key=username,
38+
secret=password,
39+
client_kwargs={'endpoint_url': server})
3940
# Make sure s3 bucket exists
4041
try:
4142
s3_fs.mkdir(bucket)
@@ -60,8 +61,8 @@ def test_Active():
6061

6162
# put s3 dummy data onto S3. then rm from local
6263
object = os.path.basename(s3_testfile)
63-
bucket_file = upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY,
64-
S3_BUCKET, object, s3_testfile)
64+
bucket_file = upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET,
65+
object, s3_testfile)
6566
os.remove(s3_testfile)
6667
s3_testfile_uri = os.path.join("s3://", bucket_file)
6768
print("S3 file uri", s3_testfile_uri)
@@ -80,7 +81,7 @@ def test_Active():
8081
result2 = active[0:2, 4:6, 7:9]
8182
print(result2)
8283

83-
assert_array_equal(result1, result2["sum"]/result2["n"])
84+
assert_array_equal(result1, result2["sum"] / result2["n"])
8485

8586

8687
@pytest.fixture
@@ -109,8 +110,8 @@ def test_with_valid_netCDF_file(test_data_path):
109110

110111
# put data onto S3. then rm from local
111112
object = os.path.basename(ncfile)
112-
bucket_file = upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY,
113-
S3_BUCKET, object, ncfile)
113+
bucket_file = upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET,
114+
object, ncfile)
114115
s3_testfile_uri = os.path.join("s3://", bucket_file)
115116
print("S3 file uri", s3_testfile_uri)
116117

@@ -124,7 +125,9 @@ def test_with_valid_netCDF_file(test_data_path):
124125

125126
# expect {'sum': array([[[2368.3232]]], dtype=float32), 'n': array([[[8]]])}
126127
# check for typing and structure
127-
assert_allclose(result1["sum"], np.array([[[2368.3232]]], dtype="float32"), rtol=1e-6)
128+
assert_allclose(result1["sum"],
129+
np.array([[[2368.3232]]], dtype="float32"),
130+
rtol=1e-6)
128131
assert_array_equal(result1["n"], np.array([[[8]]]))
129132

130133
assert_allclose(result1["sum"], result2["sum"], rtol=1e-6)
@@ -139,16 +142,18 @@ def test_reductionist_reduce_chunk():
139142
object = os.path.basename(rfile)
140143

141144
# create bucket and upload to Minio's S3 bucket
142-
upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY,
143-
S3_BUCKET, object, rfile)
144-
145+
upload_to_s3(S3_URL, S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET, object,
146+
rfile)
147+
145148
# call reductionist_reduce_chunk
146-
session = get_session(S3_ACCESS_KEY, S3_SECRET_KEY, S3_ACTIVE_STORAGE_CACERT)
149+
session = get_session(S3_ACCESS_KEY, S3_SECRET_KEY,
150+
S3_ACTIVE_STORAGE_CACERT)
147151
tmp, count = reductionist_reduce_chunk(session, S3_ACTIVE_STORAGE_URL,
148-
S3_URL, S3_BUCKET,
149-
object, offset, size, None, None,
150-
[], np.dtype("int32"), (32, ), "C",
151-
[slice(0, 2, 1), ], None, "min")
152+
S3_URL, S3_BUCKET, object, offset,
153+
size, None, None, [],
154+
np.dtype("int32"), (32, ), "C", [
155+
slice(0, 2, 1),
156+
], None, "min")
152157
assert tmp == 134351386
153158
# count is returned as a list even for one element
154159
assert count == [2]

0 commit comments

Comments
 (0)