Skip to content

Commit 2da3077

Browse files
author
Martin Durant
committed
Add s3 test
1 parent 51ec634 commit 2da3077

File tree

2 files changed

+54
-1
lines changed

2 files changed

+54
-1
lines changed

requirements_dev_optional.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ pytest-cov==2.7.1
1818
pytest-doctestplus==0.4.0
1919
pytest-remotedata==0.3.2
2020
h5py==2.10.0
21-
s3fs==0.3.4; python_version > '3.0'
21+
s3fs==0.5.0; python_version > '3.6'
22+
moto>=1.3.14; python_version > '3.6'

zarr/tests/test_storage.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -910,6 +910,58 @@ def test_eq(self):
910910
store2 = FSStore("anypath")
911911
assert store1 == store2
912912

913+
@pytest.mark.usefixtures("s3")
914+
def test_s3(self):
915+
import zarr
916+
g = zarr.open_group("s3://test/out.zarr", mode='w',
917+
storage_options=self.s3so)
918+
a = g.create_dataset("data", shape=(8,))
919+
a[:4] = [0, 1, 2, 3]
920+
921+
g = zarr.open_group("s3://test/out.zarr", mode='r',
922+
storage_options=self.s3so)
923+
924+
assert g.data[:].tolist() == [0, 1, 2, 3, 0, 0, 0, 0]
925+
926+
927+
@pytest.fixture()
928+
def s3(request):
929+
# writable local S3 system
930+
import shlex
931+
import subprocess
932+
import time
933+
if "BOTO_CONFIG" not in os.environ: # pragma: no cover
934+
os.environ["BOTO_CONFIG"] = "/dev/null"
935+
if "AWS_ACCESS_KEY_ID" not in os.environ: # pragma: no cover
936+
os.environ["AWS_ACCESS_KEY_ID"] = "foo"
937+
if "AWS_SECRET_ACCESS_KEY" not in os.environ: # pragma: no cover
938+
os.environ["AWS_SECRET_ACCESS_KEY"] = "bar"
939+
requests = pytest.importorskip("requests")
940+
s3fs = pytest.importorskip("s3fs")
941+
pytest.importorskip("moto")
942+
943+
port = 5555
944+
endpoint_uri = 'http://127.0.0.1:%s/' % port
945+
proc = subprocess.Popen(shlex.split("moto_server s3 -p %s" % port))
946+
947+
timeout = 5
948+
while timeout > 0:
949+
try:
950+
r = requests.get(endpoint_uri)
951+
if r.ok:
952+
break
953+
except Exception: # pragma: no cover
954+
pass
955+
timeout -= 0.1 # pragma: no cover
956+
time.sleep(0.1) # pragma: no cover
957+
s3so = dict(client_kwargs={'endpoint_url': endpoint_uri})
958+
s3 = s3fs.S3FileSystem(anon=False, **s3so)
959+
s3.mkdir("test")
960+
request.cls.s3so = s3so
961+
yield
962+
proc.terminate()
963+
proc.wait()
964+
913965

914966
class TestNestedDirectoryStore(TestDirectoryStore, unittest.TestCase):
915967

0 commit comments

Comments
 (0)