Skip to content

Commit d9eaa18

Browse files
authored
fix traps in test scripts (#700)
* fix traps in test scripts * change minio setup to pytest fixture * add try-finally, formatting * minor fixes
1 parent 7364802 commit d9eaa18

File tree

6 files changed

+65
-45
lines changed

6 files changed

+65
-45
lines changed

_tooling/local_minio_setup.sh

Lines changed: 0 additions & 19 deletions
This file was deleted.

webknossos/local_wk_setup.sh

100644100755
File mode changed.

webknossos/test.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
set -eEuo pipefail
33

44
source local_wk_setup.sh
5-
source ../_tooling/local_minio_setup.sh
65

76
export_vars
87

webknossos/tests/test_dataset.py

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import itertools
22
import json
3-
import os
43
import pickle
4+
import shlex
5+
import subprocess
56
import warnings
67
from pathlib import Path
7-
from typing import Optional, Tuple, cast
8+
from typing import Iterator, Optional, Tuple, cast
89

910
import numpy as np
1011
import pytest
@@ -37,11 +38,39 @@
3738

3839
from .constants import TESTDATA_DIR, TESTOUTPUT_DIR
3940

41+
MINIO_ROOT_USER = "TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur"
42+
MINIO_ROOT_PASSWORD = "ANTN35UAENTS5UIAEATD"
43+
MINIO_PORT = "8000"
44+
45+
46+
@pytest.fixture(autouse=True, scope="module")
47+
def docker_minio() -> Iterator[None]:
48+
"""Minio is an S3 clone and is used as local test server"""
49+
container_name = "minio"
50+
cmd = (
51+
"docker run"
52+
f" -p {MINIO_PORT}:9000"
53+
f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}"
54+
f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}"
55+
f" --name {container_name}"
56+
" --rm"
57+
" -d"
58+
" minio/minio server /data"
59+
)
60+
print("BEFORE", flush=True)
61+
subprocess.check_output(shlex.split(cmd))
62+
REMOTE_TESTOUTPUT_DIR.fs.mkdirs("testoutput", exist_ok=True)
63+
try:
64+
yield
65+
finally:
66+
subprocess.check_output(["docker", "stop", container_name])
67+
68+
4069
REMOTE_TESTOUTPUT_DIR = UPath(
4170
"s3://testoutput",
42-
key=os.environ["MINIO_ROOT_USER"],
43-
secret=os.environ["MINIO_ROOT_PASSWORD"],
44-
client_kwargs={"endpoint_url": "http://localhost:8000"},
71+
key=MINIO_ROOT_USER,
72+
secret=MINIO_ROOT_PASSWORD,
73+
client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"},
4574
)
4675

4776
DATA_FORMATS = [DataFormat.WKW, DataFormat.Zarr]
@@ -192,11 +221,6 @@ def assure_exported_properties(ds: Dataset) -> None:
192221
), "The properties did not match after reopening the dataset. This might indicate that the properties were not exported after they were changed in memory."
193222

194223

195-
@pytest.fixture(scope="session", autouse=True)
196-
def create_bucket() -> None:
197-
REMOTE_TESTOUTPUT_DIR.fs.mkdirs("testoutput", exist_ok=True)
198-
199-
200224
@pytest.mark.parametrize("data_format,output_path", DATA_FORMATS_AND_OUTPUT_PATHS)
201225
def test_create_dataset_with_layer_and_mag(
202226
data_format: DataFormat, output_path: Path

wkcuber/test.sh

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
#!/usr/bin/env bash
22
set -eEuo pipefail
33

4-
set +u
5-
if [[ -z $RUNNER_OS || "$RUNNER_OS" == "Linux" ]]; then
6-
source ../_tooling/local_minio_setup.sh
7-
fi
8-
set -u
9-
104
# Note that pytest should be executed via `python -m`, since
115
# this will ensure that the current directory is added to sys.path
126
# (which is standard python behavior). This is necessary so that the imports

wkcuber/tests/test_cli.py

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import sys
44
from os import environ
55
from pathlib import Path
6-
from shutil import unpack_archive
76
from typing import Union
87

98
import numpy as np
@@ -14,6 +13,8 @@
1413
from webknossos.utils import copytree, rmtree
1514

1615
from .constants import TESTDATA_DIR
16+
import subprocess
17+
import shlex
1718

1819

1920
def check_call(*args: Union[str, int, Path]) -> None:
@@ -28,16 +29,37 @@ def count_wkw_files(mag_path: Path) -> int:
2829
return len(list(mag_path.glob("**/x*.wkw")))
2930

3031

31-
@pytest.fixture(scope="session")
32+
MINIO_ROOT_USER = "TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur"
33+
MINIO_ROOT_PASSWORD = "ANTN35UAENTS5UIAEATD"
34+
MINIO_PORT = "8000"
35+
36+
37+
@pytest.fixture(scope="module")
3238
def remote_testoutput_path() -> UPath:
39+
"""Minio is an S3 clone and is used as local test server"""
40+
container_name = "minio"
41+
cmd = (
42+
"docker run"
43+
f" -p {MINIO_PORT}:9000"
44+
f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}"
45+
f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}"
46+
f" --name {container_name}"
47+
" --rm"
48+
" -d"
49+
" minio/minio server /data"
50+
)
51+
subprocess.check_output(shlex.split(cmd))
3352
remote_path = UPath(
3453
"s3://testoutput",
35-
key=environ["MINIO_ROOT_USER"],
36-
secret=environ["MINIO_ROOT_PASSWORD"],
37-
client_kwargs={"endpoint_url": "http://localhost:8000"},
54+
key=MINIO_ROOT_USER,
55+
secret=MINIO_ROOT_PASSWORD,
56+
client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"},
3857
)
3958
remote_path.fs.mkdirs("testoutput", exist_ok=True)
40-
return remote_path
59+
try:
60+
yield remote_path
61+
finally:
62+
subprocess.check_output(["docker", "stop", container_name])
4163

4264

4365
def _tiff_cubing(
@@ -91,9 +113,9 @@ def test_tiff_cubing(tmp_path: Path) -> None:
91113
)
92114
def test_tiff_cubing_zarr_s3(remote_testoutput_path: UPath) -> None:
93115
out_path = remote_testoutput_path / "tiff_cubing"
94-
environ["AWS_SECRET_ACCESS_KEY"] = environ["MINIO_ROOT_PASSWORD"]
95-
environ["AWS_ACCESS_KEY_ID"] = environ["MINIO_ROOT_USER"]
96-
environ["S3_ENDPOINT_URL"] = "http://localhost:8000"
116+
environ["AWS_SECRET_ACCESS_KEY"] = MINIO_ROOT_PASSWORD
117+
environ["AWS_ACCESS_KEY_ID"] = MINIO_ROOT_USER
118+
environ["S3_ENDPOINT_URL"] = f"http://localhost:{MINIO_PORT}"
97119

98120
_tiff_cubing(out_path, DataFormat.Zarr, 1)
99121

0 commit comments

Comments
 (0)