fix traps in test scripts (#700)

* fix traps in test scripts

* change minio setup to pytest fixture

* add try-finally, formatting

* minor fixes

Author: jstriebel

_tooling/local_minio_setup.sh

@@ -2,7 +2,6 @@
 set -eEuo pipefail
 
 source local_wk_setup.sh
-source ../_tooling/local_minio_setup.sh
 
 export_vars
 

webknossos/local_wk_setup.sh

@@ -1,10 +1,11 @@
 import itertools
 import json
-import os
 import pickle
+import shlex
+import subprocess
 import warnings
 from pathlib import Path
-from typing import Optional, Tuple, cast
+from typing import Iterator, Optional, Tuple, cast
 
 import numpy as np
 import pytest
@@ -37,11 +38,39 @@
 
 from .constants import TESTDATA_DIR, TESTOUTPUT_DIR
 
+MINIO_ROOT_USER = "TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur"
+MINIO_ROOT_PASSWORD = "ANTN35UAENTS5UIAEATD"
+MINIO_PORT = "8000"
+
+
+@pytest.fixture(autouse=True, scope="module")
+def docker_minio() -> Iterator[None]:
+    """Minio is an S3 clone and is used as local test server"""
+    container_name = "minio"
+    cmd = (
+        "docker run"
+        f" -p {MINIO_PORT}:9000"
+        f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}"
+        f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}"
+        f" --name {container_name}"
+        " --rm"
+        " -d"
+        " minio/minio server /data"
+    )
+    print("BEFORE", flush=True)
+    subprocess.check_output(shlex.split(cmd))
+    REMOTE_TESTOUTPUT_DIR.fs.mkdirs("testoutput", exist_ok=True)
+    try:
+        yield
+    finally:
+        subprocess.check_output(["docker", "stop", container_name])
+
+
 REMOTE_TESTOUTPUT_DIR = UPath(
     "s3://testoutput",
-    key=os.environ["MINIO_ROOT_USER"],
-    secret=os.environ["MINIO_ROOT_PASSWORD"],
-    client_kwargs={"endpoint_url": "http://localhost:8000"},
+    key=MINIO_ROOT_USER,
+    secret=MINIO_ROOT_PASSWORD,
+    client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"},
 )
 
 DATA_FORMATS = [DataFormat.WKW, DataFormat.Zarr]
@@ -192,11 +221,6 @@ def assure_exported_properties(ds: Dataset) -> None:
     ), "The properties did not match after reopening the dataset. This might indicate that the properties were not exported after they were changed in memory."
 
 
-@pytest.fixture(scope="session", autouse=True)
-def create_bucket() -> None:
-    REMOTE_TESTOUTPUT_DIR.fs.mkdirs("testoutput", exist_ok=True)
-
-
 @pytest.mark.parametrize("data_format,output_path", DATA_FORMATS_AND_OUTPUT_PATHS)
 def test_create_dataset_with_layer_and_mag(
     data_format: DataFormat, output_path: Path

webknossos/test.sh

@@ -1,12 +1,6 @@
 #!/usr/bin/env bash
 set -eEuo pipefail
 
-set +u
-if [[ -z $RUNNER_OS || "$RUNNER_OS" == "Linux" ]]; then
-    source ../_tooling/local_minio_setup.sh
-fi
-set -u
-
 # Note that pytest should be executed via `python -m`, since
 # this will ensure that the current directory is added to sys.path
 # (which is standard python behavior). This is necessary so that the imports

webknossos/tests/test_dataset.py

@@ -3,7 +3,6 @@
 import sys
 from os import environ
 from pathlib import Path
-from shutil import unpack_archive
 from typing import Union
 
 import numpy as np
@@ -14,6 +13,8 @@
 from webknossos.utils import copytree, rmtree
 
 from .constants import TESTDATA_DIR
+import subprocess
+import shlex
 
 
 def check_call(*args: Union[str, int, Path]) -> None:
@@ -28,16 +29,37 @@ def count_wkw_files(mag_path: Path) -> int:
     return len(list(mag_path.glob("**/x*.wkw")))
 
 
-@pytest.fixture(scope="session")
+MINIO_ROOT_USER = "TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur"
+MINIO_ROOT_PASSWORD = "ANTN35UAENTS5UIAEATD"
+MINIO_PORT = "8000"
+
+
+@pytest.fixture(scope="module")
 def remote_testoutput_path() -> UPath:
+    """Minio is an S3 clone and is used as local test server"""
+    container_name = "minio"
+    cmd = (
+        "docker run"
+        f" -p {MINIO_PORT}:9000"
+        f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}"
+        f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}"
+        f" --name {container_name}"
+        " --rm"
+        " -d"
+        " minio/minio server /data"
+    )
+    subprocess.check_output(shlex.split(cmd))
     remote_path = UPath(
         "s3://testoutput",
-        key=environ["MINIO_ROOT_USER"],
-        secret=environ["MINIO_ROOT_PASSWORD"],
-        client_kwargs={"endpoint_url": "http://localhost:8000"},
+        key=MINIO_ROOT_USER,
+        secret=MINIO_ROOT_PASSWORD,
+        client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"},
     )
     remote_path.fs.mkdirs("testoutput", exist_ok=True)
-    return remote_path
+    try:
+        yield remote_path
+    finally:
+        subprocess.check_output(["docker", "stop", container_name])
 
 
 def _tiff_cubing(
@@ -91,9 +113,9 @@ def test_tiff_cubing(tmp_path: Path) -> None:
 )
 def test_tiff_cubing_zarr_s3(remote_testoutput_path: UPath) -> None:
     out_path = remote_testoutput_path / "tiff_cubing"
-    environ["AWS_SECRET_ACCESS_KEY"] = environ["MINIO_ROOT_PASSWORD"]
-    environ["AWS_ACCESS_KEY_ID"] = environ["MINIO_ROOT_USER"]
-    environ["S3_ENDPOINT_URL"] = "http://localhost:8000"
+    environ["AWS_SECRET_ACCESS_KEY"] = MINIO_ROOT_PASSWORD
+    environ["AWS_ACCESS_KEY_ID"] = MINIO_ROOT_USER
+    environ["S3_ENDPOINT_URL"] = f"http://localhost:{MINIO_PORT}"
 
     _tiff_cubing(out_path, DataFormat.Zarr, 1)