Fix mag paths under Windows (#1329)

* fix windows paths, maybe

* ci

* tests

* ci

* windowspath

* windowspath

* uv version

* changelog

* windows minio

* ci

* ci

* ci

* ci

* ci

* utc

* utc

* ci

* ci

* ci

* lint

* absolute path

* more as_posix

* ci

* ci

* ci

* ci

* use as_posix

Author: normanrz

.github/workflows/ci.yml

@@ -44,7 +44,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
         with:
-          version: "0.6.3"
           enable-cache: true
           cache-dependency-glob: "cluster_tools/uv.lock"
       - run: uv python install ${{ matrix.python-version }}
@@ -104,7 +103,6 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.6.3"
           enable-cache: true
           cache-dependency-glob: "cluster_tools/uv.lock"
       - name: Set up Python ${{ matrix.python-version }}
@@ -142,7 +140,6 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.6.3"
           enable-cache: true
           cache-dependency-glob: "cluster_tools/uv.lock"
       - name: Set up Python ${{ matrix.python-version }}
@@ -184,7 +181,6 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.6.3"
           enable-cache: true
           cache-dependency-glob: "cluster_tools/uv.lock"
       - name: Set up Python ${{ matrix.python-version }}
@@ -217,8 +213,6 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v3
         with:
-          # Install a specific version of uv.
-          version: "0.5.26"
           enable-cache: true
           cache-dependency-glob: "webknossos/uv.lock"
 
@@ -273,6 +267,88 @@ jobs:
           git diff --no-ext-diff --exit-code
           [[ -z $(git status -s) ]]
 
+  webknossos_windows:
+    needs: changes
+    if: |
+      ${{ needs.changes.outputs.cluster_tools == 'true' }} ||
+      ${{ needs.changes.outputs.webknossos == 'true' }}
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        group: [1, 2, 3]
+      fail-fast: false
+    defaults:
+      run:
+        working-directory: webknossos
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "webknossos/uv.lock"
+
+      - name: Install proxay
+        run: npm install -g proxay
+
+      - name: Install minio
+        run: Invoke-WebRequest -Uri "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" -OutFile "minio.exe"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Python tests
+        timeout-minutes: 30
+        shell: bash
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          MULTIPROCESSING_DEFAULT_START_METHOD: spawn
+        run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration
+
+  webknossos_macos:
+    needs: changes
+    if: |
+      ${{ needs.changes.outputs.cluster_tools == 'true' }} ||
+      ${{ needs.changes.outputs.webknossos == 'true' }}
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        group: [1, 2, 3]
+      fail-fast: false
+    defaults:
+      run:
+        working-directory: webknossos
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "webknossos/uv.lock"
+
+      - name: Install proxay
+        run: npm install -g proxay
+
+      - name: Install minio
+        run: brew install minio/stable/minio
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Python tests
+        timeout-minutes: 30
+        shell: bash
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration
+
+
   coverage_report:
     needs: [webknossos_linux]
     if: success()
@@ -316,6 +392,8 @@ jobs:
       - cluster_tools_kubernetes
       - cluster_tools_dask
       - webknossos_linux
+      - webknossos_windows
+      - webknossos_macos
     if: |
       always() &&
       !contains(needs.*.result, 'failure') &&
@@ -326,9 +404,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Install uv
         uses: astral-sh/setup-uv@v3
-        with:
-          # Install a specific version of uv.
-          version: "0.6.3"
       - name: Write version file
         run: |
           pushd webknossos
@@ -393,6 +468,8 @@ jobs:
       - cluster_tools_kubernetes
       - cluster_tools_dask
       - webknossos_linux
+      - webknossos_windows
+      - webknossos_macos
     runs-on: ubuntu-latest
     if: |
       always() &&
@@ -408,9 +485,6 @@ jobs:
           path: docs/wk-repo
       - name: Install uv
         uses: astral-sh/setup-uv@v3
-        with:
-          # Install a specific version of uv.
-          version: "0.6.3"
       - name: Build Docs
         run: |
           cd docs
@@ -454,6 +528,8 @@ jobs:
       - cluster_tools_kubernetes
       - cluster_tools_dask
       - webknossos_linux
+      - webknossos_windows
+      - webknossos_macos
     if: |
       always() &&
       !contains(needs.*.result, 'failure') &&
@@ -465,9 +541,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Install uv
         uses: astral-sh/setup-uv@v3
-        with:
-          # Install a specific version of uv.
-          version: "0.6.3"
       - name: Publish python packages
         env:
           UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }}
@@ -490,16 +563,16 @@ jobs:
 
   complete:
     needs:
-      [
-        cluster_tools_dask,
-        cluster_tools_kubernetes,
-        cluster_tools_multiprocessing,
-        cluster_tools_slurm,
-        webknossos_linux,
-        webknossos_cli_docker,
-        docs,
-        pypi_and_gh_release,
-      ]
+      - cluster_tools_dask
+      - cluster_tools_kubernetes
+      - cluster_tools_multiprocessing
+      - cluster_tools_slurm
+      - webknossos_linux
+      - webknossos_windows
+      - webknossos_macos
+      - webknossos_cli_docker
+      - docs
+      - pypi_and_gh_release
     if: always()
     runs-on: ubuntu-latest
     steps:

.github/workflows/nightly.yml

@@ -23,8 +23,6 @@ jobs:
 
     - name: Install uv
       uses: astral-sh/setup-uv@v3
-      with:
-        version: "0.5.26"
 
     - name: Install proxay
       run: npm install -g proxay

.github/workflows/publish_docs.yml

@@ -17,9 +17,6 @@ jobs:
         path: docs/wk-repo
     - name: Install uv
       uses: astral-sh/setup-uv@v3
-      with:
-        # Install a specific version of uv.
-        version: "0.6.3"
 
     - name: Build Docs
       run: |

webknossos/Changelog.md

@@ -20,7 +20,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 - Warnings regarding missing pims image readers are only printed once during conversion. Error messages regarding wk-libs version are not displayed in slurm environments. [#1266](https://github.com/scalableminds/webknossos-libs/pull/1266)
 
 ### Fixed
-
+- Fixed mag and attachment paths under Windows. [#1329](https://github.com/scalableminds/webknossos-libs/pull/1329)
 
 ## [2.4.2](https://github.com/scalableminds/webknossos-libs/releases/tag/v2.4.2) - 2025-07-07
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v2.4.1...v2.4.2)

webknossos/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.12
 
-COPY --from=ghcr.io/astral-sh/uv:0.5.26 /uv /bin/uv
+COPY --from=ghcr.io/astral-sh/uv:0.7.17 /uv /bin/uv
 
 RUN mkdir /webknossos
 COPY webknossos/webknossos /webknossos/webknossos

webknossos/test.sh

@@ -4,7 +4,7 @@ set -eEuo pipefail
 source local_wk_setup.sh
 
 # Using forkserver instead of spawn is faster. Fork should never be used due to potential deadlock problems.
-export MULTIPROCESSING_DEFAULT_START_METHOD=forkserver
+export MULTIPROCESSING_DEFAULT_START_METHOD=${MULTIPROCESSING_DEFAULT_START_METHOD:-forkserver}
 
 # Note that pytest should be executed via `python -m`, since
 # this will ensure that the current directory is added to sys.path

webknossos/tests/constants.py

@@ -49,6 +49,26 @@ def use_minio() -> Iterator[None]:
             minio_process.terminate()
             sleep(1)
             rmtree(minio_path)
+    elif sys.platform == "win32":
+        minio_path = UPath("testoutput_minio")
+        rmtree(minio_path)
+        minio_process = subprocess.Popen(
+            shlex.split(f"minio.exe server --address :8000 {minio_path.absolute()}"),
+            env={
+                **os.environ,
+                "MINIO_ROOT_USER": MINIO_ROOT_USER,
+                "MINIO_ROOT_PASSWORD": MINIO_ROOT_PASSWORD,
+            },
+        )
+        sleep(3)
+        assert minio_process.poll() is None
+        REMOTE_TESTOUTPUT_DIR.fs.mkdirs("testoutput", exist_ok=True)
+        try:
+            yield
+        finally:
+            minio_process.terminate()
+            sleep(1)
+            rmtree(minio_path)
     else:
         container_name = "minio"
         cmd = (

webknossos/tests/dataset/test_add_layer_from_images.py

@@ -1,3 +1,4 @@
+import sys
 import warnings
 from collections.abc import Iterator
 from pathlib import Path
@@ -341,6 +342,7 @@ def _test_bioformats(
     return ds
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows")
 @pytest.mark.parametrize(
     "url, filename, kwargs, dtype, num_channels, size, num_layers", BIOFORMATS_ARGS
 )
@@ -509,6 +511,7 @@ def _test_test_images(
 @pytest.mark.parametrize(
     "url, filename, kwargs, dtype, num_channels, size", TEST_IMAGES_ARGS
 )
+@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows")
 def test_test_images(
     tmp_path: Path,
     url: str | list[str],

webknossos/tests/dataset/test_attachments.py

@@ -82,14 +82,14 @@ def test_attachments(tmp_upath: UPath) -> None:
 
     # segment index
     seg_layer.attachments.set_segment_index(
-        "/usr/segment_index.hdf5",
+        Path.home() / "segment_index.hdf5",
         name="main",
         data_format=AttachmentDataFormat.HDF5,
     )
     assert seg_layer._properties.attachments.segment_index is not None
     assert (
         seg_layer._properties.attachments.segment_index.path
-        == "/usr/segment_index.hdf5"
+        == (Path.home() / "segment_index.hdf5").as_posix()
     )
     assert (
         seg_layer._properties.attachments.segment_index.data_format
@@ -225,8 +225,9 @@ def test_fs_copy_layer(tmp_upath: UPath) -> None:
 
     # has not been copied
     assert copy_layer.attachments.agglomerates[0].path == agglomerate_path
-    assert copy_layer.attachments.agglomerates[0]._properties.path == str(
-        agglomerate_path
+    assert (
+        copy_layer.attachments.agglomerates[0]._properties.path
+        == agglomerate_path.as_posix()
     )
 
 
@@ -427,7 +428,7 @@ def test_add_symlink_attachments(tmp_upath: UPath) -> None:
     with pytest.warns(DeprecationWarning):
         seg_layer.attachments.add_symlink_attachments(mesh)
     assert seg_layer._properties.attachments.meshes is not None
-    assert seg_layer._properties.attachments.meshes[0].path == str(mesh_path)
+    assert seg_layer._properties.attachments.meshes[0].path == mesh_path.as_posix()
     assert seg_layer.attachments.meshes[0].name == "meshfile_4-4-1"
     assert (dataset.path / "seg" / "meshes" / "meshfile_4-4-1").exists()
     assert (dataset.path / "seg" / "meshes" / "meshfile_4-4-1").is_symlink()
@@ -446,8 +447,9 @@ def test_add_symlink_attachments(tmp_upath: UPath) -> None:
     with pytest.warns(DeprecationWarning):
         seg_layer.attachments.add_symlink_attachments(segment_index)
     assert seg_layer._properties.attachments.segment_index is not None
-    assert seg_layer._properties.attachments.segment_index.path == str(
-        segment_index_path
+    assert (
+        seg_layer._properties.attachments.segment_index.path
+        == segment_index_path.as_posix()
     )
     assert (dataset.path / "seg" / "segmentIndex" / "main").exists()
     assert (dataset.path / "seg" / "segmentIndex" / "main").is_symlink()

webknossos/tests/dataset/test_dataset.py

@@ -359,6 +359,22 @@ def recursive_chmod(ds_path: Path, mode: int) -> None:
         recursive_chmod(ds_path, 0o777)
 
 
+@pytest.mark.parametrize("data_format", DATA_FORMATS)
+def test_mag_paths(data_format: DataFormat) -> None:
+    ds_path = prepare_dataset_path(data_format, TESTOUTPUT_DIR)
+    layer = Dataset(ds_path, voxel_size=(1, 1, 4)).add_layer(
+        "color",
+        COLOR_CATEGORY,
+        bounding_box=BoundingBox((0, 0, 0), (32, 32, 32)),
+        data_format=data_format,
+    )
+    mag1 = layer.add_mag("1")
+    mag2 = layer.add_mag("2-2-1")
+
+    assert mag1._properties.path == "./color/1"
+    assert mag2._properties.path == "./color/2-2-1"
+
+
 def test_create_default_layer() -> None:
     ds_path = prepare_dataset_path(DEFAULT_DATA_FORMAT, TESTOUTPUT_DIR)
     ds = Dataset(ds_path, voxel_size=(1, 1, 1))
@@ -2890,7 +2906,7 @@ def test_rename_layer(data_format: DataFormat, output_path: UPath) -> None:
         len([layer for layer in ds._properties.data_layers if layer.name == "color2"])
         == 1
     )
-    assert ds._properties.data_layers[0].mags[0].path == "color2/1"
+    assert ds._properties.data_layers[0].mags[0].path == "./color2/1"
     assert "color2" in ds.layers.keys()
     assert "color" not in ds.layers.keys()
     assert ds.get_layer("color2").data_format == data_format