Merge branch 'develop' into fix-zarr-check

Author: shaneahmed

.github/workflows/python-package.yml

@@ -30,7 +30,7 @@ jobs:
         sudo apt update
         sudo apt-get install -y libopenslide-dev openslide-tools libopenjp2-7 libopenjp2-tools
         python -m pip install --upgrade pip
-        python -m pip install ruff==0.9.4 pytest pytest-cov pytest-runner
+        python -m pip install ruff==0.9.9 pytest pytest-cov pytest-runner
         pip install -r requirements/requirements.txt
     - name: Cache tiatoolbox static assets
       uses: actions/cache@v3

.pre-commit-config.yaml

@@ -60,7 +60,7 @@ repos:
       - id: rst-inline-touching-normal # Detect mistake of inline code touching normal text in rst.
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.9.4
+    rev: v0.9.9
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

requirements/requirements.txt

@@ -1,5 +1,6 @@
 # torch installation
 --extra-index-url https://download.pytorch.org/whl/cu118; sys_platform != "darwin"
+aiohttp>=3.8.1
 albumentations>=1.3.0
 bokeh>=3.1.1, <3.6.0
 Click>=8.1.3

requirements/requirements_dev.txt

@@ -10,7 +10,7 @@ pytest>=7.2.0
 pytest-cov>=4.0.0
 pytest-runner>=6.0
 pytest-xdist[psutil]
-ruff==0.9.4  # This will be updated by pre-commit bot to latest version
+ruff==0.9.9  # This will be updated by pre-commit bot to latest version
 toml>=0.10.2
 twine>=4.0.1
 wheel>=0.37.1

tests/test_wsireader.py

@@ -9,9 +9,8 @@
 import shutil
 from copy import deepcopy
 from pathlib import Path
-
-# When no longer supporting Python <3.9 this should be collections.abc.Iterable
 from typing import TYPE_CHECKING, Callable
+from unittest.mock import patch
 
 import cv2
 import glymur
@@ -27,7 +26,7 @@
 
 from tiatoolbox import cli, utils
 from tiatoolbox.annotation import SQLiteStore
-from tiatoolbox.utils import imread
+from tiatoolbox.utils import imread, tiff_to_fsspec
 from tiatoolbox.utils.exceptions import FileNotSupportedError
 from tiatoolbox.utils.magic import is_sqlite3
 from tiatoolbox.utils.transforms import imresize, locsize2bounds
@@ -37,6 +36,7 @@
     AnnotationStoreReader,
     ArrayView,
     DICOMWSIReader,
+    FsspecJsonWSIReader,
     JP2WSIReader,
     NGFFWSIReader,
     OpenSlideWSIReader,
@@ -221,6 +221,43 @@ def read_bounds_level_consistency(wsi: WSIReader, bounds: IntBounds) -> None:
 # Utility Test Classes & Functions
 # -------------------------------------------------------------------------------------
 
+_FSSPEC_WSI_CACHE = {}
+
+
+def fsspec_wsi(sample_svs: Path, tmp_path: Path) -> FsspecJsonWSIReader:
+    """Returns cached FsspecJsonWSIReader instance.
+
+    The reader instance opens CMU-1-Small-Region.svs image.
+
+    It's cached so the reader can be reused,
+
+    since loading the whole image using HTTP range requests from:
+
+    https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs
+
+    takes about 20 seconds.
+
+    """
+    cache_key = "sample_svs"
+
+    if cache_key in _FSSPEC_WSI_CACHE:
+        return _FSSPEC_WSI_CACHE[cache_key]  # Return cached instance
+
+    file_types = ("*.svs",)
+    files_all = utils.misc.grab_files_from_dir(
+        input_path=Path(sample_svs).parent,
+        file_types=file_types,
+    )
+    svs_file_path = str(files_all[0])
+    json_file_path = str(tmp_path / "fsspec.json")
+    final_url = (
+        "https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs"
+    )
+    tiff_to_fsspec.main(svs_file_path, json_file_path, final_url)
+
+    _FSSPEC_WSI_CACHE[cache_key] = wsireader.FsspecJsonWSIReader(json_file_path)
+    return _FSSPEC_WSI_CACHE[cache_key]
+
 
 class DummyMutableOpenSlideObject:
     """Dummy OpenSlide object with mutable properties."""
@@ -2812,3 +2849,133 @@ def test_read_multi_channel(source_image: Path) -> None:
     assert region.shape == (100, 50, (new_img_array.shape[-1]))
     assert np.abs(np.median(region.astype(int) - target.astype(int))) == 0
     assert np.abs(np.mean(region.astype(int) - target.astype(int))) < 0.2
+
+
+def test_fsspec_json_wsi_reader_instantiation() -> None:
+    """Test if FsspecJsonWSIReader is instantiated.
+
+    In case json is passed to  WSIReader.open, FsspecJsonWSIReader
+    should be instantiated.
+    """
+    input_path = "mock_path.json"
+    mpp = None
+    power = None
+
+    with (
+        patch(
+            "tiatoolbox.wsicore.wsireader.FsspecJsonWSIReader.is_valid_zarr_fsspec",
+            return_value=True,
+        ),
+        patch("tiatoolbox.wsicore.wsireader.FsspecJsonWSIReader") as mock_reader,
+    ):
+        WSIReader.open(input_path, mpp, power)
+        mock_reader.assert_called_once_with(input_path, mpp=mpp, power=power)
+
+
+def test_generate_fsspec_json_file_and_validate(
+    sample_svs: Path, tmp_path: Path
+) -> None:
+    """Test generate fsspec json file and validate it."""
+    file_types = ("*.svs",)
+
+    files_all = utils.misc.grab_files_from_dir(
+        input_path=Path(sample_svs).parent,
+        file_types=file_types,
+    )
+
+    svs_file_path = str(files_all[0])
+    json_file_path = str(tmp_path / "fsspec.json")
+    final_url = "https://example.com/some_id"
+
+    tiff_to_fsspec.main(svs_file_path, json_file_path, final_url)
+
+    assert Path(json_file_path).exists(), "Output JSON file was not created."
+
+    assert FsspecJsonWSIReader.is_valid_zarr_fsspec(json_file_path), (
+        "FSSPEC JSON file is invalid."
+    )
+
+
+def test_fsspec_wsireader_info_read(sample_svs: Path, tmp_path: Path) -> None:
+    """Test info read of the FsspecJsonWSIReader.
+
+    Generate fsspec json file and load image from:
+
+    https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+    info = wsi.info
+
+    assert info is not None, "info  should not be None."
+
+
+def test_read_bounds_fsspec_reader_baseline(sample_svs: Path, tmp_path: Path) -> None:
+    """Test FsspecJsonWSIReader read bounds at baseline.
+
+    Location coordinate is in baseline (level 0) reference frame.
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+
+    bounds = SVS_TEST_TISSUE_BOUNDS
+    size = SVS_TEST_TISSUE_SIZE
+    im_region = wsi.read_bounds(bounds, resolution=0, units="level")
+
+    assert isinstance(im_region, np.ndarray)
+    assert im_region.dtype == "uint8"
+    assert im_region.shape == (*size[::-1], 3)
+
+
+def test_read_rect_fsspec_reader_baseline(sample_svs: Path, tmp_path: Path) -> None:
+    """Test FsspecJsonWSIReader read rect at baseline.
+
+    Location coordinate is in baseline (level 0) reference frame.
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+
+    location = SVS_TEST_TISSUE_LOCATION
+    size = SVS_TEST_TISSUE_SIZE
+    im_region = wsi.read_rect(location, size, resolution=0, units="level")
+
+    assert isinstance(im_region, np.ndarray)
+    assert im_region.dtype == "uint8"
+    assert im_region.shape == (*size[::-1], 3)
+
+
+def test_fsspec_reader_open_invalid_json_file(tmp_path: Path) -> None:
+    """Ensure JSONDecodeError is handled properly.
+
+    Pass invalid JSON to  FsspecJsonWSIReader.is_valid_zarr_fsspec.
+    """
+    json_path = tmp_path / "invalid.json"
+    json_path.write_text("{invalid json}")  # Corrupt JSON
+
+    assert not FsspecJsonWSIReader.is_valid_zarr_fsspec(str(json_path))
+
+
+def test_fsspec_reader_open_oserror_handling() -> None:
+    """Ensure OSError is handled properly.
+
+    Pass non existent JSON to  FsspecJsonWSIReader.is_valid_zarr_fsspec.
+
+    """
+    with patch("builtins.open", side_effect=OSError("File not found")):
+        result = FsspecJsonWSIReader.is_valid_zarr_fsspec("non_existent.json")
+
+    assert result is False, "Function should return False for OSError"
+
+
+def test_fsspec_reader_open_pass_empty_json(tmp_path: Path) -> None:
+    """Ensure empty JSON is handled properly.
+
+    Pass empty JSON to FsspecJsonWSIReader.is_valid_zarr_fsspec and
+
+    verify that it's not valid.
+
+    """
+    json_path = tmp_path / "empty.json"
+    json_path.write_text("{}")
+
+    assert not FsspecJsonWSIReader.is_valid_zarr_fsspec(str(json_path))

tiatoolbox/annotation/storage.py

@@ -442,8 +442,8 @@ class AnnotationStore(ABC, MutableMapping[str, Annotation]):
 
     def __new__(
         cls: type[StoreInstanceType],
-        *args: str,  # noqa: ARG003
-        **kwargs: int,  # noqa: ARG003
+        *args: str,  # noqa: ARG004
+        **kwargs: int,  # noqa: ARG004
     ) -> StoreInstanceType:
         """Return an instance of a subclass of AnnotationStore."""
         if cls is AnnotationStore: