Merge branch 'develop' into add-registered-slide-vis

Author: shaneahmed

.github/workflows/python-package.yml

@@ -30,7 +30,7 @@ jobs:
         sudo apt update
         sudo apt-get install -y libopenslide-dev openslide-tools libopenjp2-7 libopenjp2-tools
         python -m pip install --upgrade pip
-        python -m pip install ruff==0.9.4 pytest pytest-cov pytest-runner
+        python -m pip install ruff==0.9.9 pytest pytest-cov pytest-runner
         pip install -r requirements/requirements.txt
     - name: Cache tiatoolbox static assets
       uses: actions/cache@v3

.pre-commit-config.yaml

@@ -60,7 +60,7 @@ repos:
       - id: rst-inline-touching-normal # Detect mistake of inline code touching normal text in rst.
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.9.4
+    rev: v0.9.9
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

requirements/requirements.txt

@@ -1,5 +1,6 @@
 # torch installation
 --extra-index-url https://download.pytorch.org/whl/cu118; sys_platform != "darwin"
+aiohttp>=3.8.1
 albumentations>=1.3.0
 bokeh>=3.1.1, <3.6.0
 Click>=8.1.3

requirements/requirements_dev.txt

@@ -10,7 +10,7 @@ pytest>=7.2.0
 pytest-cov>=4.0.0
 pytest-runner>=6.0
 pytest-xdist[psutil]
-ruff==0.9.4  # This will be updated by pre-commit bot to latest version
+ruff==0.9.9  # This will be updated by pre-commit bot to latest version
 toml>=0.10.2
 twine>=4.0.1
 wheel>=0.37.1

tests/models/test_arch_mapde.py

@@ -48,3 +48,12 @@ def test_functionality(remote_sample: Callable) -> None:
     output = model.infer_batch(model, batch, device=select_device(on_gpu=ON_GPU))
     output = model.postproc(output[0])
     assert np.all(output[0:2] == [[19, 171], [53, 89]])
+
+
+def test_multiclass_output() -> None:
+    """Test the architecture for multi-class output."""
+    multiclass_model = MapDe(num_input_channels=3, num_classes=3)
+    test_input = torch.rand((1, 3, 252, 252))
+
+    output = multiclass_model(test_input)
+    assert output.shape == (1, 3, 252, 252)

tests/test_wsireader.py

@@ -9,9 +9,8 @@
 import shutil
 from copy import deepcopy
 from pathlib import Path
-
-# When no longer supporting Python <3.9 this should be collections.abc.Iterable
 from typing import TYPE_CHECKING, Callable
+from unittest.mock import patch
 
 import cv2
 import glymur
@@ -27,7 +26,7 @@
 
 from tiatoolbox import cli, utils
 from tiatoolbox.annotation import SQLiteStore
-from tiatoolbox.utils import imread
+from tiatoolbox.utils import imread, tiff_to_fsspec
 from tiatoolbox.utils.exceptions import FileNotSupportedError
 from tiatoolbox.utils.magic import is_sqlite3
 from tiatoolbox.utils.transforms import imresize, locsize2bounds
@@ -37,6 +36,7 @@
     AnnotationStoreReader,
     ArrayView,
     DICOMWSIReader,
+    FsspecJsonWSIReader,
     JP2WSIReader,
     NGFFWSIReader,
     OpenSlideWSIReader,
@@ -221,6 +221,43 @@ def read_bounds_level_consistency(wsi: WSIReader, bounds: IntBounds) -> None:
 # Utility Test Classes & Functions
 # -------------------------------------------------------------------------------------
 
+_FSSPEC_WSI_CACHE = {}
+
+
+def fsspec_wsi(sample_svs: Path, tmp_path: Path) -> FsspecJsonWSIReader:
+    """Returns cached FsspecJsonWSIReader instance.
+
+    The reader instance opens CMU-1-Small-Region.svs image.
+
+    It's cached so the reader can be reused,
+
+    since loading the whole image using HTTP range requests from:
+
+    https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs
+
+    takes about 20 seconds.
+
+    """
+    cache_key = "sample_svs"
+
+    if cache_key in _FSSPEC_WSI_CACHE:
+        return _FSSPEC_WSI_CACHE[cache_key]  # Return cached instance
+
+    file_types = ("*.svs",)
+    files_all = utils.misc.grab_files_from_dir(
+        input_path=Path(sample_svs).parent,
+        file_types=file_types,
+    )
+    svs_file_path = str(files_all[0])
+    json_file_path = str(tmp_path / "fsspec.json")
+    final_url = (
+        "https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs"
+    )
+    tiff_to_fsspec.main(svs_file_path, json_file_path, final_url)
+
+    _FSSPEC_WSI_CACHE[cache_key] = wsireader.FsspecJsonWSIReader(json_file_path)
+    return _FSSPEC_WSI_CACHE[cache_key]
+
 
 class DummyMutableOpenSlideObject:
     """Dummy OpenSlide object with mutable properties."""
@@ -2812,3 +2849,133 @@ def test_read_multi_channel(source_image: Path) -> None:
     assert region.shape == (100, 50, (new_img_array.shape[-1]))
     assert np.abs(np.median(region.astype(int) - target.astype(int))) == 0
     assert np.abs(np.mean(region.astype(int) - target.astype(int))) < 0.2
+
+
+def test_fsspec_json_wsi_reader_instantiation() -> None:
+    """Test if FsspecJsonWSIReader is instantiated.
+
+    In case json is passed to  WSIReader.open, FsspecJsonWSIReader
+    should be instantiated.
+    """
+    input_path = "mock_path.json"
+    mpp = None
+    power = None
+
+    with (
+        patch(
+            "tiatoolbox.wsicore.wsireader.FsspecJsonWSIReader.is_valid_zarr_fsspec",
+            return_value=True,
+        ),
+        patch("tiatoolbox.wsicore.wsireader.FsspecJsonWSIReader") as mock_reader,
+    ):
+        WSIReader.open(input_path, mpp, power)
+        mock_reader.assert_called_once_with(input_path, mpp=mpp, power=power)
+
+
+def test_generate_fsspec_json_file_and_validate(
+    sample_svs: Path, tmp_path: Path
+) -> None:
+    """Test generate fsspec json file and validate it."""
+    file_types = ("*.svs",)
+
+    files_all = utils.misc.grab_files_from_dir(
+        input_path=Path(sample_svs).parent,
+        file_types=file_types,
+    )
+
+    svs_file_path = str(files_all[0])
+    json_file_path = str(tmp_path / "fsspec.json")
+    final_url = "https://example.com/some_id"
+
+    tiff_to_fsspec.main(svs_file_path, json_file_path, final_url)
+
+    assert Path(json_file_path).exists(), "Output JSON file was not created."
+
+    assert FsspecJsonWSIReader.is_valid_zarr_fsspec(json_file_path), (
+        "FSSPEC JSON file is invalid."
+    )
+
+
+def test_fsspec_wsireader_info_read(sample_svs: Path, tmp_path: Path) -> None:
+    """Test info read of the FsspecJsonWSIReader.
+
+    Generate fsspec json file and load image from:
+
+    https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+    info = wsi.info
+
+    assert info is not None, "info  should not be None."
+
+
+def test_read_bounds_fsspec_reader_baseline(sample_svs: Path, tmp_path: Path) -> None:
+    """Test FsspecJsonWSIReader read bounds at baseline.
+
+    Location coordinate is in baseline (level 0) reference frame.
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+
+    bounds = SVS_TEST_TISSUE_BOUNDS
+    size = SVS_TEST_TISSUE_SIZE
+    im_region = wsi.read_bounds(bounds, resolution=0, units="level")
+
+    assert isinstance(im_region, np.ndarray)
+    assert im_region.dtype == "uint8"
+    assert im_region.shape == (*size[::-1], 3)
+
+
+def test_read_rect_fsspec_reader_baseline(sample_svs: Path, tmp_path: Path) -> None:
+    """Test FsspecJsonWSIReader read rect at baseline.
+
+    Location coordinate is in baseline (level 0) reference frame.
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+
+    location = SVS_TEST_TISSUE_LOCATION
+    size = SVS_TEST_TISSUE_SIZE
+    im_region = wsi.read_rect(location, size, resolution=0, units="level")
+
+    assert isinstance(im_region, np.ndarray)
+    assert im_region.dtype == "uint8"
+    assert im_region.shape == (*size[::-1], 3)
+
+
+def test_fsspec_reader_open_invalid_json_file(tmp_path: Path) -> None:
+    """Ensure JSONDecodeError is handled properly.
+
+    Pass invalid JSON to  FsspecJsonWSIReader.is_valid_zarr_fsspec.
+    """
+    json_path = tmp_path / "invalid.json"
+    json_path.write_text("{invalid json}")  # Corrupt JSON
+
+    assert not FsspecJsonWSIReader.is_valid_zarr_fsspec(str(json_path))
+
+
+def test_fsspec_reader_open_oserror_handling() -> None:
+    """Ensure OSError is handled properly.
+
+    Pass non existent JSON to  FsspecJsonWSIReader.is_valid_zarr_fsspec.
+
+    """
+    with patch("builtins.open", side_effect=OSError("File not found")):
+        result = FsspecJsonWSIReader.is_valid_zarr_fsspec("non_existent.json")
+
+    assert result is False, "Function should return False for OSError"
+
+
+def test_fsspec_reader_open_pass_empty_json(tmp_path: Path) -> None:
+    """Ensure empty JSON is handled properly.
+
+    Pass empty JSON to FsspecJsonWSIReader.is_valid_zarr_fsspec and
+
+    verify that it's not valid.
+
+    """
+    json_path = tmp_path / "empty.json"
+    json_path.write_text("{}")
+
+    assert not FsspecJsonWSIReader.is_valid_zarr_fsspec(str(json_path))

tiatoolbox/annotation/storage.py

@@ -442,8 +442,8 @@ class AnnotationStore(ABC, MutableMapping[str, Annotation]):
 
     def __new__(
         cls: type[StoreInstanceType],
-        *args: str,  # noqa: ARG003
-        **kwargs: int,  # noqa: ARG003
+        *args: str,  # noqa: ARG004
+        **kwargs: int,  # noqa: ARG004
     ) -> StoreInstanceType:
         """Return an instance of a subclass of AnnotationStore."""
         if cls is AnnotationStore:

tiatoolbox/models/architecture/mapde.py

@@ -199,8 +199,11 @@ def __init__(
             dtype=np.float32,
         )
 
-        dist_filter = np.expand_dims(dist_filter, axis=(0, 1))  # NCHW
+        # For conv2d, filter shape = (out_channels, in_channels//groups, H, W)
+        dist_filter = np.expand_dims(dist_filter, axis=(0, 1))
         dist_filter = np.repeat(dist_filter, repeats=num_classes * 2, axis=1)
+        # Need to repeat for out_channels
+        dist_filter = np.repeat(dist_filter, repeats=num_classes, axis=0)
 
         self.min_distance = min_distance
         self.threshold_abs = threshold_abs

tiatoolbox/utils/tiff_to_fsspec.py

@@ -0,0 +1,110 @@
+"""Module for processing SVS metadata and generating fsspec zarr JSON file.
+
+The fsspec zarr json file is meant to be used in case SVS or TIFF files
+can be accessed using byte range HTTP API.
+
+The fsspec zarr json file can be opened using FsspecJsonWSIReader.
+
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from tifffile import TiffFile, tiff2fsspec
+
+from tiatoolbox.wsicore.wsireader import TIFFWSIReaderDelegate
+
+# Constants
+EXPECTED_KEY_VALUE_PAIRS = 2
+EXPECTED_ARG_COUNT = 4
+URL_PLACEHOLDER = "https://replace.me/"
+
+
+def convert_metadata(metadata: dict) -> dict:
+    """Convert metadata to JSON-compatible format."""
+    if isinstance(metadata, dict):
+        return {key: convert_metadata(value) for key, value in metadata.items()}
+    if isinstance(metadata, list):
+        return [convert_metadata(item) for item in metadata]
+    if isinstance(metadata, datetime):
+        return metadata.isoformat()  # Convert datetime to ISO 8601 string
+    return metadata
+
+
+def replace_url(
+    data: dict[str, Any], output_path: Path, old_url: str, new_url: str
+) -> None:
+    """Replace URL in the JSON file."""
+    for value in data.values():
+        if isinstance(value, list) and value[0] == old_url:
+            value[0] = new_url
+
+    with output_path.open("w") as json_file:
+        json.dump(data, json_file, indent=2)
+
+
+def main(svs_file_path: str, json_file_path: str, final_url: str) -> None:
+    """Main function to process an SVS file.
+
+    Args:
+        svs_file_path (str): The local file path of the SVS file to be processed.
+        json_file_path (str): The file path where the output JSON will be saved.
+        final_url (str): The URL where the SVS file is stored online
+        and can be accessed via HTTP byte range API.
+
+    Example:
+        main('/path/to/CMU-1-Small-Region.svs', '/path/to/CMU-1-Small-Region.json', 'https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs')
+
+    """
+    url_to_replace = f"{URL_PLACEHOLDER}{Path(svs_file_path).name}"
+
+    tiff = TiffFile(svs_file_path)
+
+    tiff_file_pages = tiff.pages
+
+    # Generate fsspec JSON
+    tiff2fsspec(svs_file_path, url=URL_PLACEHOLDER, out=json_file_path)
+
+    if tiff.is_svs:
+        metadata = TIFFWSIReaderDelegate.parse_svs_metadata(tiff_file_pages)
+    else:  # pragma: no cover
+        metadata = TIFFWSIReaderDelegate.parse_generic_tiff_metadata(tiff_file_pages)
+
+    # Convert metadata to JSON-compatible format
+    metadata_serializable = convert_metadata(metadata)
+
+    # Read the JSON data from the file
+    json_path = Path(json_file_path)
+    with json_path.open() as file:
+        json_data = json.load(file)
+
+    # Decode `.zattrs` JSON string into a dictionary
+    zattrs = json.loads(json_data[".zattrs"])
+
+    # Ensure "multiscales" exists and is a list
+    if "multiscales" not in zattrs or not isinstance(
+        zattrs["multiscales"], list
+    ):  # pragma: no cover
+        zattrs["multiscales"] = [{}]  # Initialize as a list with an empty dictionary
+
+    # Update metadata into `.zattrs`
+    zattrs["multiscales"][0]["metadata"] = metadata_serializable
+
+    # Convert back to a JSON string
+    json_data[".zattrs"] = json.dumps(zattrs)
+
+    # Replace URLs in the JSON file
+    replace_url(json_data, json_path, url_to_replace, final_url)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != EXPECTED_ARG_COUNT:
+        msg = " Usage: python script.py <svs_file_path> <json_file_path> <final_url>"
+        raise ValueError(msg)
+
+    main(sys.argv[1], sys.argv[2], sys.argv[3])