Added function to download Visium as SpatialData (#949)

* Added function to download example SpatialData

* self-review

* added test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* updated dataset id

* fixed tox.ini

* attempt to include sdata download

* added func import to _datasets so __all__ can see it

* attempt to hack in sdata download 2

* debug output for runner

* adjusted function

* refactored out download/extract code

* updated failing image test

* made photometric parameter robust to newest tifffile release

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: timtreis

.scripts/ci/download_data.py

@@ -5,6 +5,8 @@
 from pathlib import Path
 from typing import Any
 
+from squidpy.datasets import visium_hne_sdata
+
 _CNT = 0  # increment this when you want to rebuild the CI cache
 _ROOT = Path.home() / ".cache" / "squidpy"
 
@@ -39,14 +41,25 @@ def main(args: argparse.Namespace) -> None:
 
     if args.dry_run:
         for func_name, ext in zip(all_datasets, all_extensions):
+            if func_name == "visium_hne_sdata":
+                ext = "zarr"
             path = _ROOT / f"{func_name}.{ext}"
             _print_message(func_name, path, dry_run=True)
         return
 
     # could be parallelized, but on CI it largely does not matter (usually limited to 2 cores + bandwidth limit)
     for func_name, ext in zip(all_datasets, all_extensions):
-        path = _ROOT / f"{func_name}.{ext}"
+        if func_name == "visium_hne_sdata":
+            ext = "zarr"
+            path = _ROOT / f"{func_name}.{ext}"
 
+            _print_message(func_name, path)
+            obj = visium_hne_sdata(_ROOT)
+
+            assert path.is_dir(), f"Expected a .zarr folder at {path}"
+            continue
+
+        path = _ROOT / f"{func_name}.{ext}"
         _print_message(func_name, path)
         obj = _maybe_download_data(func_name, path)
 

src/squidpy/datasets/_10x_datasets.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import os
+import shutil
 import tarfile
 from pathlib import Path
 from typing import (
@@ -8,12 +10,16 @@
     Union,  # noqa: F401
 )
 
+import spatialdata as sd
 from anndata import AnnData
 from scanpy import _utils
+from scanpy import logging as logg
 from scanpy._settings import settings
+from scanpy._utils import check_presence_download
+from spatialdata import SpatialData
 
 from squidpy._constants._constants import TenxVersions
-from squidpy.datasets._utils import PathLike
+from squidpy.datasets._utils import DEFAULT_CACHE_DIR, PathLike, _get_zipped_dataset
 
 __all__ = ["visium"]
 
@@ -106,7 +112,9 @@ def visium(
 
     url_prefix = f"https://cf.10xgenomics.com/samples/spatial-exp/{spaceranger_version}/{sample_id}/"
     visium_files = VisiumFiles(
-        f"{sample_id}_filtered_feature_bc_matrix.h5", f"{sample_id}_spatial.tar.gz", f"{sample_id}_image.tif"
+        f"{sample_id}_filtered_feature_bc_matrix.h5",
+        f"{sample_id}_spatial.tar.gz",
+        f"{sample_id}_image.tif",
     )
 
     # download spatial data
@@ -134,3 +142,38 @@ def visium(
         )
 
     return read_visium(base_dir / sample_id)
+
+
+def visium_hne_sdata(folderpath: Path | str | None = None) -> sd.SpatialData:
+    """
+    Downloads a Visium H&E dataset into a specified folder and returns it as a `SpatialData` object.
+
+    It downloads and extracts the dataset into:
+      - `<folderpath>/visium_hne_sdata.zip` for the compressed file
+      - `<folderpath>/visium_hne_sdata.zarr` for the extracted dataset
+
+    Parameters
+    ----------
+    folderpath : Path | str
+        A folder path where the dataset will be downloaded and extracted. The resulting `.zarr`
+        folder is used to load the `SpatialData` object.
+
+    Returns
+    -------
+    SpatialData
+        The downloaded and extracted Visium H&E dataset as a `SpatialData` object.
+    """
+
+    FIGSHARE_ID = "52370645"
+    DATASET_NAME = "visium_hne_sdata"
+
+    if folderpath is None:
+        folderpath = DEFAULT_CACHE_DIR
+    else:
+        folderpath = Path(folderpath).expanduser().absolute()
+
+    return _get_zipped_dataset(
+        folderpath=folderpath,
+        dataset_name=DATASET_NAME,
+        figshare_id=FIGSHARE_ID,
+    )

src/squidpy/datasets/__init__.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
 
-from squidpy.datasets._10x_datasets import visium
+from squidpy.datasets._10x_datasets import visium, visium_hne_sdata
 from squidpy.datasets._dataset import *  # noqa: F403
 from squidpy.datasets._image import *  # noqa: F403

src/squidpy/datasets/_dataset.py

@@ -2,6 +2,7 @@
 
 from copy import copy
 
+from squidpy.datasets._10x_datasets import visium_hne_sdata
 from squidpy.datasets._utils import AMetadata
 
 _4i = AMetadata(
@@ -90,6 +91,7 @@
     "seqfish",
     "visium_hne_adata",
     "visium_hne_adata_crop",
+    "visium_hne_sdata",
     "visium_fluo_adata",
     "visium_fluo_adata_crop",
     "sc_mouse_cortex",

src/squidpy/datasets/_utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import shutil
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Sequence
 from dataclasses import dataclass, field
@@ -9,13 +10,15 @@
 from typing import Any, TypeAlias, Union
 
 import anndata
+import spatialdata as sd
 from anndata import AnnData
 from scanpy import logging as logg
 from scanpy import read
 from scanpy._utils import check_presence_download
 
 PathLike: TypeAlias = os.PathLike[str] | str
 Function_t: TypeAlias = Callable[..., AnnData | Any]
+DEFAULT_CACHE_DIR = Path.home() / ".cache" / "squidpy"
 
 
 @dataclass(frozen=True)
@@ -177,3 +180,42 @@ def _download(self, fpath: PathLike, backup_url: str, **kwargs: Any) -> Any:
     @property
     def _extension(self) -> str:
         return ".tiff"
+
+
+def _get_zipped_dataset(folderpath: Path, dataset_name: str, figshare_id: str) -> sd.SpatialData:
+    """Returns a specific dataset as SpatialData object. If the file is not present on disk, it will be downloaded and extracted."""
+
+    if not folderpath.is_dir():
+        raise ValueError(f"Expected a directory path for `folderpath`, found: {folderpath}")
+
+    download_zip = folderpath / f"{dataset_name}.zip"
+    extracted_path = folderpath / f"{dataset_name}.zarr"
+
+    # Return early if data is already extracted
+    if extracted_path.exists():
+        logg.info(f"Loading existing dataset from {extracted_path}")
+        return sd.read_zarr(extracted_path)
+
+    # Download if necessary
+    if not download_zip.exists():
+        logg.info(f"Downloading Visium H&E SpatialData to {download_zip}")
+        try:
+            check_presence_download(
+                filename=download_zip,
+                backup_url=f"https://ndownloader.figshare.com/files/{figshare_id}",
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to download dataset: {e}") from e
+
+    # Extract if necessary
+    if not extracted_path.exists():
+        logg.info(f"Extracting dataset from {download_zip} to {extracted_path}")
+        try:
+            shutil.unpack_archive(str(download_zip), folderpath)
+        except Exception as e:
+            raise RuntimeError(f"Failed to extract dataset: {e}") from e
+
+    if not extracted_path.exists():
+        raise RuntimeError(f"Expected extracted data at {extracted_path}, but not found")
+
+    return sd.read_zarr(extracted_path)

tests/datasets/test_download_visium_dataset.py

@@ -8,16 +8,22 @@
 from pathlib import Path
 
 import pytest
+import spatialdata as sd
 from anndata.tests.helpers import assert_adata_equal
 from scanpy._settings import settings
 
-from squidpy.datasets import visium
+from squidpy.datasets import visium, visium_hne_sdata
 
 
 @pytest.mark.timeout(120)
 @pytest.mark.internet()
 @pytest.mark.parametrize(
-    "sample", ["V1_Mouse_Kidney", "Targeted_Visium_Human_SpinalCord_Neuroscience", "Visium_FFPE_Human_Breast_Cancer"]
+    "sample",
+    [
+        "V1_Mouse_Kidney",
+        "Targeted_Visium_Human_SpinalCord_Neuroscience",
+        "Visium_FFPE_Human_Breast_Cancer",
+    ],
 )
 def test_visium_datasets(tmpdir, sample):
     # Tests that reading / downloading datasets works and it does not have any global effects
@@ -43,3 +49,12 @@ def test_visium_datasets(tmpdir, sample):
     process = subprocess.run(["file", "--mime-type", image_path], stdout=subprocess.PIPE)
     output = process.stdout.strip().decode()  # make process output string
     assert output == str(image_path) + ": image/tiff"
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.internet()
+def test_visium_sdata_dataset(tmpdir):
+    sdata = visium_hne_sdata(Path(tmpdir))
+    assert isinstance(sdata, sd.SpatialData)
+    assert list(sdata.shapes.keys()) == ["spots"]
+    assert list(sdata.images.keys()) == ["hne"]

tests/image/test_io.py

@@ -17,8 +17,15 @@ class TestIO:
     def _create_image(path: str, shape: tuple[int, ...]):
         dtype = np.uint8 if len(shape) <= 3 else np.float32
         img = np.random.randint(0, 255, size=shape).astype(dtype)
-        # set `photometric` to remove warnings
-        tifffile.imwrite(path, img, photometric=tifffile.TIFF.PHOTOMETRIC.MINISBLACK)
+
+        try:
+            # Old usage (works up to tifffile<2025.2.18)
+            photometric = tifffile.TIFF.PHOTOMETRIC.MINISBLACK
+        except AttributeError:
+            # New usage (works on tifffile >=2025.2.18)
+            photometric = "MINISBLACK"
+
+        tifffile.imwrite(path, img, photometric=photometric)
 
         return img