Merge branch 'main' into pre-commit-ci-update-config

Author: sophiamaedler

src/scportrait/pipeline/extraction.py

@@ -22,6 +22,7 @@
 from scportrait.pipeline._base import ProcessingStep
 from scportrait.pipeline._utils.helper import flatten
 from scportrait.processing.images._image_processing import percentile_normalization
+from scportrait.tools.sdata.write._helper import _normalize_anndata_strings
 
 
 class HDF5CellExtraction(ProcessingStep):
@@ -753,6 +754,7 @@ def _initialize_empty_anndata(self) -> None:
         adata.uns[f"{self.DEFAULT_NAME_SINGLE_CELL_IMAGES}/compression"] = self.compression_type
 
         # write to file
+        _normalize_anndata_strings(adata)
         adata.write(self.output_path)
 
     def _create_output_files(self) -> None:

src/scportrait/pipeline/featurization.py

@@ -1582,15 +1582,15 @@ def calculate_statistics(self, img: torch.Tensor, n_masks: int = 2):
         for channel in range(n_masks, img.shape[1]):
             img_selected = img[:, channel]
 
-            for mask in masks:
+            for mask, area in zip(masks, mask_statistics, strict=True):
                 _img_selected = img_selected.masked_fill(~mask, torch.nan).to(torch.float32)
 
                 mean = _img_selected.view(N, -1).nanmean(1, keepdim=True)
                 median = _img_selected.view(N, -1).nanquantile(q=0.5, dim=1, keepdim=True)
                 quant75 = _img_selected.view(N, -1).nanquantile(q=0.75, dim=1, keepdim=True)
                 quant25 = _img_selected.view(N, -1).nanquantile(q=0.25, dim=1, keepdim=True)
-                summed_intensity = _img_selected.view(N, -1).sum(1, keepdim=True)
-                summed_intensity_area_normalized = summed_intensity / mask_statistics[-1]
+                summed_intensity = _img_selected.view(N, -1).nansum(1, keepdim=True)
+                summed_intensity_area_normalized = summed_intensity / area
 
                 # save results
                 channel_statistics.extend(

src/scportrait/pipeline/project.py

@@ -37,7 +37,7 @@
     get_chunk_size,
     rechunk_image,
 )
-from scportrait.tools.sdata.write._helper import _get_image, _get_shape
+from scportrait.tools.sdata.write._helper import _get_image, _get_shape, _normalize_anndata_strings
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -1473,6 +1473,7 @@ def load_input_from_sdata(
                 table.obs["region"] = new_name
                 table.obs["region"] = table.obs["region"].astype("category")
             table.obs.rename(columns=rename_columns, inplace=True)
+            _normalize_anndata_strings(table)
 
         if keep_all:
             shutil.rmtree(self.sdata_path, ignore_errors=True)

src/scportrait/tools/sdata/write/_helper.py

@@ -2,6 +2,9 @@
 from typing import Any, Literal, TypeAlias
 
 import numpy as np
+import pandas as pd
+from anndata import AnnData
+from pandas.api.types import is_string_dtype
 from spatialdata import SpatialData, read_zarr
 from spatialdata.models import get_model
 from xarray import DataArray, DataTree
@@ -68,6 +71,32 @@ def _make_key_lookup(sdata: SpatialData) -> dict:
     return dict_lookup
 
 
+def _normalize_dataframe_strings(df: pd.DataFrame) -> None:
+    """Normalize string dtypes to object to avoid nullable string serialization issues."""
+    if is_string_dtype(df.index.dtype):
+        df.index = df.index.astype(object)
+        if df.index.isna().any():
+            df.index = df.index.where(~df.index.isna(), None)
+
+    string_cols = df.select_dtypes(include=["string"]).columns
+    if len(string_cols) > 0:
+        df[string_cols] = df[string_cols].astype(object)
+        for col in string_cols:
+            if df[col].isna().any():
+                df[col] = df[col].where(df[col].notna(), None)
+
+    cat_cols = df.select_dtypes(include=["category"]).columns
+    for col in cat_cols:
+        if is_string_dtype(df[col].cat.categories.dtype):
+            df[col] = df[col].cat.set_categories(df[col].cat.categories.astype(object))
+
+
+def _normalize_anndata_strings(adata: AnnData) -> None:
+    """Normalize obs/var string dtypes to python-backed storage."""
+    _normalize_dataframe_strings(adata.obs)
+    _normalize_dataframe_strings(adata.var)
+
+
 def _force_delete_object(sdata: SpatialData, name: str) -> None:
     """Force delete an object from the SpatialData object and directory.
 
@@ -110,6 +139,9 @@ def add_element_sdata(sdata: SpatialData, element: Any, element_name: str, overw
 
         _force_delete_object(sdata, element_name)
 
+    if isinstance(element, AnnData):
+        _normalize_anndata_strings(element)
+
     # the element needs to validate with exactly one of the models
     get_model(element)
 

tests/conftest.py

@@ -1,15 +1,30 @@
 import shutil
 
+import matplotlib
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import pytest
 from anndata import AnnData
+from matplotlib.figure import Figure
 from spatialdata import SpatialData
 from spatialdata.datasets import blobs
 
+from scportrait.tools.sdata.write._helper import _normalize_anndata_strings
+
 rng = np.random.default_rng()
 
 
+@pytest.fixture(autouse=True)
+def _disable_matplotlib_show(monkeypatch):
+    # Force a non-interactive backend for e2e runs
+    matplotlib.use("Agg", force=True)
+
+    # Disable any implicit rendering during tests
+    monkeypatch.setattr(plt, "show", lambda *args, **kwargs: None)
+    monkeypatch.setattr(Figure, "show", lambda *args, **kwargs: None)
+
+
 @pytest.fixture
 def h5sc_object() -> AnnData:
     # Two cells, two channels, small images
@@ -47,6 +62,7 @@ def h5sc_object() -> AnnData:
 @pytest.fixture()
 def sdata(tmp_path) -> SpatialData:
     sdata = blobs()
+    _normalize_anndata_strings(sdata["table"])
     # Write to temporary location
     sdata_path = tmp_path / "sdata.zarr"
     sdata.write(sdata_path)
@@ -57,6 +73,7 @@ def sdata(tmp_path) -> SpatialData:
 @pytest.fixture
 def sdata_with_labels() -> SpatialData:
     sdata = blobs()
+    _normalize_anndata_strings(sdata["table"])
     sdata["table"].obs["labelling_categorical"] = sdata["table"].obs["instance_id"].astype("category")
     sdata["table"].obs["labelling_continous"] = (sdata["table"].obs["instance_id"] > 10).astype(float)
     return sdata

tests/e2e_tests/test_data_loaders.py

@@ -8,11 +8,13 @@
 
 from scportrait.data._datasets import dataset_1_omezarr
 from scportrait.pipeline.project import Project
+from scportrait.tools.sdata.write._helper import _normalize_anndata_strings
 
 
 @pytest.fixture()
 def sdata_path(tmp_path):
     sdata = blobs()
+    _normalize_anndata_strings(sdata["table"])
     # Write to temporary location
     sdata_path = tmp_path / "sdata.zarr"
     sdata.write(sdata_path)

tests/unit_tests/pipeline/test_featurization.py

@@ -1,3 +1,4 @@
+import numpy as np
 import pytest
 import torch
 
@@ -51,6 +52,21 @@ def test_cell_featurizer(tmp_path):
     assert nuc_mean > cyto_mean
     assert abs(nuc_mean - cyto_mean) > 1.0
 
+    # Ensure masks are applied and sums are finite (no NaNs from masked sum).
+    assert not np.isnan(feat_map["ch0_summed_intensity_nucleus"])
+    assert not np.isnan(feat_map["ch0_summed_intensity_cytosol"])
+    assert not np.isnan(feat_map["ch0_summed_intensity_cytosol_only"])
+
+    # Check exact expected values for this synthetic setup.
+    assert feat_map["ch0_summed_intensity_nucleus"] == pytest.approx(10.0)
+    assert feat_map["ch0_summed_intensity_cytosol"] == pytest.approx(13.0)
+    assert feat_map["ch0_summed_intensity_cytosol_only"] == pytest.approx(3.0)
+
+    # Area-normalized sums should use each mask's own area (not the last mask).
+    assert feat_map["ch0_summed_intensity_area_normalized_nucleus"] == pytest.approx(10.0)
+    assert feat_map["ch0_summed_intensity_area_normalized_cytosol"] == pytest.approx(13.0 / 4.0)
+    assert feat_map["ch0_summed_intensity_area_normalized_cytosol_only"] == pytest.approx(1.0)
+
 
 def test_mask_bool_integrity_not_corrupted():
     mask = torch.tensor([[True, False], [False, True]])

tests/unit_tests/tools/sdata/test_write.py

@@ -1,8 +1,10 @@
 import shutil
 
 import numpy as np
+import pandas as pd
 import pytest
 import scportrait.tl.sdata.write as write
+from anndata import AnnData
 from spatialdata import SpatialData, read_zarr
 from spatialdata.datasets import blobs
 
@@ -59,6 +61,31 @@ def test_add_element_sdata(sdata, sdata_path, element_name):
     assert element_name in sdata
 
 
+def _make_table_from_dataframe() -> AnnData:
+    obs = pd.DataFrame(
+        {
+            "region": pd.Series(["1", "2", "3"], dtype="string"),
+            "group": pd.Series(["a", "b", "a"], dtype="string").astype("category"),
+        },
+        index=pd.Index(["1", "2", "3"], dtype="string"),
+    )
+    var = pd.DataFrame(index=pd.Index(["feature_0"], dtype="string"))
+    X = np.zeros((3, 1), dtype=np.float32)
+    return AnnData(X=X, obs=obs, var=var)
+
+
+def test_add_element_sdata_table_from_dataframe(sdata_path):
+    sdata_new = SpatialData()
+    sdata_new.write(sdata_path)
+
+    table = _make_table_from_dataframe()
+    write._helper.add_element_sdata(sdata_new, table, "table", overwrite=True)
+
+    assert "table" in sdata_new
+    sdata = read_zarr(sdata_path)
+    assert "table" in sdata
+
+
 ### test scportrait.tools.sdata.write._helper.rename_image_element
 @pytest.mark.parametrize(
     "old_name, new_name",