Merge pull request #210 from siapy/develop

Author: janezlapajne

Makefile

@@ -58,6 +58,10 @@ generate-docs:
 
 .PHONY: serve-docs  ## Serve the docs
 serve-docs:
+	pdm run mkdocs serve
+
+.PHONY: serve-docs-mike  ## Serve the docs using mike
+serve-docs-mike:
 	pdm run mike serve
 
 .PHONY: version  ## Check project version

docs/api/entities/helpers.md

@@ -0,0 +1 @@
+::: siapy.entities.helpers

docs/api/entities/images/mock.md

@@ -0,0 +1 @@
+::: siapy.entities.images.mock

docs/api/models/metrics.md

@@ -0,0 +1 @@
+::: siapy.optimizers.metrics

docs/api/optimizers/metrics.md

@@ -87,20 +87,21 @@ nav:
               - Interfaces: api/entities/images/interfaces.md
               - Rasterio Library: api/entities/images/rasterio_lib.md
               - Spectral Library: api/entities/images/spectral_lib.md
+              - Mock Image: api/entities/images/mock.md
               - Spectral Images: api/entities/images/spimage.md
           - Shape: api/entities/shapes/shape.md
           - Image Sets: api/entities/imagesets.md
           - Pixels: api/entities/pixels.md
           - Signatures: api/entities/signatures.md
+          - Helpers: api/entities/helpers.md
       - Features:
           - Features: api/features/features.md
           - Helpers: api/features/helpers.md
           - Spectral Indices: api/features/spectral_indices.md
-      - Models:
-          - Metrics: api/models/metrics.md
       - Optimizers:
           - Configs: api/optimizers/configs.md
           - Evaluators: api/optimizers/evaluators.md
+          - Metrics: api/optimizers/metrics.md
           - Optimizers: api/optimizers/optimizers.md
           - Parameters: api/optimizers/parameters.md
           - Scorers: api/optimizers/scorers.md

mkdocs.yml

@@ -49,4 +49,5 @@ def generate_regression_target(
 
 
 def merge_signals_from_multiple_cameras(data: "TabularDatasetData") -> None:
-    data.signals.copy()
+    # TODO: Implement the function to merge signals from multiple cameras
+    pass

pdm.lock

@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-from typing import Any, Iterable
+from dataclasses import dataclass
+from typing import Any, Iterable, Optional
 
 import pandas as pd
 from pydantic import BaseModel, ConfigDict
 
 from siapy.core.exceptions import InvalidInputError
+from siapy.entities import Signatures
 
 from .helpers import generate_classification_target, generate_regression_target
 
@@ -88,7 +90,7 @@ def reset_index(self) -> "ClassificationTarget":
 
 class RegressionTarget(Target):
     value: pd.Series
-    name: str
+    name: str = "value"
 
     def __getitem__(self, indices: Any) -> "RegressionTarget":
         value = self.value.iloc[indices]
@@ -105,7 +107,7 @@ def from_iterable(cls, data: Iterable[Any]) -> "RegressionTarget":
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "RegressionTarget":
         value = pd.Series(data["value"], name="value")
-        name = data["name"]
+        name = data["name"] if "name" in data else "value"
         return cls(value=value, name=name)
 
     def to_dict(self) -> dict[str, Any]:
@@ -121,42 +123,38 @@ def reset_index(self) -> "RegressionTarget":
         return RegressionTarget(value=self.value.reset_index(drop=True), name=self.name)
 
 
-class TabularDatasetData(BaseModel):
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-    pixels: pd.DataFrame
-    signals: pd.DataFrame
+@dataclass
+class TabularDatasetData:
+    signatures: Signatures
     metadata: pd.DataFrame
     target: Target | None = None
 
-    def __init__(self, *args: Any, **kwargs: Any):
-        super().__init__(*args, **kwargs)
-        self._validate_lengths()
+    def __len__(self) -> int:
+        return len(self.signatures)
 
-    def __setattr__(self, name: str, value: Any) -> None:
-        super().__setattr__(name, value)
-        if name in self.model_fields.keys():
-            self._validate_lengths()
+    def __repr__(self) -> str:
+        return f"TabularDatasetData(signatures={self.signatures}, metadata={self.metadata}, target={self.target})"
 
     def __getitem__(self, indices: Any) -> "TabularDatasetData":
-        pixels = self.pixels.iloc[indices]
-        signals = self.signals.iloc[indices]
+        signatures = self.signatures[indices]
         metadata = self.metadata.iloc[indices]
+        if isinstance(metadata, pd.Series):
+            metadata = pd.DataFrame(metadata).T
         target = None if self.target is None else self.target.__getitem__(indices)
-        return TabularDatasetData(pixels=pixels, signals=signals, metadata=metadata, target=target)
+        return TabularDatasetData(signatures=signatures, metadata=metadata, target=target)
 
-    def __len__(self) -> int:
-        return len(self.pixels)
+    def __post_init__(self) -> None:
+        self._validate_lengths()
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "TabularDatasetData":
-        pixels = pd.DataFrame(data["pixels"])
-        signals = pd.DataFrame(data["signals"])
+        signatures = Signatures.from_dict({"pixels": data["pixels"], "signals": data["signals"]})
         metadata = pd.DataFrame(data["metadata"])
         target = TabularDatasetData.target_from_dict(data.get("target", None))
-        return cls(pixels=pixels, signals=signals, metadata=metadata, target=target)
+        return cls(signatures=signatures, metadata=metadata, target=target)
 
     @staticmethod
-    def target_from_dict(data: dict[str, Any] | None) -> Target | None:
+    def target_from_dict(data: dict[str, Any] | None = None) -> Optional[Target]:
         if data is None:
             return None
 
@@ -172,14 +170,13 @@ def target_from_dict(data: dict[str, Any] | None) -> Target | None:
             raise InvalidInputError(data, "Invalid target dict.")
 
     def _validate_lengths(self) -> None:
-        if not (len(self.pixels) == len(self.signals) == len(self.metadata)):
+        if len(self.signatures) != len(self.metadata):
             raise InvalidInputError(
                 {
-                    "pixels_length": len(self.pixels),
-                    "signals_length": len(self.signals),
+                    "signatures_length": len(self.signatures),
                     "metadata_length": len(self.metadata),
                 },
-                "Lengths of pixels, signals, and metadata must be equal",
+                "Lengths of signatures and metadata must be equal",
             )
         if self.target is not None and len(self.target) != len(self):
             raise InvalidInputError(
@@ -190,25 +187,77 @@ def _validate_lengths(self) -> None:
                 "Target length must be equal to the length of the dataset.",
             )
 
+    def set_attributes(
+        self,
+        *,
+        signatures: Signatures | None = None,
+        metadata: pd.DataFrame | None = None,
+        target: Target | None = None,
+    ) -> "TabularDatasetData":
+        current_data = self.copy()
+        signatures = signatures if signatures is not None else current_data.signatures
+        metadata = metadata if metadata is not None else current_data.metadata
+        target = target if target is not None else current_data.target
+        return TabularDatasetData(signatures=signatures, metadata=metadata, target=target)
+
     def to_dict(self) -> dict[str, Any]:
+        signatures_dict = self.signatures.to_dict()
         return {
-            "pixels": self.pixels.to_dict(),
-            "signals": self.signals.to_dict(),
+            "pixels": signatures_dict["pixels"],
+            "signals": signatures_dict["signals"],
             "metadata": self.metadata.to_dict(),
             "target": self.target.to_dict() if self.target is not None else None,
         }
 
     def to_dataframe(self) -> pd.DataFrame:
-        combined_df = pd.concat([self.pixels, self.signals, self.metadata], axis=1)
+        combined_df = pd.concat([self.signatures.to_dataframe(), self.metadata], axis=1)
         if self.target is not None:
             target_series = self.target.to_dataframe()
             combined_df = pd.concat([combined_df, target_series], axis=1)
         return combined_df
 
+    def to_dataframe_multiindex(self) -> pd.DataFrame:
+        signatures_df = self.signatures.to_dataframe_multiindex()
+
+        metadata_columns = pd.MultiIndex.from_tuples(
+            [("metadata", col) for col in self.metadata.columns], names=["category", "field"]
+        )
+        metadata_df = pd.DataFrame(self.metadata.values, columns=metadata_columns)
+
+        combined_df = pd.concat([signatures_df, metadata_df], axis=1)
+
+        if self.target is not None:
+            target_df = self.target.to_dataframe()
+            if isinstance(self.target, ClassificationTarget):
+                target_columns = pd.MultiIndex.from_tuples(
+                    [("target", col) for col in target_df.columns],
+                    names=["category", "field"],
+                )
+            elif isinstance(self.target, RegressionTarget):
+                target_columns = pd.MultiIndex.from_tuples(
+                    [("target", self.target.name)],
+                    names=["category", "field"],
+                )
+            else:
+                raise InvalidInputError(
+                    self.target,
+                    "Invalid target type. Expected ClassificationTarget or RegressionTarget.",
+                )
+            target_df = pd.DataFrame(target_df.values, columns=target_columns)
+            combined_df = pd.concat([combined_df, target_df], axis=1)
+
+        return combined_df
+
     def reset_index(self) -> "TabularDatasetData":
         return TabularDatasetData(
-            pixels=self.pixels.reset_index(drop=True),
-            signals=self.signals.reset_index(drop=True),
+            signatures=self.signatures.reset_index(),
             metadata=self.metadata.reset_index(drop=True),
             target=self.target.reset_index() if self.target is not None else None,
         )
+
+    def copy(self) -> "TabularDatasetData":
+        return TabularDatasetData(
+            signatures=self.signatures.copy(),
+            metadata=self.metadata.copy(),
+            target=self.target.model_copy() if self.target is not None else None,
+        )

siapy/datasets/helpers.py

@@ -9,6 +9,7 @@
 from siapy.core.types import ImageContainerType
 from siapy.datasets.schemas import TabularDatasetData
 from siapy.entities import Signatures, SpectralImage, SpectralImageSet
+from siapy.entities.helpers import get_signatures_within_convex_hull
 
 __all__ = [
     "TabularDataset",
@@ -62,9 +63,8 @@ def process_image_data(self) -> None:
         self.data_entities.clear()
         for image_idx, image in enumerate(self.image_set):
             for shape_idx, shape in enumerate(image.geometric_shapes.shapes):
-                convex_hulls = shape.get_pixels_within_convex_hull()
-                for geometry_idx, pixels in enumerate(convex_hulls):
-                    signatures = image.to_signatures(pixels)
+                signatures_hull = get_signatures_within_convex_hull(image, shape)
+                for geometry_idx, signatures in enumerate(signatures_hull):
                     entity = TabularDataEntity(
                         image_idx=image_idx,
                         shape_idx=shape_idx,
@@ -79,8 +79,7 @@ def process_image_data(self) -> None:
 
     def generate_dataset_data(self, mean_signatures: bool = True) -> TabularDatasetData:
         self._check_data_entities()
-        pixels_dfs = []
-        signals_dfs = []
+        signatures_dfs = []
         metadata_dfs = []
         for entity in self.data_entities:
             signatures_df = entity.signatures.to_dataframe().dropna()
@@ -104,17 +103,13 @@ def generate_dataset_data(self, mean_signatures: bool = True) -> TabularDatasetD
                 "Sanity check failed! The columns in metadata_df do not match MetaDataEntity fields."
             )
 
-            signatures = Signatures.from_dataframe(signatures_df)
-
-            pixels_dfs.append(signatures.pixels.df)
-            signals_dfs.append(signatures.signals.df)
+            signatures_dfs.append(signatures_df)
             metadata_dfs.append(metadata_df)
 
-        return TabularDatasetData(
-            pixels=pd.concat(pixels_dfs, ignore_index=True),
-            signals=pd.concat(signals_dfs, ignore_index=True),
-            metadata=pd.concat(metadata_dfs, ignore_index=True),
-        )
+        signatures_concat = pd.concat(signatures_dfs, ignore_index=True)
+        metadata_concat = pd.concat(metadata_dfs, ignore_index=True)
+        signatures = Signatures.from_dataframe(signatures_concat)
+        return TabularDatasetData(signatures=signatures, metadata=metadata_concat)
 
     def _check_data_entities(self) -> None:
         if not self.data_entities: