core\refac: #68 init tfds for crowd seg

- init tfds style dataset for crowd seg data

Author: blotero

core/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 description = "Framework for handling image segmentation in the context of multiple annotators"
 name = "seg_tgce"
-version = "0.2.1.dev3"
+version = "0.2.2.dev1"
 readme = "README.md"
 authors = [{ name = "Brandon Lotero", email = "blotero@gmail.com" }]
 maintainers = [{ name = "Brandon Lotero", email = "blotero@gmail.com" }]
@@ -15,7 +15,7 @@ Issues = "https://github.com/blotero/seg_tgce/issues"
 
 [tool.poetry]
 name = "seg_tgce"
-version = "0.2.1.dev3"
+version = "0.2.2.dev1"
 authors = ["Brandon Lotero <blotero@gmail.com>"]
 description = "A package for the SEG TGCE project"
 readme = "README.md"

core/seg_tgce/data/crowd_seg/__init__.py

@@ -1,49 +1,3 @@
-from typing import Tuple
+from .generator import CrowdSegDataGenerator, Stage, get_crowd_seg_data
 
-from .generator import CrowdSegDataGenerator, DataSchema
-from .stage import Stage
-
-DEFAULT_TARGET_SIZE = (512, 512)
-
-
-def get_all_data(
-    image_size: Tuple[int, int] = DEFAULT_TARGET_SIZE,
-    batch_size: int = 32,
-    shuffle: bool = False,
-    with_sparse_data: bool = False,
-    trim_n_scorers: int | None = None,
-) -> Tuple[CrowdSegDataGenerator, ...]:
-    """
-    Retrieve all data generators for the crowd segmentation task.
-    returns a tuple of ImageDataGenerator instances for the train, val, and test stages.
-    """
-    return tuple(
-        CrowdSegDataGenerator(
-            batch_size=batch_size,
-            image_size=image_size,
-            shuffle=shuffle,
-            stage=stage,
-            schema=DataSchema.MA_SPARSE if with_sparse_data else DataSchema.MA_RAW,
-            trim_n_scorers=trim_n_scorers,
-        )
-        for stage in (Stage.TRAIN, Stage.VAL, Stage.TEST)
-    )
-
-
-def get_stage_data(
-    stage: Stage,
-    image_size: Tuple[int, int] = DEFAULT_TARGET_SIZE,
-    batch_size: int = 32,
-    shuffle: bool = False,
-    with_sparse_data: bool = False,
-) -> CrowdSegDataGenerator:
-    """
-    Retrieve a data generator for a specific stage of the crowd segmentation task.
-    """
-    return CrowdSegDataGenerator(
-        batch_size=batch_size,
-        image_size=image_size,
-        shuffle=shuffle,
-        stage=stage,
-        schema=DataSchema.MA_SPARSE if with_sparse_data else DataSchema.MA_RAW,
-    )
+__all__ = ["CrowdSegDataGenerator", "Stage", "get_crowd_seg_data"]

core/seg_tgce/data/crowd_seg/__main__.py

@@ -1,50 +1,23 @@
 from matplotlib import pyplot as plt
 
-from seg_tgce.data.crowd_seg import get_all_data
-from seg_tgce.data.crowd_seg.generator import CrowdSegDataGenerator
+from seg_tgce.data.crowd_seg import get_crowd_seg_data
 
 
 def main() -> None:
     print("Loading data...")
-    train, val, test = get_all_data(batch_size=16)
+    train, val, test = get_crowd_seg_data(batch_size=128)
 
-    # Get a sample batch from each generator
-    train_batch = next(iter(train))
-    val_batch = next(iter(val))
-    test_batch = next(iter(test))
-
-    # Print shapes
-    print("\nTrain data shapes:")
-    print(f"Images shape: {train_batch[0].shape}")
-    print(f"Ground truth mask shape: {train_batch[1].shape}")
-    print(f"Labeler masks shape: {train_batch[2].shape}")
-
-    print("\nValidation data shapes:")
-    print(f"Images shape: {val_batch[0].shape}")
-    print(f"Ground truth mask shape: {val_batch[1].shape}")
-    print(f"Labeler masks shape: {val_batch[2].shape}")
-
-    print("\nTest data shapes:")
-    print(f"Images shape: {test_batch[0].shape}")
-    print(f"Ground truth mask shape: {test_batch[1].shape}")
-    print(f"Labeler masks shape: {test_batch[2].shape}")
-
-    fig = val.visualize_sample(batch_index=75, sample_indexes=[0, 1, 4, 5])
-    fig.tight_layout()
-    fig.savefig(
-        "/home/brandon/unal/maestria/master_thesis/Cap1/Figures/multiannotator-segmentation.png"
+    fig = train.visualize_sample(
+        batch_index=6, sample_indexes=[0, 1, 30, 31, 63, 64, 126, 127]
     )
+    fig.tight_layout()
+    # fig.savefig(
+    # "/home/brandon/unal/maestria/master_thesis/Cap1/Figures/multiannotator-segmentation.png"
+    # )
+    plt.show()
     print(f"Train: {len(train)} batches, {len(train) * train.batch_size} samples")
     print(f"Val: {len(val)} batches, {len(val) * val.batch_size} samples")
     print(f"Test: {len(test)} batches, {len(test) * test.batch_size} samples")
 
-    print("Loading train data with trimmed scorers...")
-    train = CrowdSegDataGenerator(
-        batch_size=8,
-        trim_n_scorers=6,
-    )
-    print(f"Train: {len(train)} batches, {len(train) * train.batch_size} samples")
-    print(f"Train scorers tags: {train.scorers_tags}")
-
 
 main()

core/seg_tgce/data/crowd_seg/__retrieve.py

@@ -6,7 +6,7 @@
 from botocore import UNSIGNED
 from botocore.client import Config
 
-from .stage import Stage
+from seg_tgce.data.crowd_seg.types import Stage
 
 LOGGER = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
@@ -21,11 +21,11 @@
 
 
 def get_masks_dir(stage: Stage) -> str:
-    return os.path.join(_TARGET_DIR, "masks", stage.value)
+    return os.path.join(_TARGET_DIR, "masks", stage.capitalize())
 
 
 def get_patches_dir(stage: Stage) -> str:
-    return os.path.join(_TARGET_DIR, "patches", stage.value)
+    return os.path.join(_TARGET_DIR, "patches", stage.capitalize())
 
 
 def _unzip_dirs() -> None:
@@ -62,13 +62,9 @@ def verify_path(path: str, with_raise: bool = False) -> bool:
 
 
 def fetch_data() -> None:
-    paths_to_verify = [
-        get_patches_dir(Stage.TRAIN),
-        get_patches_dir(Stage.VAL),
-        get_patches_dir(Stage.TEST),
-        get_masks_dir(Stage.TRAIN),
-        get_masks_dir(Stage.VAL),
-        get_masks_dir(Stage.TEST),
+    stages: tuple[Stage, ...] = ("train", "val", "test")
+    paths_to_verify = [get_patches_dir(stage) for stage in stages] + [
+        get_masks_dir(stage) for stage in stages
     ]
     if all(verify_path(path) for path in paths_to_verify):
         return