Merge pull request #47 from neuroinformatics-unit/data-from-gin-fro-binder

Use pooch to retrieve data from gin

Author: lauraporta

derotation/sample_data.py

@@ -0,0 +1,62 @@
+"""Fetch and load sample datasets for derotation examples.
+
+This module provides functions for fetching sample data used in examples.
+The data are stored in a remote repository and are downloaded to the user's
+local machine the first time they are used.
+"""
+
+from pathlib import Path
+
+import pooch
+
+# URL to the remote data repository on GIN
+DATA_URL = "https://gin.g-node.org/l.porta/derotation_examples/raw/master/"
+
+# Save data in ~/.derotation/data
+DATA_DIR = Path("~", ".derotation", "data").expanduser()
+DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+# File registry with SHA256 checksums for data integrity validation
+REGISTRY = {
+    "angles_per_line.npy": (
+        "0a5dab081acdfb47ebd5cdda4094cc622b1ff708c63f6fadc1e7898d30789896"
+    ),
+    "rotation_sample.tif": (
+        "ad8aae61cda9495d9006fb37a1f87be8b6dd6477318b1e8a409417cace208f56"
+    ),
+    "analog_signals.npy": (
+        "e04437f86d07db8077c256f912e2bcca75794712d66715f301b99cd9a8d66d95"
+    ),
+    "stimulus_randperm.csv": (
+        "214617992d786ee210a7d2f22dbe7420997016846a79c673979913a5b77f0a11"
+    ),
+}
+
+# Create a download manager
+SAMPLE_DATA = pooch.create(
+    path=DATA_DIR,
+    base_url=DATA_URL,
+    registry=REGISTRY,
+)
+
+
+def fetch_data(filename: str) -> Path:
+    """Fetch a sample data file.
+
+    Parameters
+    ----------
+    filename : str
+        Name of the file to fetch.
+
+    Returns
+    -------
+    Path
+        Path to the downloaded file.
+    """
+    if filename not in REGISTRY:
+        raise ValueError(
+            f"File '{filename}' not found in registry. "
+            f"Available files: {list(REGISTRY.keys())}"
+        )
+
+    return Path(SAMPLE_DATA.fetch(filename, progressbar=True))

examples/core_function.py

@@ -20,32 +20,25 @@
 """
 
 # %%
-from pathlib import Path
 
 import matplotlib.pyplot as plt
 import numpy as np
 import tifffile
 
 from derotation.derotate_by_line import derotate_an_image_array_line_by_line
-
-# %%
-# Set up
-# -----------
-
-current_module_path = Path.cwd()
-data_folder = current_module_path / "data"
+from derotation.sample_data import fetch_data
 
 # %%
 # Load per-line rotation angles
 
-angles_path = data_folder / "angles_per_line.npy"
+angles_path = fetch_data("angles_per_line.npy")
 print(f"Loading per-line rotation angles from {angles_path}")
 angles_per_line = np.load(angles_path)  # Shape: (height,)
 
 # %%
 # Load the TIFF stack (shape: num_frames × height × width)
 
-tif_path = data_folder / "rotation_sample.tif"
+tif_path = fetch_data("rotation_sample.tif")
 print(f"Loading image stack from {tif_path}")
 image_stack = tifffile.imread(tif_path)
 

examples/pipeline_with_real_data.py

@@ -36,20 +36,20 @@
 
 from derotation.analysis.full_derotation_pipeline import FullPipeline
 from derotation.config.load_config import load_config, update_config_paths
+from derotation.sample_data import fetch_data
 
 # %%
 # Load and update configuration
 # -----------------------------
 # We define paths relative to the current working directory
 current_module_path = Path.cwd()
-data_folder = current_module_path / "data"
 
 config = load_config()
 config = update_config_paths(
     config=config,
-    tif_path=str(data_folder / "rotation_sample.tif"),
-    aux_path=str(data_folder / "analog_signals.npy"),
-    stim_randperm_path=str(data_folder / "stimulus_randperm.csv"),
+    tif_path=str(fetch_data("rotation_sample.tif")),
+    aux_path=str(fetch_data("analog_signals.npy")),
+    stim_randperm_path=str(fetch_data("stimulus_randperm.csv")),
     output_folder=str(current_module_path),
 )
 

examples/use_plotting_hooks.py

@@ -38,22 +38,21 @@
 
 from derotation.analysis.full_derotation_pipeline import FullPipeline
 from derotation.config.load_config import load_config, update_config_paths
+from derotation.sample_data import fetch_data
 
 # %%
 # Load and configure paths
 # ------------------------
 # We'll use a small example dataset and write output to the current folder.
 
-current_module_path = Path.cwd()
-data_folder = current_module_path / "data"
-output_folder = current_module_path
+output_folder = Path.cwd()
 
 config = load_config()
 config = update_config_paths(
     config=config,
-    tif_path=str(data_folder / "rotation_sample.tif"),
-    aux_path=str(data_folder / "analog_signals.npy"),
-    stim_randperm_path=str(data_folder / "stimulus_randperm.csv"),
+    tif_path=str(fetch_data("rotation_sample.tif")),
+    aux_path=str(fetch_data("analog_signals.npy")),
+    stim_randperm_path=str(fetch_data("stimulus_randperm.csv")),
     output_folder=str(output_folder),
 )
 

pyproject.toml

@@ -32,6 +32,7 @@ dependencies = [
     "scikit-learn",
     "scikit-image",
     "bayesian-optimization",
+    "pooch",
 ]
 
 [project.urls]