Merge pull request #100 from EliHei2/segger_taylors_version

Segger Taylor's Version

TLDR:
Simplified package dependencies and installation
Rolled back model definition, training, and prediction to previous working versions
Note: This PR adds guides for pre-processing CosMX and MERSCOPE data for segger, but these platform are still not supported.
More Detailed:
Made the following changes for usability and bug fixing:

Added uv for dependency management; removed dask and RAPIDS as dependencies
Updated README with installation instructions
Changes to model were untested and broke model evaluation; reverted to previous model definition
Added testing to ensure full pipeline works with minimal xenium example
Removed redundant global imports in __init__.py files which otherwise require the entire segger package and all dependencies to be imported everywhere in the repo.
Removed out-of-date tests.
Added utility functions and a README.md guide for preprocessing CosMX dataset to output nuclear boundaries (needed for shape estimation during dataset creation)
Added installation guide and param file for preprocessing MERSCOPE data using VPT to get a nuclear segmentation.

Author: EliHei2

.dev/_get_imports.py

@@ -0,0 +1,128 @@
+# Re-import after code reset
+import importlib.metadata
+import os
+import ast
+import sys
+import re
+import pandas as pd
+import pathlib
+from importlib.metadata import distributions
+import tomllib
+
+
+def extract_third_party_imports(root_dir: str) -> pd.DataFrame:
+    """
+    Walk codebase and collect third-party root import names.
+    """
+    stdlib = (
+        set(sys.stdlib_module_names) if hasattr(sys, "stdlib_module_names") else set()
+    )
+    rows = []
+
+    for dirpath, _, filenames in os.walk(root_dir):
+        for filename in filenames:
+            if not filename.endswith(".py"):
+                continue
+            full_path = os.path.join(dirpath, filename)
+            try:
+                with open(full_path, "r", encoding="utf-8") as f:
+                    tree = ast.parse(f.read(), filename=full_path)
+                for node in ast.walk(tree):
+                    if isinstance(node, ast.Import):
+                        for alias in node.names:
+                            name = alias.name.split(".")[0]
+                            if name not in stdlib:
+                                rows.append((full_path, name))
+                    elif isinstance(node, ast.ImportFrom):
+                        if node.module:
+                            name = node.module.split(".")[0]
+                            if name not in stdlib:
+                                rows.append((full_path, name))
+            except (SyntaxError, UnicodeDecodeError):
+                continue
+
+    return pd.DataFrame(rows, columns=["filename", "root_package"]).drop_duplicates()
+
+
+def _extract_pkg_name(dep: str) -> str:
+    return re.split(r"[<>=~! ]", dep, 1)[0].strip().lower()
+
+
+def _get_import_names(declared: set[str]) -> set[str]:
+    """
+    Given a set of declared package names, return the set of all import names
+    associated with those packages based on installed distributions.
+    """
+    dist_map = importlib.metadata.packages_distributions()
+    import_names = set()
+
+    for dep in declared:
+        dep_matches = {k for k, v in dist_map.items() if dep in v}
+        if dep_matches:
+            import_names.update(dep_matches)
+        else:
+            import_names.add(dep.lower().replace("-", "_"))
+
+    return import_names
+
+
+def find_missing_dependencies(project_path: os.PathLike) -> set[str]:
+    """
+    Compare third-party imports with declared dependencies in pyproject.toml.
+
+    Parameters
+    ----------
+    project_path : str
+        Base path of the Python project
+
+    Returns
+    -------
+    pd.DataFrame
+        Subset of `imports_df` where the root_package is used but not declared
+        in pyproject.toml.
+    """
+    project_path = pathlib.Path(project_path)
+    with open(project_path / "pyproject.toml", "rb") as f:
+        toml = tomllib.load(f)
+
+    declared = {_extract_pkg_name(d) for d in toml["project"]["dependencies"]}
+    optional = toml["project"].get("optional-dependencies", {})
+    for group in optional.values():
+        declared.update(_extract_pkg_name(d) for d in group)
+    project_name = toml["project"]["name"].replace("-", "_").lower()
+    declared.add(project_name)
+    declared = _get_import_names(declared)
+
+    imports = extract_third_party_imports(project_path / "src")
+
+    return imports[~imports["root_package"].isin(declared)]
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Find undeclared third-party imports.")
+    parser.add_argument(
+        "--base",
+        type=str,
+        help="Path to the base Python package or source root.",
+        default="./",
+    )
+    parser.add_argument(
+        "--exclude",
+        nargs="*",
+        default=[],
+        help="List of package names to exclude from the check.",
+    )
+
+    args = parser.parse_args()
+    missing_df = find_missing_dependencies(pathlib.Path(args.base))
+
+    if args.exclude:
+        missing_df = missing_df[~missing_df["root_package"].isin(args.exclude)]
+
+    if missing_df.empty:
+        print("No missing dependencies found.")
+    else:
+        print("Missing dependencies:")
+        print(missing_df.sort_values("root_package").to_string(index=False))

.gitignore

@@ -172,3 +172,8 @@ figure*
 dev*
 .DS_Store
 .idea/
+
+
+# Custom
+*_old*
+.dev

.python_version

@@ -0,0 +1 @@
+3.11.11

.scripts/create_dataset.py

@@ -81,7 +81,9 @@ def main(args):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Create dataset from Xenium Human Pancreatic data.")
+    parser = argparse.ArgumentParser(
+        description="Create dataset from Xenium Human Pancreatic data."
+    )
     parser.add_argument(
         "--raw_data_dir",
         type=str,
@@ -100,7 +102,9 @@ def main(args):
         required=True,
         help="URL for transcripts data.",
     )
-    parser.add_argument("--nuclei_url", type=str, required=True, help="URL for nuclei data.")
+    parser.add_argument(
+        "--nuclei_url", type=str, required=True, help="URL for nuclei data."
+    )
     parser.add_argument(
         "--min_qv",
         type=int,
@@ -121,9 +125,15 @@ def main(args):
     )
     parser.add_argument("--x_size", type=int, default=200, help="Width of each tile.")
     parser.add_argument("--y_size", type=int, default=200, help="Height of each tile.")
-    parser.add_argument("--margin_x", type=int, default=None, help="Margin in x direction.")
-    parser.add_argument("--margin_y", type=int, default=None, help="Margin in y direction.")
-    parser.add_argument("--r_tx", type=int, default=3, help="Radius for building the graph.")
+    parser.add_argument(
+        "--margin_x", type=int, default=None, help="Margin in x direction."
+    )
+    parser.add_argument(
+        "--margin_y", type=int, default=None, help="Margin in y direction."
+    )
+    parser.add_argument(
+        "--r_tx", type=int, default=3, help="Radius for building the graph."
+    )
     parser.add_argument(
         "--val_prob",
         type=float,
@@ -142,7 +152,9 @@ def main(args):
         default=3,
         help="Number of nearest neighbors for nuclei.",
     )
-    parser.add_argument("--dist_nc", type=int, default=10, help="Distance threshold for nuclei.")
+    parser.add_argument(
+        "--dist_nc", type=int, default=10, help="Distance threshold for nuclei."
+    )
     parser.add_argument(
         "--k_tx",
         type=int,
@@ -161,8 +173,12 @@ def main(args):
         default=True,
         help="Whether to compute edge labels.",
     )
-    parser.add_argument("--sampling_rate", type=float, default=1, help="Rate of sampling tiles.")
-    parser.add_argument("--parallel", action="store_true", help="Use parallel processing.")
+    parser.add_argument(
+        "--sampling_rate", type=float, default=1, help="Rate of sampling tiles."
+    )
+    parser.add_argument(
+        "--parallel", action="store_true", help="Use parallel processing."
+    )
     parser.add_argument(
         "--num_workers",
         type=int,

.scripts/predict.py

@@ -49,15 +49,21 @@ def main(args: argparse.Namespace) -> None:
         required=True,
         help="Path to the model checkpoint",
     )
-    parser.add_argument("--init_emb", type=int, default=8, help="Initial embedding size")
+    parser.add_argument(
+        "--init_emb", type=int, default=8, help="Initial embedding size"
+    )
     parser.add_argument(
         "--hidden_channels",
         type=int,
         default=64,
         help="Number of hidden channels",
     )
-    parser.add_argument("--out_channels", type=int, default=16, help="Number of output channels")
-    parser.add_argument("--heads", type=int, default=4, help="Number of attention heads")
+    parser.add_argument(
+        "--out_channels", type=int, default=16, help="Number of output channels"
+    )
+    parser.add_argument(
+        "--heads", type=int, default=4, help="Number of attention heads"
+    )
     parser.add_argument("--aggr", type=str, default="sum", help="Aggregation method")
     parser.add_argument(
         "--score_cut",
@@ -71,7 +77,9 @@ def main(args: argparse.Namespace) -> None:
         default=4,
         help="Number of nearest neighbors for nuclei",
     )
-    parser.add_argument("--dist_nc", type=int, default=20, help="Distance threshold for nuclei")
+    parser.add_argument(
+        "--dist_nc", type=int, default=20, help="Distance threshold for nuclei"
+    )
     parser.add_argument(
         "--k_tx",
         type=int,

.scripts/train_model.py

@@ -95,19 +95,31 @@ def main(args):
         default=4,
         help="Batch size for validation",
     )
-    parser.add_argument("--init_emb", type=int, default=8, help="Initial embedding size")
+    parser.add_argument(
+        "--init_emb", type=int, default=8, help="Initial embedding size"
+    )
     parser.add_argument(
         "--hidden_channels",
         type=int,
         default=64,
         help="Number of hidden channels",
     )
-    parser.add_argument("--out_channels", type=int, default=16, help="Number of output channels")
-    parser.add_argument("--heads", type=int, default=4, help="Number of attention heads")
+    parser.add_argument(
+        "--out_channels", type=int, default=16, help="Number of output channels"
+    )
+    parser.add_argument(
+        "--heads", type=int, default=4, help="Number of attention heads"
+    )
     parser.add_argument("--aggr", type=str, default="sum", help="Aggregation method")
-    parser.add_argument("--accelerator", type=str, default="cuda", help="Type of accelerator")
-    parser.add_argument("--strategy", type=str, default="auto", help="Training strategy")
-    parser.add_argument("--precision", type=str, default="16-mixed", help="Precision mode")
+    parser.add_argument(
+        "--accelerator", type=str, default="cuda", help="Type of accelerator"
+    )
+    parser.add_argument(
+        "--strategy", type=str, default="auto", help="Training strategy"
+    )
+    parser.add_argument(
+        "--precision", type=str, default="16-mixed", help="Precision mode"
+    )
     parser.add_argument("--devices", type=int, default=4, help="Number of devices")
     parser.add_argument("--epochs", type=int, default=100, help="Number of epochs")
     parser.add_argument(