filled with deets, not yet tested

Author: toqduj

README.md

@@ -1,2 +1,67 @@
-# openbis_mouse_uploader
-Small uploader utility with CLI for uploading MOUSE pre-processed and processed data to OpenBIS, linked to projects, samples, people. 
+# MOUSE OpenBIS Uploader
+
+A small CLI tool to upload MOUSE SAXS measurement batches to OpenBIS based on a **YMD** day code
+(e.g. `20251220`). It reads the measurement metadata from the Excel logbook and uploads/updates
+OpenBIS objects and datasets (RAW and PROCESSED).
+
+## Install (editable)
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -U pip
+pip install -e .
+```
+
+## Token-based authentication
+
+The CLI uses a token stored in a file (default: `~/.datastore_token`).
+
+- Make sure the token file exists and contains a non-empty token string.
+- Override the path via `--datastore-token-path`.
+
+## Usage
+
+Minimal (only required argument is the YMD code):
+
+```bash
+mouse-uploader 20251220
+```
+
+More verbose logging:
+
+```bash
+mouse-uploader 20251220 --log-level DEBUG
+```
+
+Continue from a later row in the logbook:
+
+```bash
+mouse-uploader 20251220 --start-row 50
+```
+
+Dry-run (no writes; logs intended actions):
+
+```bash
+mouse-uploader 20251220 --dry-run
+```
+
+## Notes on behavior
+
+- **People collections are grouped by proposal year**: `PEOPLE_<proposal-year>`.
+- **Duplicate objects should not exist**. If they do, the uploader logs a warning and deletes
+  duplicates permanently until a single object remains.
+- If the project-leader BAM record is not found, the `project_leader_bam` property is **omitted**.
+
+## Development
+
+- Formatting/linting: `ruff`
+- Typing: `mypy` (configured to ignore missing third-party stubs)
+
+Example:
+
+```bash
+ruff check .
+mypy src
+pytest
+```

pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "mouse-openbis-uploader"
+version = "0.1.0"
+description = "CLI tool to upload MOUSE SAXS measurement batches to OpenBIS using a YMD filter."
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "BSD-3-Clause"}
+authors = [
+  {name = "BAM MOUSE team"}
+]
+dependencies = [
+  "attrs>=23.2.0",
+  "pybis>=1.36.0",
+  "logbook2mouse",
+  "jupyter-analysis-tools",
+]
+
+[project.urls]
+Repository = "https://example.invalid/replace-me"
+
+[project.scripts]
+openbis-mouse-uploader = "mouse_openbis_uploader.cli:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false
+ignore_missing_imports = true

src/mouse_openbis_uploader/__init__.py

@@ -0,0 +1,4 @@
+"""MOUSE OpenBIS Uploader package."""
+
+__all__ = ["__version__"]
+__version__ = "0.1.0"

src/mouse_openbis_uploader/cli.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import argparse
+import logging
+from pathlib import Path
+from typing import Optional, Sequence
+
+from pybis import Openbis
+
+from logbook2mouse.logbook_reader import Logbook2MouseReader
+
+from .config import UploadConfig
+from .logging_utils import setup_logger
+from .uploader import OpenBISUploader
+from .utils import read_token, validate_ymd
+from .failures import FailureRecorder
+
+
+def build_parser() -> argparse.ArgumentParser:
+    d = UploadConfig()  # defaults from class
+    p = argparse.ArgumentParser(
+        prog="mouse-uploader",
+        description="Upload MOUSE measurement batches to OpenBIS using a YMD filter.",
+    )
+
+    p.add_argument("ymd", type=validate_ymd, help="Measurement day code, e.g. 20251220")
+
+    p.add_argument("--ds-username", default=d.ds_username, help=f"Username label for logging (default: {d.ds_username})")
+    p.add_argument("--logbook-path", type=Path, default=d.logbook_path, help=f"Excel logbook (default: {d.logbook_path})")
+    p.add_argument("--proposal-base-path", type=Path, default=d.proposal_base_path, help=f"Proposal base path (default: {d.proposal_base_path})")
+    p.add_argument("--base-data-path", type=Path, default=d.base_data_path, help=f"Base data path (default: {d.base_data_path})")
+    p.add_argument("--datastore-token-path", type=Path, default=d.datastore_token_path, help=f"Token file path (default: {d.datastore_token_path})")
+
+    p.add_argument("--space-name", default=d.space_name, help=f"Space name (default: {d.space_name})")
+    p.add_argument("--projects-prepend", default=d.projects_prepend, help=f"Project prefix (default: {d.projects_prepend})")
+    p.add_argument("--start-row", type=int, default=d.start_row, help=f"Start row index (default: {d.start_row})")
+
+    p.add_argument("--server-url", default=d.server_url, help=f"OpenBIS URL (default: {d.server_url})")
+    p.add_argument("--sleep-seconds-between-ops", type=float, default=d.sleep_seconds_between_ops, help=f"Sleep between ops (default: {d.sleep_seconds_between_ops})")
+    p.add_argument("--sleep-seconds-between-datasets", type=float, default=d.sleep_seconds_between_datasets, help=f"Sleep between datasets (default: {d.sleep_seconds_between_datasets})")
+
+    p.add_argument("--instrument-name-pattern", default=d.instrument_name_pattern, help=f"Instrument name pattern (default: {d.instrument_name_pattern})")
+    p.add_argument("--people-collection-prefix", default=d.people_collection_prefix, help=f"People collection prefix (default: {d.people_collection_prefix})")
+
+    p.add_argument("--raw-dataset-type", default=d.raw_dataset_type, help=f"Raw dataset type (default: {d.raw_dataset_type})")
+    p.add_argument("--processed-dataset-type", default=d.processed_dataset_type, help=f"Processed dataset type (default: {d.processed_dataset_type})")
+    p.add_argument("--log-file", type=Path, default=None, help="Log file path (default: None, logs to stdout only)")
+    p.add_argument("--failure-file", type=Path, default=Path("upload_failures.jsonl"), help="Failure records file (default: upload_failures.jsonl)")
+
+    p.add_argument(
+        "--log-level",
+        default="INFO",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        help="Logging verbosity",
+    )
+    p.add_argument("--dry-run", action="store_true", help="No writes; log intended actions")
+
+    return p
+
+
+def _validate_args(args: argparse.Namespace) -> None:
+    if args.start_row < 0:
+        raise SystemExit("--start-row must be >= 0")
+    if args.sleep_seconds_between_ops < 0:
+        raise SystemExit("--sleep-seconds-between-ops must be >= 0")
+    if args.sleep_seconds_between_datasets < 0:
+        raise SystemExit("--sleep-seconds-between-datasets must be >= 0")
+
+    if not args.logbook_path.is_file():
+        raise SystemExit(f"Logbook file not found: {args.logbook_path}")
+    if not args.proposal_base_path.exists():
+        raise SystemExit(f"Proposal base path does not exist: {args.proposal_base_path}")
+    if not args.base_data_path.exists():
+        raise SystemExit(f"Base data path does not exist: {args.base_data_path}")
+    if not args.datastore_token_path.is_file():
+        raise SystemExit(f"Token file not found: {args.datastore_token_path}")
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    _validate_args(args)
+
+    log_level = getattr(logging, args.log_level.upper(), logging.INFO)
+    logger = setup_logger(level=log_level)
+
+    cfg = UploadConfig(
+        ymd_filter=args.ymd,
+        ds_username=args.ds_username,
+        logbook_path=args.logbook_path,
+        proposal_base_path=args.proposal_base_path,
+        base_data_path=args.base_data_path,
+        datastore_token_path=args.datastore_token_path,
+        space_name=args.space_name,
+        projects_prepend=args.projects_prepend,
+        start_row=args.start_row,
+        server_url=args.server_url,
+        sleep_seconds_between_ops=args.sleep_seconds_between_ops,
+        sleep_seconds_between_datasets=args.sleep_seconds_between_datasets,
+        instrument_name_pattern=args.instrument_name_pattern,
+        people_collection_prefix=args.people_collection_prefix,
+        raw_dataset_type=args.raw_dataset_type,
+        processed_dataset_type=args.processed_dataset_type,
+    )
+
+    token = read_token(cfg.datastore_token_path)
+    ds = Openbis(url=cfg.server_url, verify_certificates=True)
+    ds.set_token(token)
+    logger.info("Connected to OpenBIS at %s as %s", cfg.server_url, cfg.ds_username)
+
+    reader = Logbook2MouseReader(
+        cfg.logbook_path,
+        project_base_path=cfg.proposal_base_path,
+        load_all=True,
+    )
+
+    logger = setup_logger(level=log_level, log_file=args.log_file)
+    failure_recorder = FailureRecorder(args.failure_file)
+
+    uploader = OpenBISUploader(ds=ds, config=cfg, logger=logger, dry_run=args.dry_run, failure_recorder=failure_recorder)
+    uploader.process_entries(reader)
+    logger.info(f"Upload run completed. Failures recorded (if any) to: {args.failure_file}")
+    return 0

src/mouse_openbis_uploader/config.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from attrs import define
+
+
+@define(frozen=True, slots=True)
+class UploadConfig:
+    """Configuration for uploading a measurement batch to OpenBIS."""
+
+    # Required (CLI positional)
+    ymd_filter: str  # e.g. "20251220"
+
+    # Optional
+    ds_username: str = "bpauw"
+
+    logbook_path: Path = Path(
+        "/mnt/vsi-db/Measurements/SAXS002/logbooks/Logbook_MOUSE_Dataprocessing.xlsx"
+    )
+    proposal_base_path: Path = Path("/mnt/vsi-db/Proposals/SAXS002/")
+    base_data_path: Path = Path("/mnt/vsi-db/Measurements/SAXS002/data")
+    datastore_token_path: Path = Path.home() / ".datastore_token"
+
+    space_name: str = "6.5_PROJECTS"
+    projects_prepend: str = "MOUSE_PROJECTS_"
+
+    start_row: int = 1
+
+    server_url: str = "https://main.datastore.bam.de"
+    sleep_seconds_between_ops: float = 0.5
+    sleep_seconds_between_datasets: float = 1.0
+
+    instrument_name_pattern: str = "MOUSE*"
+    people_collection_prefix: str = "PEOPLE_"
+
+    raw_dataset_type: str = "RAW_DATA"
+    processed_dataset_type: str = "PROCESSED_DATA"

src/mouse_openbis_uploader/failures.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+
+@dataclass(slots=True)
+class FailureRecord:
+    stage: str                 # e.g. "upsert.PROJECT", "upload.RAW_DATA"
+    ymd: str
+    batchnum: str
+    proposal: str
+    identifier: Optional[str]  # openbis identifier if known
+    message: str
+    extra: dict[str, Any]
+
+
+class FailureRecorder:
+    """
+    Appends one JSON object per failure to a .jsonl file.
+    Designed for large batch runs (streaming write, no big memory use).
+    """
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+
+    def record(self, rec: FailureRecord) -> None:
+        payload = {
+            "stage": rec.stage,
+            "ymd": rec.ymd,
+            "batchnum": rec.batchnum,
+            "proposal": rec.proposal,
+            "identifier": rec.identifier,
+            "message": rec.message,
+            "extra": rec.extra,
+        }
+        with self.path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(payload, ensure_ascii=False) + "\n")

src/mouse_openbis_uploader/logging_utils.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+
+
+def setup_logger(
+    name: str = "openbis_upload_MOUSE",
+    level: int = logging.INFO,
+    *,
+    log_file: Path | None = None,
+) -> logging.Logger:
+    logger = logging.getLogger(name)
+
+    if not logger.handlers:
+        date_fmt = "%Y%m%dT%H:%M:%S"
+        fmt = logging.Formatter(
+            fmt="%(asctime)s - %(name)s - %(levelname)s: %(message)s",
+            datefmt=date_fmt,
+        )
+
+        sh = logging.StreamHandler(stream=sys.stdout)
+        sh.setFormatter(fmt)
+        logger.addHandler(sh)
+
+        if log_file is not None:
+            log_file.parent.mkdir(parents=True, exist_ok=True)
+            fh = logging.FileHandler(log_file, encoding="utf-8")
+            fh.setFormatter(fmt)
+            logger.addHandler(fh)
+
+    logger.setLevel(level)
+    return logger

src/mouse_openbis_uploader/runner.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import logging
+
+from pybis import Openbis
+from logbook2mouse.logbook_reader import Logbook2MouseReader
+
+from .config import UploadConfig
+from .logging_utils import setup_logger
+from .uploader import OpenBISUploader
+from .utils import read_token
+
+
+def run_upload(config: UploadConfig, *, log_level: int = logging.INFO, dry_run: bool = False) -> None:
+    """Programmatic entry point (useful for notebooks)."""
+    logger = setup_logger(level=log_level)
+
+    reader = Logbook2MouseReader(
+        config.logbook_path,
+        project_base_path=config.proposal_base_path,
+        load_all=True,
+    )
+
+    token = read_token(config.datastore_token_path)
+    ds = Openbis(url=config.server_url, verify_certificates=True)
+    ds.set_token(token)
+    logger.info("Connected to OpenBIS at %s as %s", config.server_url, config.ds_username)
+
+    uploader = OpenBISUploader(ds=ds, config=config, logger=logger, dry_run=dry_run)
+    uploader.process_entries(reader)