Merge pull request #210 from AllenNeuralDynamics/remove-watchdog-platform

Add `aind-watchdog-service` as a source for models

Author: bruno-f-cruz

pyproject.toml

@@ -40,6 +40,7 @@ Changelog = "https://github.com/AllenNeuralDynamics/clabe/releases"
 
 [project.optional-dependencies]
 aind-services = [
+    "aind-watchdog-service==0.1.6.dev8",
     "aind-data-schema",
     "ms-active-directory",
     "aind-data-transfer-service",

src/clabe/data_transfer/_aind_watchdog_models.py

@@ -5,27 +5,27 @@
 import subprocess
 from os import PathLike
 from pathlib import Path, PurePosixPath
-from typing import Callable, ClassVar, Dict, List, Literal, Optional, Union
+from typing import Callable, ClassVar, Dict, List, Optional, Union
 
 import aind_data_transfer_service.models.core
 import pydantic
 import requests
 import yaml
 from aind_behavior_services import AindBehaviorSessionModel
+from aind_watchdog_service.models import (
+    ManifestConfig,
+)
+from aind_watchdog_service.models.watch_config import WatchConfig
 from pydantic import BaseModel, SerializeAsAny, TypeAdapter
 from requests.exceptions import HTTPError
 
 from ..services import ServiceSettings
-from ._aind_watchdog_models import (
-    DEFAULT_TRANSFER_ENDPOINT,
-    BucketType,
-    ManifestConfig,
-    WatchConfig,
-)
 from ._base import DataTransfer
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_TRANSFER_ENDPOINT: str = "http://aind-data-transfer-service-dev/api/v2/submit_jobs"
+
 TransferServiceTask = Dict[
     str, Union[aind_data_transfer_service.models.core.Task, Dict[str, aind_data_transfer_service.models.core.Task]]
 ]
@@ -44,8 +44,6 @@ class WatchdogSettings(ServiceSettings):
     destination: Path
     schedule_time: Optional[datetime.time] = datetime.time(hour=20)
     project_name: str
-    s3_bucket: BucketType = "private"
-    force_cloud_sync: bool = False
     transfer_endpoint: str = DEFAULT_TRANSFER_ENDPOINT
     delete_modalities_source_after_success: bool = False
     extra_identifying_info: Optional[dict] = None
@@ -54,10 +52,6 @@ class WatchdogSettings(ServiceSettings):
     extra_modality_data: Optional[Dict[str, List[Path]]] = pydantic.Field(
         default=None, description="Additional modality data to include in the transfer"
     )
-    mount: Optional[None] = pydantic.Field(default=None, deprecated=True)
-    platform: Literal["behavior"] = pydantic.Field(default="behavior", deprecated=True)
-    capsule_id: Optional[None] = pydantic.Field(default=None, deprecated=True)
-    script: Optional[Dict[str, List[str]]] = pydantic.Field(default=None, deprecated=True)
 
 
 class WatchdogDataTransferService(DataTransfer[WatchdogSettings]):
@@ -258,23 +252,18 @@ def _create_manifest_from_session(self, session: AindBehaviorSessionModel) -> Ma
 
         _manifest_config = ManifestConfig(
             name=self._session.session_name,
-            modalities={m: [Path(p) for p in paths] for m, paths in _modality_candidates.items()},
+            modalities={m: [str(Path(p)) for p in paths] for m, paths in _modality_candidates.items()},
             subject_id=int(session.subject),
             acquisition_datetime=session.date,
-            schemas=[Path(value) for value in self._find_schema_candidates(source)],
-            destination=Path(destination),
-            mount=self._settings.mount,
-            processor_full_name=",".join(session.experimenter),
+            schemas=[str(Path(value)) for value in self._find_schema_candidates(source)],
+            destination=str(Path(destination)),
             project_name=self._settings.project_name,
             schedule_time=self._settings.schedule_time,
-            platform=self._settings.platform,
-            capsule_id=self._settings.capsule_id,
-            s3_bucket=self._settings.s3_bucket,
-            script=self._settings.script if self._settings.script else {},
-            force_cloud_sync=self._settings.force_cloud_sync,
             transfer_endpoint=self._settings.transfer_endpoint,
             delete_modalities_source_after_success=self._settings.delete_modalities_source_after_success,
             extra_identifying_info=self._settings.extra_identifying_info,
+            transfer_service_job_type=self._settings.job_type,
+            checksum_mode_override=None,
         )
 
         _manifest_config = self._make_transfer_args(
@@ -352,14 +341,12 @@ def _make_transfer_args(
         upload_job_configs_v2 = aind_data_transfer_service.models.core.UploadJobConfigsV2(
             job_type=job_type,
             project_name=manifest.project_name,
-            platform=aind_data_transfer_service.models.core.Platform.from_abbreviation(manifest.platform),
             modalities=[
                 aind_data_transfer_service.models.core.Modality.from_abbreviation(m) for m in manifest.modalities.keys()
             ],
             subject_id=str(manifest.subject_id),
             acq_datetime=manifest.acquisition_datetime.strftime("%Y-%m-%d %H:%M:%S"),
             tasks=tasks,
-            s3_bucket=manifest.s3_bucket,
             acquisition_datetime=manifest.acquisition_datetime,
         )
 
@@ -513,10 +500,10 @@ def dump_manifest_config(self, path: Optional[os.PathLike] = None, make_dir: boo
         if make_dir and not path.parent.exists():
             path.parent.mkdir(parents=True, exist_ok=True)
 
-        manifest_config.destination = Path(manifest_config.destination)
-        manifest_config.schemas = [Path(schema) for schema in manifest_config.schemas]
+        manifest_config.destination = str(Path(manifest_config.destination))
+        manifest_config.schemas = [str(Path(schema)) for schema in manifest_config.schemas]
         for modality in manifest_config.modalities:
-            manifest_config.modalities[modality] = [_path for _path in manifest_config.modalities[modality]]
+            manifest_config.modalities[modality] = [str(Path(_path)) for _path in manifest_config.modalities[modality]]
 
         self._write_yaml(manifest_config, path)
         return path

src/clabe/data_transfer/aind_watchdog.py

@@ -83,15 +83,8 @@ def watchdog_service(mock_ui_helper, source, settings, mock_session):
         acquisition_datetime=datetime(2023, 1, 1, 0, 0, 0),
         schemas=["path/to/schema"],
         destination="path/to/destination",
-        mount="mount_path",
-        processor_full_name="processor_name",
         project_name="test_project",
         schedule_time=settings.schedule_time,
-        platform="behavior",
-        capsule_id="capsule_id",
-        s3_bucket="private",
-        script={"script_key": ["script_value"]},
-        force_cloud_sync=True,
         transfer_endpoint="http://aind-data-transfer-service-dev/api/v2/submit_jobs",
     )