Feat 341 add derivatives (#346)

jtyoung84 · web-flow · commit f708c05675b2 · 2025-12-08T09:21:59.000-08:00
diff --git a/docs/examples/basic_upload.py b/docs/examples/basic_upload.py
@@ -8,10 +8,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/behavior_videos_compression.py b/docs/examples/behavior_videos_compression.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/custom_codeocean_pipeline_settings.py b/docs/examples/custom_codeocean_pipeline_settings.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/custom_metadata_mapper_settings.py b/docs/examples/custom_metadata_mapper_settings.py
@@ -9,10 +9,6 @@
 
 import requests
 from aind_data_schema_models.modalities import Modality
-
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
 from aind_metadata_mapper.bergamo.models import (
     JobSettings as BergamoSessionSettings,
 )
@@ -23,6 +19,8 @@
     SessionSettings,
 )
 
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/hcr_example.py b/docs/examples/hcr_example.py
@@ -5,10 +5,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/remove_source_folders.py b/docs/examples/remove_source_folders.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/skip_s3_check.py b/docs/examples/skip_s3_check.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/upload_with_custom_slurm_settings.py b/docs/examples/upload_with_custom_slurm_settings.py
@@ -8,10 +8,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/upload_with_notification.py b/docs/examples/upload_with_notification.py
@@ -8,10 +8,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/source/UserGuide.rst b/docs/source/UserGuide.rst
@@ -38,10 +38,12 @@ job_types.
    -  **gather_preliminary_metadata**: Automatically gathers metadata for
       subject, procedures, and data_description. Will also gather metadata from
       aind-metadata-mapper if options are set.
+   -  copy_derivatives_folder: Can specificy a derivatives folder to upload.
+      Set the job_settings like {"input_source": "path_to_folder"}.
    -  **modality_transformation_settings**: This creates the settings for
       transforming the modality folders.
-   -  compress_data: This is a mapped task. Each Task in the
-      modality_transformation_settings will run in parallel in a separate
+   -  compress_data/monitor_compress_data: This is a mapped task. Each Task in
+      the modality_transformation_settings will run in parallel in a separate
       container here.
    -  gather_final_metadata: Automatically generates a processing.json and
       metadata.nd.json file.
@@ -128,10 +130,13 @@ portal can accessed at
 -  Optional fields
 
    -  platform: Standardized way of collecting and processing data
-      (chosen from drop down menu)
+      (chosen from drop down menu) Note: This field is deprecated and will be
+      removed when aind-data-schema 2.0 is rolled out.
    -  metadata_dir: If metadata files are pre-compiled and saved to a
       directory, you can add the Posix style path to the directory under
       this column
+   -  derivatives_dir: If a derivatives folder is available for upload, can be
+      specified as a Posix style path to the directory under this column
    -  s3_bucket: As default, data will be uploaded to a default bucket
       in S3 managed by AIND. Please reach out to the Scientific
       Computing department if you wish to upload to a different bucket.
diff --git a/src/aind_data_transfer_service/configs/csv_handler.py b/src/aind_data_transfer_service/configs/csv_handler.py
@@ -143,8 +143,14 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
         if "metadata_dir" in job_configs
         else None
     )
+    derivatives_task = (
+        Task(job_settings={"input_source": job_configs.pop("derivatives_dir")})
+        if "derivatives_dir" in job_configs
+        else None
+    )
     tasks = {
         "gather_preliminary_metadata": metadata_task,
+        "copy_derivatives_folder": derivatives_task,
         "check_s3_folder_exists": check_s3_folder_exists_task,
         "modality_transformation_settings": modality_tasks,
         "codeocean_pipeline_settings": (
