Merge pull request #348 from AllenNeuralDynamics/release-v1.23.0

helen-m-lin · web-flow · commit 47dd6503be87 · 2025-12-08T16:25:10.000-08:00
Release v1.23.0
diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml
@@ -5,30 +5,23 @@ on:
       - dev
 
 jobs:
-  publish:
+  compute-docker-tag:
     runs-on: ubuntu-latest
+    outputs:
+      docker_tag: ${{ steps.compute-tag.outputs.docker_tag }}
     steps:
       - uses: actions/checkout@v4
       - name: Compute new docker image tag
+        id: compute-tag
         run: |
           echo "sha_short=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV"
           echo "branch=$(echo ${GITHUB_REF_NAME})" >> "$GITHUB_ENV"
-          echo "docker_tag=$(echo ${GITHUB_REF_NAME})-$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV"
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Github Packages
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build image and push to GitHub Container Registry
-        uses: docker/build-push-action@v3
-        with:
-          # relative path to the place where source code with Dockerfile is located
-          context: .
-          push: true
-          tags: |
-            ghcr.io/allenneuraldynamics/aind-data-transfer-service:${{ env.docker_tag }}
-            ghcr.io/allenneuraldynamics/aind-data-transfer-service:dev
+          echo "docker_tag=$(echo ${GITHUB_REF_NAME})-$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_OUTPUT"
+  publish-image:
+    needs: compute-docker-tag
+    uses: AllenNeuralDynamics/.github/.github/workflows/release-publish-docker-image.yml@main
+    with:
+      default-branch: dev
+      docker-tag: ${{ needs.compute-docker-tag.outputs.docker_tag }}
+    secrets:
+      repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/publish_main.yml b/.github/workflows/publish_main.yml
@@ -5,42 +5,24 @@ on:
       - main
 
 jobs:
-  tag_and_publish:
-    name: Parse version
+  tag:
+    name: Tag
+    uses: AllenNeuralDynamics/.github/.github/workflows/release-tag.yml@main
+    secrets:
+      repo-token: ${{ secrets.GITHUB_TOKEN }}
+  publish-image:
+    needs: tag
+    uses: AllenNeuralDynamics/.github/.github/workflows/release-publish-docker-image.yml@main
+    with:
+      docker-tag: ${{ needs.tag.outputs.new_version }}
+    secrets:
+      repo-token: ${{ secrets.GITHUB_TOKEN }}
+  publish-pypi:
+    needs: tag
     runs-on: ubuntu-latest
-    outputs:
-      pkg_version: ${{ steps.output_version.outputs.pkg_version }}
     steps:
-    - uses: actions/checkout@v4
-    - name: Get version from file
-      run: |
-        pkg_name=$(grep -P 'version = \{attr = .*\}' pyproject.toml | grep -oP '\w+.__version__')
-        init_file="./src/${pkg_name//.__version__}/__init__.py"
-        pkg_version=$(grep -Po '[0-9]+\.[0-9]+\.[0-9]+' "$init_file")
-        echo "docker_tag=$pkg_version" >> "$GITHUB_ENV"
-    - name: Create git tag
-      run: |
-        git tag "v${{ env.docker_tag }}"
-    - name: Push git tag
-      run: git push origin "v${{ env.docker_tag }}"
-    - name: Set up Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@v2
-    - name: Login to Github Packages
-      uses: docker/login-action@v2
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Build image and push to GitHub Container Registry
-      uses: docker/build-push-action@v3
-      with:
-        # relative path to the place where source code with Dockerfile is located
-        context: .
-        push: true
-        tags: |
-          ghcr.io/allenneuraldynamics/aind-data-transfer-service:${{ env.docker_tag }}
-          ghcr.io/allenneuraldynamics/aind-data-transfer-service:latest
+    - name: Checkout repository
+      uses: actions/checkout@v5
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
@@ -53,4 +35,4 @@ jobs:
     - name: Publish on PyPI
       uses: pypa/gh-action-pypi-publish@release/v1.12
       with:
-        password: ${{ secrets.AIND_PYPI_TOKEN }}
+        password: ${{ secrets.AIND_PYPI_TOKEN }}
diff --git a/.github/workflows/run_dev_tests.yml b/.github/workflows/run_dev_tests.yml
@@ -7,20 +7,9 @@ on:
 
 jobs:
   ci:
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [ '3.10', '3.11', '3.12' ]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: | 
-        python -m pip install -e .[dev]
-    - name: Run linter checks
-      run: flake8 . && interrogate --verbose .
-    - name: Run tests and coverage
-      run: coverage run -m unittest discover && coverage report
+    uses: AllenNeuralDynamics/.github/.github/workflows/test-ci.yml@main
+    with:
+      python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/run_main_tests.yml b/.github/workflows/run_main_tests.yml
@@ -8,27 +8,16 @@ on:
 
 jobs:
   ci:
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [ '3.10', '3.11', '3.12' ]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: | 
-        python -m pip install -e .[dev]
-    - name: Run linter checks
-      run: flake8 . && interrogate --verbose .
-    - name: Run tests and coverage
-      run: coverage run -m unittest discover && coverage report
+    uses: AllenNeuralDynamics/.github/.github/workflows/test-ci.yml@main
+    with:
+      python-version: ${{ matrix.python-version }}
   verify_version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Check version incremented
         run: |
           pkg_name=$(grep -P 'version = \{attr = .*\}' pyproject.toml | grep -oP '\w+.__version__')
diff --git a/docs/examples/basic_upload.py b/docs/examples/basic_upload.py
@@ -8,10 +8,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/behavior_videos_compression.py b/docs/examples/behavior_videos_compression.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/custom_codeocean_pipeline_settings.py b/docs/examples/custom_codeocean_pipeline_settings.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/custom_metadata_mapper_settings.py b/docs/examples/custom_metadata_mapper_settings.py
@@ -9,10 +9,6 @@
 
 import requests
 from aind_data_schema_models.modalities import Modality
-
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
 from aind_metadata_mapper.bergamo.models import (
     JobSettings as BergamoSessionSettings,
 )
@@ -23,6 +19,8 @@
     SessionSettings,
 )
 
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/hcr_example.py b/docs/examples/hcr_example.py
@@ -5,10 +5,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/remove_source_folders.py b/docs/examples/remove_source_folders.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/skip_s3_check.py b/docs/examples/skip_s3_check.py
@@ -9,10 +9,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/upload_with_custom_slurm_settings.py b/docs/examples/upload_with_custom_slurm_settings.py
@@ -8,10 +8,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/examples/upload_with_notification.py b/docs/examples/upload_with_notification.py
@@ -8,10 +8,8 @@
 import requests
 from aind_data_schema_models.modalities import Modality
 
-# Note, the platform field is optional and will be deprecated. If adding a
-# platform please use aind-data-transfer-service <= 1.17.2
-from aind_data_schema_models.platforms import Platform
-
+# Note, the platform field is optional and will be deprecated.
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import (
     SubmitJobRequestV2,
     Task,
diff --git a/docs/source/UserGuide.rst b/docs/source/UserGuide.rst
@@ -38,10 +38,12 @@ job_types.
    -  **gather_preliminary_metadata**: Automatically gathers metadata for
       subject, procedures, and data_description. Will also gather metadata from
       aind-metadata-mapper if options are set.
+   -  copy_derivatives_folder: Can specificy a derivatives folder to upload.
+      Set the job_settings like {"input_source": "path_to_folder"}.
    -  **modality_transformation_settings**: This creates the settings for
       transforming the modality folders.
-   -  compress_data: This is a mapped task. Each Task in the
-      modality_transformation_settings will run in parallel in a separate
+   -  compress_data/monitor_compress_data: This is a mapped task. Each Task in
+      the modality_transformation_settings will run in parallel in a separate
       container here.
    -  gather_final_metadata: Automatically generates a processing.json and
       metadata.nd.json file.
@@ -128,10 +130,13 @@ portal can accessed at
 -  Optional fields
 
    -  platform: Standardized way of collecting and processing data
-      (chosen from drop down menu)
+      (chosen from drop down menu) Note: This field is deprecated and will be
+      removed when aind-data-schema 2.0 is rolled out.
    -  metadata_dir: If metadata files are pre-compiled and saved to a
       directory, you can add the Posix style path to the directory under
       this column
+   -  derivatives_dir: If a derivatives folder is available for upload, can be
+      specified as a Posix style path to the directory under this column
    -  s3_bucket: As default, data will be uploaded to a default bucket
       in S3 managed by AIND. Please reach out to the Scientific
       Computing department if you wish to upload to a different bucket.
diff --git a/src/aind_data_transfer_service/__init__.py b/src/aind_data_transfer_service/__init__.py
@@ -2,7 +2,7 @@
 
 import os
 
-__version__ = "1.22.0"
+__version__ = "1.23.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")
diff --git a/src/aind_data_transfer_service/configs/csv_handler.py b/src/aind_data_transfer_service/configs/csv_handler.py
@@ -143,8 +143,14 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
         if "metadata_dir" in job_configs
         else None
     )
+    derivatives_task = (
+        Task(job_settings={"input_source": job_configs.pop("derivatives_dir")})
+        if "derivatives_dir" in job_configs
+        else None
+    )
     tasks = {
         "gather_preliminary_metadata": metadata_task,
+        "copy_derivatives_folder": derivatives_task,
         "check_s3_folder_exists": check_s3_folder_exists_task,
         "modality_transformation_settings": modality_tasks,
         "codeocean_pipeline_settings": (
