Tests and CI/CD

.github/workflows/build_test_push_images.yml

@@ -0,0 +1,95 @@
+name: CI/CD
+
+on:
+  pull_request:
+    branches: [main]
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  prepare-matrix-jobs:
+    name: Prepare Build and Test Jobs
+    runs-on: "ubuntu-latest"
+    outputs:
+      has_changed_files: ${{ steps.changed-files.outputs.any_changed }}
+      sorters: ${{ steps.get-changed-sorters.outputs.sorters }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v23
+        with:
+          files: |
+            */Dockerfile
+          files_ignore: |
+            hdsort-compiled/*
+            ironclust-compiled/*
+            kilosort_no_license/*
+            kilosort_with_license/*
+            spikeinterface-jupyterlab/*
+            tests/*
+            waveclus-compiled/*
+
+      - name: Prepare jobs matrix
+        id: get-changed-sorters
+        run: |
+          # sorters variable will be a JSON-like array
+          sorters="["
+
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            # Removing /Dockerfile from string
+            parsed=${file%*/Dockerfile*}
+
+            # Appending to final string
+            sorters+="\"${parsed}\", "
+          done
+          sorters=${sorters%*, *}
+          sorters+="]"
+
+          echo ${sorters}
+          echo "::set-output name=sorters::${sorters}"
+
+  build-and-test-image:
+    name: Build and Test (${{ matrix.sorter }})
+    if: needs.prepare-matrix-jobs.outputs.has_changed_files == 'true'
+    needs: prepare-matrix-jobs
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # "macos-latest", "windows-latest"
+        os: ["ubuntu-latest", ]
+        sorter: ${{ fromJSON(needs.prepare-matrix-jobs.outputs.sorters) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          python-version: 3.8
+
+      - name: Build ${{ matrix.sorter }} Docker image
+        run: |
+          cd ${{ matrix.sorter }}
+          bash build.sh
+          cd ..
+
+      - uses: eWaterCycle/setup-singularity@v7
+        with:
+          singularity-version: 3.8.3
+
+      - name: Install dependencies & Test ${{ matrix.sorter }} image
+        run: |
+          if  [[ ${{ matrix.sorter }} == 'tridesclous' ]]; then
+            pip install tridesclous
+          fi
+          pip install -r requirements_test.txt
+          pytest -sv tests/test_singularity_containers.py::test_${{ matrix.sorter }}
+
+      - name: Docker login & Push image
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          cd ${{ matrix.sorter }}
+          bash push.sh

kilosort_no_license/kilosort2_5-compiled/ks2_5_compiled.m

@@ -12,7 +12,18 @@ function ks2_5_compiled(fpath)
         rez = preprocessDataSub(ops);
 
         % NEW STEP TO DO DATA REGISTRATION
-        rez = datashift2(rez, 1); % last input is for shifting data
+        if isfield(ops, 'do_correction')
+            do_correction = ops.do_correction;
+        else 
+            do_correction = 1;
+        end
+
+        if do_correction
+            fprintf("Drift correction ENABLED\n");
+        else
+            fprintf("Drift correction DISABLED\n");
+        end
+        rez = datashift2(rez, do_correction); % last input is for shifting data
 
         % ORDER OF BATCHES IS NOW RANDOM, controlled by random number generator
         iseed = 1;

kilosort_no_license/kilosort3-compiled/ks3_compiled.m

@@ -12,7 +12,18 @@ function ks3_compiled(fpath)
         rez = preprocessDataSub(ops);
 
         % run data registration
-        rez = datashift2(rez, 1); % last input is for shifting data
+        if isfield(ops, 'do_correction')
+            do_correction = ops.do_correction;
+        else 
+            do_correction = 1;
+        end
+
+        if do_correction
+            fprintf("Drift correction ENABLED\n");
+        else
+            fprintf("Drift correction DISABLED\n");
+        end
+        rez = datashift2(rez, do_correction); % last input is for shifting data
 
         [rez, st3, tF] = extract_spikes(rez);