Reorganize repo and add workflows for testing and releasing

.github/workflows/create_release.yml

@@ -0,0 +1,176 @@
+# Test the toolbox across all supported releases of MATLAB, package toolbox, create release
+# Adapted from: https://github.com/mathworks/climatedatastore/blob/main/.github/workflows/release.yml
+
+name: Create new release
+
+# Run workflow when a tag is created
+on:
+  push:
+    tags: 
+      - 'v*'
+
+jobs:
+  # This workflow contains:
+  # 1. a matrixed test job run across a bunch of releases of MATLAB
+  # 2. a reporting job that summarizes the tests, and updates release badge
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        MATLABVersion: [R2020b, R2021a, R2021b, R2022a, R2022b, R2023a, R2023b, R2024a, R2024b]
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it
+      - uses: actions/checkout@v4
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          release: ${{ matrix.MATLABVersion }}
+
+      # Runs all tests in the project.  Put results in a version specific subdirectory
+      - name: Run tests
+        uses: matlab-actions/run-command@v2
+        with:
+          command: | 
+            addpath(genpath("tools"));
+            testToolbox('ReportSubdirectory',"${{ matrix.MATLABVersion }}", 'CreateBadge', false)     
+
+      # Save the contents of the report directory from each release into an artifact.
+      - name: Save Report Directory
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: reports-${{ matrix.MATLABVersion }}
+          path: docs/reports
+
+  # Report on what releases tested successfully.
+  # Generate a draft release based on the tag
+  # Recreate the tag with the final version of files
+  release:
+    needs: test
+    if: always()
+    runs-on: ubuntu-latest
+    env:
+      TOOLBOX_FILE_NAME: Brain_Observatory_Toolbox
+
+    steps:
+      # Use deploy key to push back to protected branch
+      - name: Checkout repository using deploy key
+        uses: actions/checkout@v4
+        with:
+            ref: refs/heads/main
+            ssh-key: ${{ secrets.DEPLOY_KEY }}
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+
+      # Copy all the reports down into the container
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: reports-*
+          path: docs/reports
+          merge-multiple: true
+
+      # Generate the JSON for the releases tested badge
+      - name: Generate tested with badge
+        uses: matlab-actions/run-command@v2
+        with:
+          command: addpath(genpath("tools")), createTestedWithBadgeforToolbox("${{  github.ref_name }}")     
+
+      # Publish test results from all the releases
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          junit_files: "docs/reports/*/test-results.xml"
+
+      # Package the MLTBX
+      - name: Package Toolbox
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath(genpath("tools"));
+            packageToolbox("specific","${{  github.ref_name }}")     
+
+      # Define the versionNumber using underscores, as this is used in the MLTBX
+      - name: Set version number
+        id: set_version
+        run: |
+          versionNumber=$(echo "${{ github.ref_name }}" | sed 's/\./_/g')
+          echo "versionNumber=$versionNumber" >> $GITHUB_ENV
+
+      # Save the MLTBX.
+      - name: Save packaged toolbox
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{TOOLBOX_FILE_NAME}}${{ env.versionNumber }}.mltbx
+          path: releases/${{TOOLBOX_FILE_NAME}}_${{ env.versionNumber }}.mltbx
+
+      # Commit the updated Contents.m
+      - name: Commit updated Contents.m file
+        continue-on-error: true
+        run: |
+          git config user.name "${{ github.workflow }} by ${{ github.actor }}"
+          git config user.email "<>"
+          git status
+          git add code/Contents.m
+          git commit -m "Final checkins for release ${{  github.ref_name }}"
+          git fetch
+          git push
+
+      # Commit the JSON for the MATLAB releases test badge to gh-badges branch
+      - name: Checkout gh-badges branch
+        uses: actions/checkout@v4
+        with:
+          ref: gh-badges
+          path: gh-badges
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push to gh-badges
+        run: |
+          mkdir -p gh-badges/.github/badges/${{  github.ref_name }}
+          cp .github/badges/${{  github.ref_name }}/tested_with.json gh-badges/.github/badges/${{  github.ref_name }}/tested_with.json
+          cd gh-badges
+
+          git config user.name "${{ github.workflow }} by ${{ github.actor }}"
+          git config user.email "<>"
+
+          # Only proceed with commit and push if changes are detected
+          if [[ $(git add .github/badges/* --dry-run | wc -l) -gt 0 ]]; then
+            git add .github/badges/*
+            git commit -m "Update tested with badge for release"
+            git push -f
+          else
+            echo "Nothing to commit"
+          fi
+
+      # Retag the repo so that the updated files are included in the release tag
+      - name: Update tag
+        if: always()
+        continue-on-error: true
+        run: |
+          git config user.name "${{ github.workflow }} by ${{ github.actor }}"
+          git config user.email "<>"
+
+          # Delete the existing tag locally and remotely
+          git tag -d "${{ github.ref_name }}" 
+          git push origin --delete "${{ github.ref_name }}"
+
+          # Recreate the tag with a message, including [skip ci] to prevent CI workflows
+          git tag -a "${{ github.ref_name }}" -m "Release ${{ github.ref_name }} [skip ci]" 
+
+          # Push the new tag to the remote repository
+          git push origin "${{ github.ref_name }}"
+
+      # Create the release
+      - name: Create GitHub Release
+        uses: ncipollo/release-action@v1
+        with:
+          draft: true        
+          artifacts: "releases/${{TOOLBOX_FILE_NAME}}_${{ env.versionNumber }}.mltbx"
+          generateReleaseNotes: true
+          body: "![MATLAB Versions Tested](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2F${{github.repository}}%2Fgh-badges%2F.github%2Fbadges%2F${{ github.ref_name }}%2Ftested_with.json)"
+

.github/workflows/run_tests.yml

@@ -0,0 +1,93 @@
+name: Run tests
+
+on:
+  # Triggers the workflow on push or pull request events for the "main" branch
+  push:
+    branches: [ "main" ]
+    paths-ignore:
+      - '*md'
+      - '.github/**'
+      - 'docs/reports/**'
+  pull_request:
+    branches: [ "main" ]
+
+  # Allows for manually running this workflow from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  # This workflow contains a single job called "test"
+  test:
+    name: Test toolbox code
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it
+      - name: Check out repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+
+      # Check for MATLAB code issues in the project.
+      - name: Check for MATLAB code issues
+        uses: matlab-actions/run-command@v2
+        if: always()
+        with:
+          command: addpath(genpath("tools")), codecheckToolbox()
+
+      # Upload code issues report
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          # Path to SARIF file relative to the root of the repository
+          sarif_file: docs/reports/code_issues.sarif
+
+      # Runs all tests in the project.
+      - name: Run tests
+        uses: matlab-actions/run-command@v2
+        if: always()
+        with:
+          command: addpath(genpath("tools")), testToolbox()
+
+      # Commit updated SVG badges for the issues and tests (if changed)
+      - name: Commit svg badges if updated
+        if: always()
+        continue-on-error: true
+        run: |
+          git config user.name "${{ github.workflow }} by ${{ github.actor }}"
+          git config user.email "<>"
+          git fetch 
+
+          if [[ $(git add .github/badges/* --dry-run | wc -l) -gt 0 ]]; then
+            git add .github/badges/*
+            git commit -m "Update code issues and tests badges"
+            git push -f
+          else
+            echo "Nothing to commit"
+          fi
+
+      - name: Upload code coverage report to Codecov
+        uses: codecov/codecov-action@v4
+        if: always()
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: docs/reports/codecoverage.xml
+
+      # Publish test results
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: "docs/reports/test-results.xml"
+
+      # Save the contents of the reports directory as an artifact
+      - name: Save reports directory
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: reports
+          path: docs/reports

.gitignore

@@ -1,6 +1,9 @@
 # BOT Data Cache
 Cache/
 
+# Ignore everything in docs/reports
+docs/reports/*
+
 # Autosave files
 *.asv
 *.m~

+bot/+behavior/+item/EphysSession.m → code/+bot/+behavior/+item/EphysSession.m

@@ -1074,9 +1074,13 @@ function displayScalarObject(self)
         function tableRow = findManifestTableRow(obj, itemId)
             % Ensure ID is correct type
             itemId = uint32(round(itemId));
-
-            tableRow = bot.listSessions(obj.DATASET, obj.DATASET_TYPE, ...
-                "Id", itemId, "IncludeBehaviorOnly", true);
+            for i = 0:1
+                tableRow = bot.listSessions(obj.DATASET, obj.DATASET_TYPE, ...
+                    "Id", itemId, "IncludeBehaviorOnly", logical(i));
+                if ~isempty(tableRow)
+                    break
+                end
+            end
         end
     end
 

+bot/+behavior/+item/OphysSession.m → code/+bot/+behavior/+item/OphysSession.m

@@ -161,9 +161,13 @@ function initLinkedItems(obj)
         function tableRow = findManifestTableRow(obj, itemId)
             % Ensure ID is correct type
             itemId = uint32(round(itemId));
-
-            tableRow = bot.listSessions(obj.DATASET, obj.DATASET_TYPE, ...
-                "Id", itemId, "IncludeBehaviorOnly", true);
+            for i = 0:1
+                tableRow = bot.listSessions(obj.DATASET, obj.DATASET_TYPE, ...
+                    "Id", itemId, "IncludeBehaviorOnly", logical(i));
+                if ~isempty(tableRow)
+                    break
+                end
+            end
         end
     end
 

code/+bot/+internal/+util/toolboxdir.m

@@ -0,0 +1,8 @@
+function toolboxFolderPath = toolboxdir()
+
+    thisFolderPath = fileparts( mfilename('fullpath') );
+    filesepLoc = regexp(thisFolderPath, filesep);
+
+    % Get path of folder 3 levels up:
+    toolboxFolderPath = extractBefore(thisFolderPath, filesepLoc(end-2));
+end

+bot/getSessions.m → code/+bot/getSessions.m

@@ -87,9 +87,11 @@
         warning('off', 'BOT:ListSessions:BehaviorOnlyNotPresent')
         for iName = string(datasetNames')
             for jType = string(datasetTypes')
-                sessionTable = bot.listSessions(iName, jType, "IncludeBehaviorOnly", true);
-                ids = sessionTable.id;
-                sessionIdMap(ids)={{iName, jType}};
+                for k = 0:1
+                    sessionTable = bot.listSessions(iName, jType, "IncludeBehaviorOnly", logical(k));
+                    ids = sessionTable.id;
+                    sessionIdMap(ids)={{iName, jType}};
+                end
             end
         end
         warning('on', 'BOT:ListSessions:BehaviorOnlyNotPresent')

+bot/listSessions.m → code/+bot/listSessions.m

@@ -47,7 +47,7 @@
     % Note: This function can support returning sessions from multiple
     % datasets, but this functionality is currently not enabled
 
-    datasetNames = [dataset.Name];
+    datasetNames = dataset.Name;
     datasetType = string(datasetType);
 
     if options.IncludeBehaviorOnly

Contents.m → code/Contents.m

@@ -1,8 +1,8 @@
-% Brain Observatory Toolbox
+% Brain-Observatory-Toolbox
 % Version 0.9.4 19-feb-2024
 %
-% Copyright (c) 2017, Ethan Meyers
-% ----------------------------------
+% Copyright (c) 2024, Ethan Meyers
+% --------------------------------
 %
 % A MATLAB toolbox for accessing and using the public neural 
 % recording datasets from the Allen Brain Observatory.

tools/+bottools/projectdir.m

@@ -0,0 +1,4 @@
+function folderPath = botProjectdir()
+% projectdir - Get project root directory for a matlab toolbox code repository
+    folderPath = fileparts(fileparts(fileparts(mfilename('fullpath'))));
+end

tools/MLToolboxInfo.json

@@ -0,0 +1,14 @@
+{
+    "Namespace" : "bot",
+    "ToolboxOptions": {
+        "Identifier": "85a3255c-4ff5-42ef-9c10-b441318b4322",
+        "ToolboxName": "Brain-Observatory-Toolbox",
+        "AuthorName": "Ethan Meyers",
+        "AuthorEmail": "",
+        "AuthorCompany": "",
+        "Summary": "A MATLAB toolbox for interacting with the Allen Brain Observatory",
+        "Description": "A MATLAB toolbox for accessing and using the neural recording public datasets from the Allen Brain Observatory. Available datasets: Available datasets: Visual Coding, Visual Behavior (Neuropixels and 2-Photon Calcium Imaging)",
+        "MinimumMatlabRelease": "",
+        "MaximumMatlabRelease": ""
+    }
+}

tools/tasks/codecheckToolbox.m

@@ -0,0 +1,5 @@
+function codecheckToolbox()
+    installMatBox("commit")
+    projectRootDirectory = bottools.projectdir();
+    matbox.tasks.codecheckToolbox(projectRootDirectory)
+end