Merge pull request #2578 from PrincetonUniversity/devel

Devel

Author: kmantel

.github/actions/install-pnl/action.yml

@@ -49,18 +49,31 @@ runs:
           sed -i /modeci_mdf/d requirements.txt
           # pywinpty is a transitive dependency and v1.0+ removed support for x86 wheels
           # terminado >= 0.10.0 pulls in pywinpty >= 1.1.0
-          [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10"
+          # scipy >=1.9.2 doesn't provide win32 wheel and GA doesn't have working fortran on windows
+          # scikit-learn >= 1.1.3 doesn't provide win32 wheel
+          [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" "scikit-learn<1.1.3" "statsmodels<0.13.3" "jupyter-server<2" -c requirements.txt
         fi
 
     - name: Install updated package
       if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' }}
       shell: bash
       id: new_package
       run: |
-        export NEW_PACKAGE=`echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//'`
-        echo "::set-output name=new_package::$NEW_PACKAGE"
-        pip install "`echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1`"
-        pip show "$NEW_PACKAGE" | grep 'Version' | tee new_version.deps
+        # The branch name pattern is: "dependabot/pip/$TARGET/$PACKAGE{-gt-$MINVERSION,,}{-lt-$MAXVERSION,}{-$VERSION,}
+        # The expression below extracts just the $PACKAGE part
+        export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//' | sed 's/-[0-9\.]*$//' )
+        if grep "$NEW_PACKAGE" *requirements.txt; then 
+          echo "new_package=$NEW_PACKAGE" >> $GITHUB_OUTPUT
+          # save a list of all installed packages (including pip, wheel; it's never empty)
+          pip freeze --all > orig
+          pip install "$(echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1)"
+          pip show "$NEW_PACKAGE" | grep 'Version' | tee new_version.deps
+          # uninstall new packages but skip those from previous steps (pywinpty, terminado on windows x86)
+          # the 'orig' list is not empty (includes at least pip, wheel)
+          pip uninstall -y $(pip freeze -r orig | sed '1,/## /d')
+        else
+          echo "new_package=''" >> $GITHUB_OUTPUT
+        fi
 
     - name: Python dependencies
       shell: bash
@@ -81,9 +94,11 @@ runs:
         done
 
     - name: Check updated package
-      if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' }}
+      if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' && steps.new_package.outputs.new_package != '' }}
       shell: bash
       run: |
-        pip show ${{ steps.new_package.outputs.new_package }} | grep 'Version' | tee installed_version.deps
-        cmp -s new_version.deps installed_version.deps || echo "::error::Package version restricted by dependencies: ${{ steps.new_package.outputs.new_package }}"
-        diff new_version.deps installed_version.deps
+        if [ $(pip list | grep -o ${{ steps.new_package.outputs.new_package }} | wc -l) != "0" ] ; then
+          pip show ${{ steps.new_package.outputs.new_package }} | grep 'Version' | tee installed_version.deps
+          cmp -s new_version.deps installed_version.deps || echo "::error::Package version restricted by dependencies: ${{ steps.new_package.outputs.new_package }}"
+          diff new_version.deps installed_version.deps
+        fi

.github/actions/on-branch/action.yml

@@ -25,4 +25,4 @@ runs:
         git describe --always --tags
         export ON_BRANCH=$(git branch -a --contains ${{ github.ref }} | grep -q '^  remotes/origin/${{ inputs.branch }}$' && echo "${{ inputs.branch }}" || echo "")
         echo "Found out: ${ON_BRANCH}"
-        echo "::set-output name=on_branch::$ON_BRANCH"
+        echo "on_branch=$ON_BRANCH" >> $GITHUB_OUTPUT

.github/workflows/codeql.yml

@@ -3,11 +3,20 @@ name: "CodeQL"
 on:
   push:
     branches: [ "master", "devel" ]
+    paths-ignore:
+      - 'docs/**'
   pull_request:
-    branches: [ "master" ]
+    branches: [ "master", "devel" ]
   schedule:
     - cron: "14 21 * * 5"
 
+# run only the latest instance of this workflow job for the current branch/PR
+# cancel older runs
+# fall back to run id if not available (run id is unique -> no cancellations)
+concurrency:
+  group: ci-${{ github.ref || github.run_id }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze

.github/workflows/pnl-ci-docs.yml

@@ -8,6 +8,13 @@ on:
       - 'v*'
   pull_request:
 
+# run only the latest instance of this workflow job for the current branch/PR
+# cancel older runs
+# fall back to run id if not available (run id is unique -> no cancellations)
+concurrency:
+  group: ci-${{ github.ref || github.run_id }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   docs-build:
     runs-on: ${{ matrix.os }}
@@ -76,14 +83,14 @@ jobs:
       run: |
         python -m pip install -U pip
         python -m pip --version
-        echo ::set-output name=pip_cache_dir::$(python -m pip cache dir)
+        echo "pip_cache_dir=$(python -m pip cache dir)" | tee -a $GITHUB_OUTPUT
 
     - name: Wheels cache
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels
-        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }}
-        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2
+        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }}-${{ github.sha }}
+        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }}
 
     # We need to install all PNL deps since docs config imports psyneulink module
     - name: Install local, editable PNL package

.github/workflows/pnl-ci.yml

@@ -11,29 +11,60 @@ on:
       - 'v**'
   pull_request:
 
+env:
+  SELF_HOSTED_MACOS: ${{ secrets.SELF_HOSTED_MACOS }}
+  SELF_HOSTED_LINUX: ${{ secrets.SELF_HOSTED_LINUX }}
+  SELF_HOSTED_WINDOWS: ${{ secrets.SELF_HOSTED_WINDOWS }}
+
+# run only the latest instance of this workflow job for the current branch/PR
+# cancel older runs
+# fall back to run id if not available (run id is unique -> no cancellations)
+concurrency:
+  group: ci-${{ github.ref || github.run_id }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
+  # A job to select self-hosted runner if requested by an env var
+  select-runner:
+    runs-on: ubuntu-latest
+
+    outputs:
+      self_hosted_macos: ${{ steps.is_self_hosted.outputs.macos && 'macos' || '' }}
+      self_hosted_linux: ${{ steps.is_self_hosted.outputs.linux && 'linux' || '' }}
+      self_hosted_windows: ${{ steps.is_self_hosted.outputs.windows && 'windows' || '' }}
+
+    steps:
+    - name: Add macos
+      id: is_self_hosted
+      run: |
+        echo "macos=$SELF_HOSTED_MACOS" | tee -a $GITHUB_OUTPUT
+        echo "linux=$SELF_HOSTED_LINUX" | tee -a $GITHUB_OUTPUT
+        echo "windows=$SELF_HOSTED_WINDOWS" | tee -a $GITHUB_OUTPUT
+
+  # the main build job
   build:
-    runs-on: ${{ matrix.os }}
+    needs: select-runner
+    runs-on: ${{ (contains(needs.select-runner.outputs.*, matrix.os) && fromJSON(format('[ "self-hosted","{0}", "X64" ]', matrix.os))) || format('{0}-latest', matrix.os) }}
     strategy:
       fail-fast: false
       matrix:
         python-version: [3.7, 3.8, 3.9]
         python-architecture: ['x64']
         extra-args: ['']
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu, macos, windows]
         include:
           # add 32-bit build on windows
           - python-version: 3.8
             python-architecture: 'x86'
-            os: windows-latest
+            os: windows
           # code-coverage build on macos python 3.9
           - python-version: 3.9
-            os: macos-latest
+            os: macos
             extra-args: '--cov=psyneulink'
         exclude:
           # 3.7 is broken on macos-11, https://github.com/actions/virtual-environments/issues/4230
           - python-version: 3.7
-            os: macos-latest
+            os: macos
 
     steps:
     # increased fetch-depth and tag checkout needed to get correct
@@ -61,14 +92,14 @@ jobs:
       run: |
         python -m pip install -U pip
         python -m pip --version
-        echo ::set-output name=pip_cache_dir::$(python -m pip cache dir)
+        echo "pip_cache_dir=$(python -m pip cache dir)" | tee -a $GITHUB_OUTPUT
 
     - name: Wheels cache
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels
-        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }}
-        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2
+        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'dev_requirements.txt') }}-${{ github.sha }}
+        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'dev_requirements.txt') }}
 
     - name: Install local, editable PNL package
       uses: ./.github/actions/install-pnl

.github/workflows/test-release.yml

@@ -34,8 +34,8 @@ jobs:
         python setup.py sdist
         python setup.py bdist_wheel
         cd dist
-        echo ::set-output name=sdist::$(ls *.tar.gz)
-        echo ::set-output name=wheel::$(ls *.whl)
+        echo "sdist=$(ls *.tar.gz)" >> $GITHUB_OUTPUT
+        echo "wheel=$(ls *.whl)" >> $GITHUB_OUTPUT
 
     - name: Upload Python dist files
       uses: actions/upload-artifact@v3

.gitignore

@@ -196,4 +196,5 @@ tests/*.pdf
 # mypy cache
 .mypy_cache
 
-/tests/json/*.json
+/tests/mdf/*.json
+/tests/mdf/*.yml

Scripts/Models (Under Development)/N-back.py

@@ -0,0 +1,34 @@
+import numpy as np
+from psyneulink import *
+
+NUM_TRIALS = 48
+
+stims = np.array([x[0] for x in em.memory])
+contexts = np.array([x[1] for x in em.memory])
+cos = Distance(metric=COSINE)
+dist = Distance(metric=EUCLIDEAN)
+diffs = [np.sum([contexts[i+1] - contexts[1]]) for i in range(NUM_TRIALS)]
+diffs_1 = [np.sum([contexts[i+1] - contexts[i]]) for i in range(NUM_TRIALS)]
+diffs_2 = [np.sum([contexts[i+2] - contexts[i]]) for i in range(NUM_TRIALS-1)]
+dots = [[contexts[i+1] @ contexts[1]] for i in range(NUM_TRIALS)]
+dot_diffs_1 = [[contexts[i+1] @ contexts[i]] for i in range(NUM_TRIALS)]
+dot_diffs_2 = [[contexts[i+2] @ contexts[i]] for i in range(NUM_TRIALS-1)]
+angle = [cos([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)]
+angle_1 = [cos([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)]
+angle_2 = [cos([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)]
+euclidean = [dist([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)]
+euclidean_1 = [dist([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)]
+euclidean_2 = [dist([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)]
+print("STIMS:", stims, "\n")
+print("DIFFS:", diffs, "\n")
+print("DIFFS 1:", diffs_1, "\n")
+print("DIFFS 2:", diffs_2, "\n")
+print("DOT PRODUCTS:", dots, "\n")
+print("DOT DIFFS 1:", dot_diffs_1, "\n")
+print("DOT DIFFS 2:", dot_diffs_2, "\n")
+print("ANGLE: ", angle, "\n")
+print("ANGLE_1: ", angle_1, "\n")
+print("ANGLE_2: ", angle_2, "\n")
+print("EUCILDEAN: ", euclidean, "\n")
+print("EUCILDEAN 1: ", euclidean_1, "\n")
+print("EUCILDEAN 2: ", euclidean_2, "\n")