Merge branch '8.4.x' into rsync

Author: wxtim

.github/workflows/1_create_release_pr.yml

@@ -11,6 +11,19 @@ on:
         required: false
         default: 'master'
 
+concurrency:
+  # Only let this run 1 at a time
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  FORCE_COLOR: 2
+  PIP_PROGRESS_BAR: off
+
 jobs:
   create-release-pr:
     runs-on: ubuntu-latest
@@ -54,6 +67,6 @@ jobs:
       - name: Create pull request
         uses: cylc/release-actions/stage-1/create-release-pr@v1
         with:
-          test-workflows: test_fast.yml, test_functional.yml, bash.yml, test_manylinux.yml, test_conda-build.yml
+          test-workflows: test_fast.yml, test_functional.yml, bash.yml, test_conda-build.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/2_auto_publish_release.yml

@@ -7,7 +7,18 @@ on:
     # NOTE: While this is too generic, we use the `if` condition of the job to narrow it down
     # NOTE: Don't use `branches` as we might create release on any branch
 
+concurrency:
+  # Only let this run 1 at a time
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
 env:
+  FORCE_COLOR: 2
+  PIP_PROGRESS_BAR: off
   # Best not to include the GH token here, only do it for the steps that need it
   MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }}
   CHANGELOG_FILE: CHANGES.md

.github/workflows/bash.yml

@@ -31,6 +31,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
+env:
+  FORCE_COLOR: 2
+  PIP_PROGRESS_BAR: off
+
 jobs:
   bash-docker:
     runs-on: ubuntu-latest

.github/workflows/build.yml

@@ -10,6 +10,20 @@ on:
       - 'MANIFEST.in'     # check packaging
       - 'pyproject.toml'  # check build config
       - 'setup.cfg'       # check deps and project config
+      - '.gitignore'
+      - '.github/workflows/build.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
+env:
+  FORCE_COLOR: 2
+  PIP_PROGRESS_BAR: off
 
 jobs:
   test:
@@ -18,29 +32,31 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['ubuntu-latest']
-        python: ['3.8', '3.9', '3.10', '3.11']
-        include:
-          - os: 'ubuntu-22.04'
-            python: '3.7'
+        os: ['ubuntu-latest', 'macos-latest']
+        python: ['3.7', '3.8', '3.9', '3.10', '3']
+        exclude:
           - os: 'macos-latest'
-            python: '3.8'
+            python: '3.7'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: ${{ matrix.python }}
+          cache-environment: true
+          post-cleanup: 'all'
+          environment-name: cylc-build
+          create-args: >-
+            python=${{ matrix.python }}
 
       - name: Build
         uses: cylc/release-actions/build-python-package@v1
 
       - name: Inspect
         run: |
           unzip -l dist/*.whl | tee files
-          grep 'cylc_flow.*.dist-info/COPYING' files
+          grep -E 'cylc_flow.*.dist-info/.*COPYING' files
           grep 'cylc/flow/py.typed' files
           grep 'cylc/flow/etc' files
           grep 'cylc/flow/etc/cylc-completion.bash' files

.github/workflows/test_conda-build.yml

@@ -13,6 +13,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
+env:
+  FORCE_COLOR: 2
+
 jobs:
   test_conda_install:
     runs-on: ubuntu-latest

.github/workflows/test_fast.yml

@@ -12,6 +12,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}"
+
+env:
+  PIP_PROGRESS_BAR: off
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -20,43 +27,41 @@ jobs:
       fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.8', '3.10', '3.11', '3']
+        python-version: ['3.7', '3.8', '3.10', '3.11', '3.12', '3']
         include:
-          - os: 'ubuntu-22.04'
-            python-version: '3.7'
           - os: 'macos-latest'
-            python-version: '3.9'  # oldest supported version
+            python-version: '3.8'  # oldest version with arm64 builds
           # non-utc timezone test
           - os: 'ubuntu-latest'
             python-version: '3.9'  # not the oldest, not the most recent version
             time-zone: 'XXX-09:35'
-
     env:
       TZ: ${{ matrix.time-zone }}
       PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Configure Python
-        uses: actions/setup-python@v5
+      - name: Install System Dependencies
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Apt-Get Install
-        if: startsWith(matrix.os, 'ubuntu')
+          cache-environment: true
+          post-cleanup: 'all'
+          environment-name: cylc-fast-test
+          create-args: >-
+            python=${{ matrix.python-version }}
+            pip
+            bash
+            subversion
+            sqlite
+
+      - name: Install Python Dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y sqlite3
+          pip install -e ."[all]"
 
       - name: Patch DNS
         uses: cylc/release-actions/patch-dns@v1
 
-      - name: Install
-        run: |
-          pip install -e ."[all]"
-
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
@@ -99,21 +104,28 @@ jobs:
   lint:
     runs-on: 'ubuntu-latest'
     timeout-minutes: 10
+    strategy:
+      matrix:
+        python-version: ['3']
+    env:
+      FORCE_COLOR: 2
     steps:
-      - name: Apt-Get Install
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y shellcheck
-
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Configure Python
-        uses: actions/setup-python@v5
+      - name: Install System Dependencies
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: 3.x
-
-      - name: Install
+          cache-environment: true
+          post-cleanup: 'all'
+          environment-name: cylc-lint-test
+          create-args: >-
+            python=${{ matrix.python-version }}
+            pip
+            bash
+            shellcheck
+
+      - name: Install Python Dependencies
         run: |
           pip install -e ."[tests]"
 

.github/workflows/test_functional.yml

@@ -32,6 +32,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}"
+
+env:
+  FORCE_COLOR: 2
+  PIP_PROGRESS_BAR: off
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -56,34 +64,34 @@ jobs:
             platform: '_local_background*'
           # tests/k
           - name: 'flaky'
-            os: 'ubuntu-22.04'
+            os: 'ubuntu-latest'
             python-version: '3.7'
             test-base: 'tests/k'
             chunk: '1/1'
             platform: '_local_background* _local_at*'
           # remote platforms
           - name: '_remote_background_indep_poll'
-            os: 'ubuntu-22.04'
+            os: 'ubuntu-latest'
             python-version: '3.7'
             test-base: 'tests/f tests/k'
             chunk: '1/1'
             platform: '_remote_background_indep_poll _remote_at_indep_poll'
           - name: '_remote_background_indep_tcp'
-            os: 'ubuntu-22.04'
+            os: 'ubuntu-latest'
             test-base: 'tests/f tests/k'
             python-version: '3.7'
             chunk: '1/1'
             platform: '_remote_background_indep_tcp _remote_at_indep_tcp'
           # macos
           - name: 'macos 1/5'
             os: 'macos-latest'
-            python-version: '3.9'
+            python-version: '3.8'  # oldest available
             test-base: 'tests/f'
             chunk: '1/5'
             platform: '_local_background*'
           - name: 'macos 2/5'
             os: 'macos-latest'
-            python-version: '3.9'
+            python-version: '3'  # newest available
             test-base: 'tests/f'
             chunk: '2/5'
             platform: '_local_background*'
@@ -101,9 +109,21 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Configure Python
-        uses: actions/setup-python@v5
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          cache-environment: true
+          post-cleanup: 'all'
+          environment-name: cylc-functional-test
+          create-args: >-
+            python=${{ matrix.python-version }}
+            pip
+            bash
+            coreutils
+            grep
+            sed
+            sqlite
+            subversion
+            tree
 
       - name: Create global config
         run: |
@@ -128,36 +148,11 @@ jobs:
           __HERE__
           cat "$GLOBAL_CFG_PATH"
 
-      - name: Brew Install
-        if: startsWith(matrix.os, 'macos')
-        run: |
-          # install system deps
-          brew update
-          brew install bash coreutils gnu-sed grep
-
-          # add GNU coreutils and sed to the user PATH
-          # (see instructions in brew install output)
-          echo "$(brew --prefix)/opt/coreutils/libexec/gnubin" >> "${GITHUB_PATH}"
-          echo "$(brew --prefix)/opt/grep/libexec/gnubin" >> "${GITHUB_PATH}"
-          echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> "${GITHUB_PATH}"
-
-          # add coreutils to the bashrc too (for jobs)
-          cat >> "${HOME}/.bashrc" <<__HERE__
-          PATH="$(brew --prefix)/opt/coreutils/libexec/gnubin:$PATH"
-          PATH="$(brew --prefix)/opt/grep/libexec/gnubin:$PATH"
-          PATH="$(brew --prefix)/opt/gnu-sed/libexec/gnubin:$PATH"
-          export PATH
-          __HERE__
-
-      - name: Apt-Get Install
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y sqlite3 tree at
-
       - name: Add .github/bin/ to PATH
         # Sets up mocked mail command & any other custom executables
-        run: echo "${{ github.workspace }}/.github/bin" >> $GITHUB_PATH
+        # Adding to $GITHUB_PATH does not work when using setup-micromamba and/or login shell
+        run: |
+          echo "export PATH=\"${{ github.workspace }}/.github/bin:$PATH\"" >> ~/.bash_profile
 
       - name: Install
         run: |
@@ -167,6 +162,8 @@ jobs:
       - name: Configure Atrun
         if: contains(matrix.platform, '_local_at')
         run: |
+          sudo apt-get update
+          sudo apt-get install -y at
           cat >> "$GLOBAL_CFG_PATH" << __HERE__
             [platforms]
                 [[_local_at_indep_tcp]]