Merge branch 'develop' into refactor_drinfeld_modules_classes

Author: DavidAyotte

.ci/merge-fixes.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Merge open PRs from sagemath/sage labeled "blocker".
+REPO="sagemath/sage"
+GH="gh -R $REPO"
+PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number')"
+if [ -z "$PRs" ]; then
+    echo 'Nothing to do: Found no open PRs with "blocker" status.'
+else
+    echo "Found PRs: " $PRs
+    export GIT_AUTHOR_NAME="ci-sage workflow"
+    export GIT_AUTHOR_EMAIL="[email protected]"
+    export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+    export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+    git tag -f test_base
+    git commit -q -m "Uncommitted changes" --no-allow-empty -a
+    for a in $PRs; do
+        echo "::group::Merging PR https://github.com/$REPO/pull/$a"
+        git tag -f test_head
+        $GH pr checkout -b pr-$a $a
+        git fetch --unshallow --all
+        git checkout -q test_head
+        if git merge --no-edit --squash -q pr-$a; then
+            echo "::endgroup::"
+            if git commit -q -m "Merge https://github.com/$REPO/pull/$a" -a --no-allow-empty; then
+                echo "Merged #$a"
+            else
+                echo "Empty, skipped"
+            fi
+        else
+            echo "::endgroup::"
+            echo "Failure merging #$a, resetting"
+            git reset --hard
+        fi
+    done
+    git log test_base..HEAD
+fi

.github/workflows/build.yml

@@ -2,8 +2,11 @@ name: Build & Test
 
 on:
   pull_request:
+  merge_group:
   push:
-    branches: ['**']
+    branches:
+      - master
+      - develop
     # Ignore pushes on tags to prevent two uploads of codecov reports
     tags-ignore: ['**']
   workflow_dispatch:
@@ -24,9 +27,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  get_ci_fixes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Store CI fixes in upstream artifact
+        run: |
+          mkdir -p upstream
+          if git format-patch --stdout test_base > ci_fixes.patch; then
+            cp ci_fixes.patch upstream/
+          fi
+      - uses: actions/upload-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
   build:
     runs-on: ubuntu-latest
     container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
+    needs: [get_ci_fixes]
     steps:
       - name: Checkout
         id: checkout
@@ -68,6 +94,19 @@ jobs:
           if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
           (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
 
+      - name: Download upstream artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
+      - name: Apply CI fixes from sagemath/sage
+        # After applying the fixes, make sure all changes are marked as uncommitted changes.
+        run: |
+          if [ -r upstream/ci_fixes.patch ]; then
+            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
+          fi
+
       - name: Incremental build, test changed files (sage -t --new)
         id: incremental
         run: |
@@ -76,15 +115,15 @@ jobs:
           ./bootstrap && make build && ./sage -t --new -p2
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Build and test modularized distributions
         if: always() && steps.worktree.outcome == 'success'
         run: make V=0 tox && make pypi-wheels
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Set up node to install pyright
@@ -123,7 +162,7 @@ jobs:
           make build
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Pytest

.github/workflows/ci-conda.yml

@@ -7,13 +7,6 @@ on:
     branches:
       - 'public/build/**-runci'
   pull_request:
-    types:
-      # Defaults
-      - opened
-      - synchronize
-      - reopened
-      # When a CI label is added
-      - labeled
   workflow_dispatch:
     # Allow to run manually
 
@@ -27,20 +20,9 @@ jobs:
     name: Conda
     runs-on: ${{ matrix.os }}
 
-    # Run on push, workflow dispatch and when certain labels are added or are present
-    if: |
-      github.event_name != 'pull_request' ||
-      ((github.event.action != 'labeled' &&
-         (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
-          contains(github.event.pull_request.labels.*.name, 'c: packages: optional') ||
-          contains(github.event.pull_request.labels.*.name, 's: run conda ci'))) ||
-       (github.event.action == 'labeled' &&
-         (github.event.label.name == 'c: packages: optional' ||
-          github.event.label.name == 'c: packages: standard' ||
-          github.event.label.name == 's: run conda ci')))
-
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event_name == 'pull_request' }}
+      max-parallel: ${{ github.event_name == 'pull_request' && 2 || 6 }}
       matrix:
         os: [ubuntu-latest, macos-latest]
         python: ['3.9', '3.10', '3.11']
@@ -51,6 +33,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Check for Miniconda
         id: check_conda
         run: echo ::set-output name=installed::$CONDA
@@ -99,9 +87,6 @@ jobs:
           echo "::remove-matcher owner=configure-system-package-warning::"
           echo "::remove-matcher owner=configure-system-package-error::"
 
-          # Manually install ptyprocess for now, until https://github.com/sagemath/sage/issues/32147 is fixed
-          pip install --no-build-isolation -v -v ptyprocess==0.5.1
-
       - name: Build
         shell: bash -l {0}
         run: |

.github/workflows/ci-cygwin-standard.yml

@@ -60,7 +60,8 @@ jobs:
           echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
           echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([-_.a-z0-9]*)/[^ ]* *,\2-ensure,;'; done | sort -u))" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
-  minimal:
+
+  test:
     needs: [changed_files]
     if: |
       github.event_name != 'pull_request' ||
@@ -94,3 +95,38 @@ jobs:
           ["standard",
            "minimal"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
+
+  site:
+    needs: [changed_files]
+    if: |
+      github.event_name != 'pull_request' ||
+      ((github.event.action != 'labeled' &&
+         (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
+          contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) ||
+       (github.event.action == 'labeled' &&
+         (github.event.label.name == 'c: packages: optional' ||
+          github.event.label.name == 'c: packages: standard')))
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets"
+      targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      # Only test systems with a usable system python (>= 3.9)
+      tox_system_factors: >-
+        ["ubuntu-jammy",
+         "ubuntu-mantic",
+         "debian-bullseye",
+         "debian-bookworm",
+         "fedora-33",
+         "fedora-38",
+         "gentoo-python3.11",
+         "archlinux",
+         "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard-sitepackages"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/