Merge tag '10.1.beta7' into t/34346/document_new_conda_packages_sagemath_sirocco_and_sagemath_bliss

SageMath version 10.1.beta7, Release Date: 2023-07-20

Author: Matthias Koeppe

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -3,33 +3,11 @@ description: Report a bug
 title: "<title>"
 labels: "t: bug"
 body:
-  - type: checkboxes
-    attributes:
-      label: Is there an existing issue for this?
-      description: Please search to see if an issue already exists for the bug you encountered.
-      options:
-      - label: I have searched the existing issues for a bug report that matches the one I want to file, without success.
-        required: true
-  - type: checkboxes
-    attributes:
-      label: Did you read the documentation and troubleshoot guide?
-      description: Please read [README.md](https://github.com/sagemath/sage/blob/develop/README.md) and [the Troubleshooting section in the Installation Guide](https://doc.sagemath.org/html/en/installation/troubles.html).
-      options:
-      - label: I have read the documentation and troubleshoot guide
-        required: true
-  - type: textarea
+  - type: markdown
     attributes:
-      label: Environment
-      description: |
-        examples:
-          - **OS**: Ubuntu 20.04
-          - Sage Version: 9.2
       value: |
-          - **OS**:
-          - **Sage Version**: 
-      render: markdown
-    validations:
-      required: true
+        * Please search to see if an issue already exists for the bug you encountered.
+        * Please read [README.md](https://github.com/sagemath/sage/blob/develop/README.md) and [the Troubleshooting section in the Installation Guide](https://doc.sagemath.org/html/en/installation/troubles.html).
   - type: textarea
     attributes:
       label: Steps To Reproduce
@@ -60,3 +38,24 @@ body:
         Links? References? Anything that will give us more context about the issue you are encountering!
     validations:
       required: false
+  - type: textarea
+    attributes:
+      label: Environment
+      description: |
+        examples:
+          - **OS**: Ubuntu 20.04
+          - Sage Version: 9.2
+      value: |
+          - **OS**:
+          - **Sage Version**:
+      render: markdown
+    validations:
+      required: true
+  - type: checkboxes
+    attributes:
+      label: Checklist
+      options:
+      - label: I have searched the existing issues for a bug report that matches the one I want to file, without success.
+        required: true
+      - label: I have read the documentation and troubleshoot guide
+        required: true

.github/ISSUE_TEMPLATE/failure_building_from_source.yml

@@ -4,20 +4,11 @@ title: "<title>"
 labels: ['c: build', 't: bug']
 assignees: []
 body:
-  - type: checkboxes
+  - type: markdown
     attributes:
-      label: Is there an existing issue for this?
-      description: Please search to see if an issue already exists for the bug you encountered.
-      options:
-      - label: I have searched the existing issues for a bug report that matches the one I want to file, without success.
-        required: true
-  - type: checkboxes
-    attributes:
-      label: Did you read the documentation and troubleshoot guide?
-      description: Please read [README.md](https://github.com/sagemath/sage/blob/develop/README.md) and [the Troubleshooting sectionin the Installation Guide](https://doc.sagemath.org/html/en/installation/troubles.html).
-      options:
-      - label: I have read the documentation and troubleshoot guide
-        required: true
+      value: |
+        * Please search to see if an issue already exists for the bug you encountered.
+        * Please read [README.md](https://github.com/sagemath/sage/blob/develop/README.md) and [the Troubleshooting section in the Installation Guide](https://doc.sagemath.org/html/en/installation/troubles.html).
   - type: textarea
     attributes:
       label: Environment
@@ -65,3 +56,11 @@ body:
         Links? References? Anything that will give us more context about the issue you are encountering!
     validations:
       required: false
+  - type: checkboxes
+    attributes:
+      label: Checklist
+      options:
+      - label: I have searched the existing issues for a bug report that matches the one I want to file, without success.
+        required: true
+      - label: I have read the documentation and troubleshoot guide
+        required: true

.github/ISSUE_TEMPLATE/feature_report.yml

@@ -3,13 +3,10 @@ description: Suggest an idea
 title: "<title>"
 labels: "t: enhancement"
 body:
-  - type: checkboxes
+  - type: markdown
     attributes:
-      label: Is there an existing issue for this?
-      description: Please search to see if an issue already exists for the bug you encountered.
-      options:
-      - label: I have searched the existing issues for a bug report that matches the one I want to file, without success.
-        required: true
+      value: |
+        * Please search to see if an issue already exists for the bug you encountered.
   - type: textarea
     attributes:
       label: Problem Description
@@ -34,3 +31,9 @@ body:
       description: Add any other context about the problem here.
     validations:
       required: false
+  - type: checkboxes
+    attributes:
+      label: Is there an existing issue for this?
+      options:
+      - label: I have searched the existing issues for a bug report that matches the one I want to file, without success.
+        required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,17 +1,20 @@
-<!-- Please provide a concise, informative and self-explanatory title. -->
-<!-- Don't put issue numbers in the title. Put it in the Description below. -->
-<!-- For example, instead of "Fixes #12345", use "Add a new method to multiply two integers" -->
-
-### :books: Description
+<!-- ^^^^^
+Please provide a concise, informative and self-explanatory title.
+Don't put issue numbers in there, do this in the PR body below.
+For example, instead of "Fixes #1234" use "Introduce new method to calculate 1+1"
+-->
+<!-- Describe your changes here in detail -->
 
-<!-- Describe your changes here in detail. -->
 <!-- Why is this change required? What problem does it solve? -->
 <!-- If this PR resolves an open issue, please link to it here. For example "Fixes #12345". -->
 <!-- If your change requires a documentation PR, please link it appropriately. -->
 
 ### :memo: Checklist
 
-<!-- Put an `x` in all the boxes that apply. It should be `[x]` not `[x ]`. -->
+<!-- Put an `x` in all the boxes that apply. -->
+<!-- If your change requires a documentation PR, please link it appropriately -->
+<!-- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
+<!-- Feel free to remove irrelevant items. -->
 
 - [ ] The title is concise, informative, and self-explanatory.
 - [ ] The description explains in detail what this PR is about.

.github/workflows/build.yml

@@ -32,50 +32,84 @@ jobs:
         id: checkout
         uses: actions/checkout@v3
 
-      - name: Prepare
+      - name: Update system packages
         id: prepare
         run: |
-          # Install test tools.
-          if apt-get update && apt-get install -y git python3-venv; then
-             # Debian-specific temporary code:
-             # Installation of python3-venv can be removed as soon as a
-             # base image with a release including #33822 is available
-             :
-          else
-             export PATH="build/bin:$PATH"
-             eval $(sage-print-system-package-command auto update)
-             eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git)
-          fi
-          # Reuse built SAGE_LOCAL contained in the Docker image 
-          ./bootstrap
-          ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-editable --enable-download-from-upstream-url
-      
+          export PATH="build/bin:$PATH"
+          eval $(sage-print-system-package-command auto update)
+          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git)
+
+      - name: Add prebuilt tree as a worktree
+        id: worktree
+        run: |
+          set -ex
+          git config --global user.email "[email protected]"
+          git config --global user.name "Build & Test workflow"
+          git config --global --add safe.directory $(pwd)
+          # If actions/checkout downloaded our source tree using the GitHub REST API
+          # instead of with git (because do not have git installed in our image),
+          # we first make the source tree a repo.
+          if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
+          # Tag this state of the source tree "new". This is what we want to build and test.
+          git tag -f new
+          # Our container image contains a source tree in /sage with a full build of Sage.
+          # But /sage is not a git repository.
+          # We make /sage a worktree whose index is at tag "new".
+          # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
+          # Then we update worktree and index with "git reset --hard new".
+          # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
+          # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
+          # The changed files now show up as uncommitted changes.
+          # The final "git add -N" makes sure that files that were added in "new" do not show
+          # as untracked files, which would be removed by "git clean -fx".
+          git worktree add --detach worktree-image
+          rm -rf /sage/.git && mv worktree-image/.git /sage/
+          rm -rf worktree-image && ln -s /sage worktree-image
+          if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
+          (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+
+      - name: Incremental build, test changed files (sage -t --new)
+        id: incremental
+        run: |
+          # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
+          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
+          ./bootstrap && make build && ./sage -t --new -p2
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2
+          SAGE_NUM_THREADS: 2
+
       - name: Build and test modularized distributions
-        if: always() && steps.prepare.outcome == 'success'
+        if: always() && steps.worktree.outcome == 'success'
         run: make V=0 tox && make pypi-wheels
+        working-directory: ./worktree-image
         env:
           MAKE: make -j2
           SAGE_NUM_THREADS: 2
 
       - name: Set up node to install pyright
-        if: always() && steps.prepare.outcome == 'success'
+        if: always() && steps.worktree.outcome == 'success'
         uses: actions/setup-node@v3
         with:
           node-version: '12'
 
       - name: Install pyright
-        if: always() && steps.prepare.outcome == 'success'
+        if: always() && steps.worktree.outcome == 'success'
         # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239
         run: npm install -g [email protected]
 
       - name: Static code check with pyright
-        if: always() && steps.prepare.outcome == 'success'
-        run: pyright    
-      
-      - name: Build
+        if: always() && steps.worktree.outcome == 'success'
+        run: pyright
+        working-directory: ./worktree-image
+
+      - name: Build (fallback to non-incremental)
         id: build
-        if: always() && steps.prepare.outcome == 'success'
-        run: make build
+        if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
+        run: |
+          set -ex
+          ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
+        working-directory: ./worktree-image
         env:
           MAKE: make -j2
           SAGE_NUM_THREADS: 2
@@ -85,27 +119,28 @@ jobs:
         run: |
           ../sage -python -m pip install coverage pytest-xdist
           ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true
-        working-directory: ./src
+        working-directory: ./worktree-image/src
         env:
           # Increase the length of the lines in the "short summary"
           COLUMNS: 120
 
-      - name: Test
-        if: always() && steps.build.outcome == 'success'
+      - name: Test all files (sage -t --all --long)
+        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
         run: |
           ../sage -python -m pip install coverage
-          ../sage -python -m coverage run ./bin/sage-runtests --all -p2 --random-seed=286735480429121101562228604801325644303
-        working-directory: ./src
+          ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303
+        working-directory: ./worktree-image/src
 
       - name: Prepare coverage results
-        if: always() && steps.build.outcome == 'success'
+        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
         run: |
           ./venv/bin/python3 -m coverage combine src/.coverage/
           ./venv/bin/python3 -m coverage xml
           find . -name *coverage*
-      
+        working-directory: ./worktree-image
+
       - name: Upload coverage to codecov
-        if: always() && steps.build.outcome == 'success'
+        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
         uses: codecov/codecov-action@v3
         with:
-          files: ./coverage.xml
+          files: ./worktree-image/coverage.xml

.github/workflows/ci-conda.yml

@@ -34,7 +34,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
         python: ['3.9', '3.10', '3.11']
-        conda-env: [environment, environment-optional]
+        # Optional environment is disabled for now as its not yet working
+        # environment: [environment, environment-optional]
+        conda-env: [environment]
 
     steps:
       - uses: actions/checkout@v3
@@ -100,19 +102,20 @@ jobs:
           SAGE_NUM_THREADS: 2
 
       - name: Verify dependencies
-        if: always()
+        if: success() || failure()
         shell: bash -l {0}
         run: pip check
 
       - name: Test
+        if: success() || failure()
         shell: bash -l {0}
         run: ./sage -t --all -p0
 
       - name: Print logs
+        if: always()
         run: |
           for file in $(find . -type f -name "*.log"); do
               echo "::group::$file"
               cat "$file"
               echo "::endgroup::"
           done
-        if: always()