Merge branch 'develop' into pr-lattice

Author: grhkm21

.devcontainer/devcontainer.json

@@ -8,7 +8,7 @@
 	"remoteUser": "vscode",
 
 	// Setup conda environment
-	"onCreateCommand": ".devcontainer/onCreate-conda.sh",
+	"onCreateCommand": ".devcontainer/onCreate-conda.sh || true",
 
 	// Install additional features.
 	"features": {

.devcontainer/downstream-archlinux-latest/devcontainer.json

@@ -3,7 +3,7 @@
     "name": "archlinux:latest downstream Sage",
     "image": "archlinux:latest",
     // Create an empty bashrc to avoid the error "No such file or directory" when opening a terminal.
-    "onCreateCommand": "EXTRA_SYSTEM_PACKAGES='sagemath sagemath-doc' EXTRA_SAGE_PACKAGES='notebook pip' .devcontainer/onCreate.sh && touch ~/.bashrc",
+    "onCreateCommand": "sed -i '/^NoExtract/d' /etc/pacman.conf; EXTRA_SYSTEM_PACKAGES='sagemath sagemath-doc' EXTRA_SAGE_PACKAGES='notebook pip' .devcontainer/onCreate.sh && touch ~/.bashrc",
     // There's no SAGE_LOCAL, so remove the symlink 'prefix'.
     "updateContentCommand": "rm -f prefix && ln -sf /usr venv",
     "extensions": [

.devcontainer/onCreate-conda.sh

@@ -2,9 +2,8 @@
 set -e
 
 # Create conda environment
-./bootstrap-conda
 conda install mamba -n base -c conda-forge -y
-mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml
+mamba env create --file src/environment-dev-3.11-linux.yml || mamba env update --file src/environment-dev-3.11-linux.yml
 conda init bash
 
 # Build sage

.github/workflows/build.yml

@@ -172,11 +172,23 @@ jobs:
           MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
+      - name: Check that all modules can be imported
+        run: |
+          # The following command checks that all modules can be imported.
+          # The output also includes a long list of modules together with the number of tests in each module.
+          # This can be ignored.
+          ../sage -python -m pip install pytest-xdist
+          ../sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
+        working-directory: ./worktree-image/src
+        env:
+          # Increase the length of the lines in the "short summary"
+          COLUMNS: 120
+
       - name: Pytest
         if: contains(github.ref, 'pytest')
         run: |
           ../sage -python -m pip install coverage pytest-xdist
-          ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true
+          ../sage -python -m coverage run -m pytest -c tox.ini --doctest || true
         working-directory: ./worktree-image/src
         env:
           # Increase the length of the lines in the "short summary"

.github/workflows/ci-conda.yml

@@ -18,11 +18,11 @@ concurrency:
 jobs:
   test:
     name: Conda
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}-latest
 
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu, macos]
         python: ['3.9', '3.10', '3.11']
         # Optional environment is disabled for now as its not yet working
         # environment: [environment, environment-optional]
@@ -38,9 +38,6 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
 
-      - name: Create conda environment files
-        run: ./bootstrap-conda
-
       - name: Cache conda packages
         uses: actions/cache@v3
         with:
@@ -57,7 +54,7 @@ jobs:
           channels: conda-forge
           channel-priority: true
           activate-environment: sage
-          environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml
+          environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && 'macos' || 'linux' }}.yml
 
       - name: Print Conda environment
         shell: bash -l {0}

.github/workflows/ci-linux-incremental.yml

@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Get all packages that have changed
         id: changed-packages
-        uses: tj-actions/changed-files@v38
+        uses: tj-actions/changed-files@v41
         with:
           files_yaml: |
             configures:
@@ -63,6 +63,21 @@ jobs:
           echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
           echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+          path: worktree-base
+        if: github.base_ref
+      - name: Compute metrics
+        run: |
+          export PATH=build/bin:$PATH
+          if [ -d worktree-base ]; then
+            (echo "# $GITHUB_BASE_REF"; SAGE_ROOT=worktree-base sage-package metrics :all:) > base-metrics.txt
+            (echo "# $GITHUB_REF"; sage-package metrics :all:) > metrics.txt
+            diff --color=always --width=100 --side-by-side --left-column base-metrics.txt metrics.txt || true
+          else
+            sage-package metrics :all:
+          fi
 
   test:
     needs: [changed_files]
@@ -83,7 +98,7 @@ jobs:
       from_docker_target: "with-targets"
       from_docker_tag: "dev"
       docker_targets: "with-targets"
-      targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      targets: "${{needs.changed_files.outputs.uninstall_targets}} reconfigure ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
       tox_system_factors: >-
         ["ubuntu-focal",
          "ubuntu-jammy",
@@ -119,7 +134,7 @@ jobs:
       from_docker_target: "with-targets"
       from_docker_tag: "dev"
       docker_targets: "with-targets"
-      targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      targets: "${{needs.changed_files.outputs.uninstall_targets}} reconfigure ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
       # Only test systems with a usable system python (>= 3.9)
       # with only a small number of test failures as of 10.2.rc0
       tox_system_factors: >-

.github/workflows/conda-lock-update.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import subprocess
+
+script_dir = Path(__file__).resolve().parent
+root_dir = script_dir / '..' / '..'
+
+subprocess.run([str(root_dir / "bootstrap-conda")])
+
+platforms = {
+    "linux-64": "linux",
+    "linux-aarch64": "linux-aarch64",
+    "osx-64": "macos",
+    "osx-arm64": "macos-arm64"
+    #"win-64": "win",
+}
+pythons = ["3.9", "3.10", "3.11"]
+tags = ["", "-dev"]
+sources = ["", "src"]
+
+for platform_key, platform_value in platforms.items():
+    for python in pythons:
+        for tag in tags:
+            for src in sources:
+                env_file = root_dir / src / f"environment{tag}-{python}.yml"
+                lock_file = root_dir / src / f"environment{tag}-{python}-{platform_value}"
+
+                if not env_file.exists():
+                    continue
+
+                print(f"Updating lock file for {env_file} at {lock_file}", flush=True)
+                subprocess.run(["conda-lock", "--channel", "conda-forge", "--kind", "env", "--platform", platform_key, "--file", str(env_file), "--lockfile", str(lock_file), "--filename-template", str(lock_file)])

.github/workflows/doc-build-pdf.yml

@@ -57,7 +57,7 @@ jobs:
           export PATH="build/bin:$PATH"
           eval $(sage-print-system-package-command auto update)
           eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip)
-          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive)
+          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive texlive_luatex free_fonts xindy)
 
       - name: Add prebuilt tree as a worktree
         id: worktree

.github/workflows/doc-build.yml

@@ -161,7 +161,7 @@ jobs:
           export PATH="build/bin:$PATH"
           eval $(sage-print-system-package-command auto update)
           eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip)
-          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive)
+          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive texlive_luatex free_fonts xindy)
           export SAGE_USE_CDNS=yes
           export SAGE_LIVE_DOC=yes
           export SAGE_JUPYTER_SERVER=binder:sagemath/sage-binder-env/dev

.gitpod.yml

@@ -7,8 +7,7 @@ tasks:
   - name: Setup
     # Create conda environment, then configure and build sage
     init: >-
-      ./bootstrap-conda
-      && mamba env create --file src/environment-dev-3.11.yml --prefix venv
+      && mamba env create --file src/environment-dev-3.11-linux.yml --prefix venv
       && conda config --append envs_dirs $(pwd)
       && conda activate $(pwd)/venv
       && ./bootstrap