Merge branch 'develop' into imports-total

Author: tobiasdiez

.ci/write-dockerfile.sh

@@ -275,7 +275,7 @@ cat <<EOF
 FROM with-system-packages AS bootstrapped
 #:bootstrapping:
 RUN rm -rf /new /sage/.git
-$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap configure.ac sage .homebrew-build-env tox.ini .gitignore /new/
+$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap conftest.py configure_wrapper configure.ac sage .homebrew-build-env tox.ini .gitignore /new/
 $ADD config/config.rpath /new/config/config.rpath
 $ADD src/doc/bootstrap /new/src/doc/bootstrap
 $ADD src/bin /new/src/bin

.devcontainer/onCreate-conda.sh

@@ -10,5 +10,4 @@ mamba env create -y --file environment-3.11-linux.yml || mamba env update -y --f
 conda init bash
 
 # Build sage
-conda run -n sage-dev ./bootstrap
-conda run -n sage-dev pip install --no-build-isolation -v -v -e ./src
+conda run -n sage-dev pip install --no-build-isolation -v -v -e .

.github/workflows/build.yml

@@ -15,7 +15,7 @@ on:
       platform:
         description: 'Platform'
         required: true
-        default: 'ubuntu-jammy-standard'
+        default: 'ubuntu-noble-standard'
       docker_tag:
         description: 'Docker tag'
         required: true
@@ -29,7 +29,7 @@ concurrency:
 #
 # The three workflows:
 #
-#  - build.yml (with jobs test-new, test-mod, test-long),
+#  - build.yml (with jobs test-new, test-long),
 #  - doc-build.yml,
 #  - doc-build-pdf.yml
 #
@@ -50,7 +50,7 @@ concurrency:
 # This baseline is transparently improved by our use of the GH Actions cache,
 # see https://docs.docker.com/build/ci/github-actions/cache/#cache-backend-api.
 #
-# Jobs test-mod and test-long are only started after test-new completed;
+# Jobs test-long is only started after test-new completed;
 # but the workflows doc-build.yml and doc-build-pdf.yml are started independently.
 #
 #  - When nothing is cached and the 3 workflows are launched in parallel,
@@ -68,293 +68,16 @@ concurrency:
 
 env:
   # Adapted from docker.yml
-  TOX_ENV:                "docker-${{ github.event.inputs.platform || 'ubuntu-jammy-standard' }}-incremental"
-  BUILD_IMAGE:            "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-jammy-standard' }}-with-targets:ci"
+  TOX_ENV:                "docker-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-incremental"
+  BUILD_IMAGE:            "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-with-targets:ci"
   FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/"
   FROM_DOCKER_TARGET:     "with-targets"
   FROM_DOCKER_TAG:        ${{ github.event.inputs.docker_tag || 'dev'}}
   EXTRA_CONFIGURE_ARGS:   --enable-fat-binary
 
 jobs:
-  test-new:
-    runs-on: ubuntu-latest
-    outputs:
-      build_targets:     ${{ steps.build-targets.outputs.build_targets }}
-    services:
-      # https://docs.docker.com/build/ci/github-actions/local-registry/
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
-    steps:
-      - name: Maximize build disk space
-        uses: easimon/maximize-build-space@v10
-        with:
-          # need space in /var for Docker images
-          root-reserve-mb:      30000
-          remove-dotnet:        true
-          remove-android:       true
-          remove-haskell:       true
-          remove-codeql:        true
-          remove-docker-images: true
-
-      - name: Checkout
-        id: checkout
-        uses: actions/checkout@v4
-
-      - name: Get changed files and packages
-        id: changed-files
-        uses: tj-actions/changed-files@v45
-        with:
-          # File extensions for doctests per sage.doctest.control.skipfile
-          files_yaml: |
-            configures:
-              - 'build/pkgs/*/spkg-configure.m4'
-            pkgs:
-              - 'build/pkgs/**'
-              - '!build/pkgs/_**'
-              - '!build/pkgs/configure/**'
-              - 'pkgs/**'
-            doctests:
-              - 'src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}'
-              - '!src/{setup,conftest*}.py'
-
-      - name: Determine targets to build
-        id: build-targets
-        run: |
-          uninstall_targets=$(echo $(
-            for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do
-              # Extract package name from the file path and append '-uninstall'
-              echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'
-            done | sort -u  # Sort and ensure uniqueness
-          ))
-          build_targets=$(echo $(
-            for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do
-              # Extract package name, replace '-' with '_', and strip extra parts from the path
-              SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;')
-              # Check if key files exist in the package directory
-              if [ -f "build/pkgs/$SPKG/checksums.ini" ] || \
-                 [ -f "build/pkgs/$SPKG/requirements.txt" ] || \
-                 [ -f "build/pkgs/$SPKG/spkg-install" ]; then
-                echo "$SPKG-ensure"  # add the "$SPKG-ensure" target
-              fi
-            done | sort -u  # Sort and ensure uniqueness
-          ))
-          if [ -n "$uninstall_targets" ]; then
-            echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
-          else
-            echo "build_targets=$build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
-          fi
-          cat $GITHUB_OUTPUT
-
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.base_ref }}
-          path: worktree-base
-        if: github.base_ref && steps.changed-files.outputs.pkgs_all_changed_files
-
-      - name: Compute metrics
-        run: |
-          export PATH=build/bin:$PATH
-          if [ -d worktree-base ]; then
-            (echo "# $GITHUB_BASE_REF"; SAGE_ROOT=worktree-base sage-package metrics :all:) > base-metrics.txt
-            (echo "# $GITHUB_REF"; sage-package metrics :all:) > metrics.txt
-            diff --color=always --width=100 --side-by-side --left-column base-metrics.txt metrics.txt || true
-          else
-            sage-package metrics :all:
-          fi
-
-      - name: Install test prerequisites
-        # From docker.yml
-        run: |
-          sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
-          sudo apt-get clean
-          df -h
-
-      - name: Merge CI fixes from sagemath/sage
-        # From docker.yml
-        # This step needs to happen after the commit sha is put in DOCKER_TAG
-        # so that multi-stage builds can work correctly.
-        run: |
-          mkdir -p upstream
-          .ci/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log
-        env:
-          GH_TOKEN: ${{ github.token }}
-          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
-
-      # Building
-
-      - name: Generate Dockerfile
-        # From docker.yml
-        run: |
-          tox -e ${{ env.TOX_ENV }}
-          cp .tox/${{ env.TOX_ENV }}/Dockerfile .
-        env:
-          # Only generate the Dockerfile, do not run 'docker build' here
-          DOCKER_TARGETS: ""
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver-opts: network=host
-
-      - name: Build Docker image
-        id: image
-        uses: docker/build-push-action@v6
-        with:
-          # push and load may not be set together at the moment
-          #
-          # We are using "push" (to the local registry) because it was
-          # more reliable than "load", for which we observed random failure
-          # conditions in which the built image could not be found.
-          #
-          push:       ${{ steps.changed-files.outputs.doctests_all_changed_files && true || false }}
-          load:       false
-          context:    .
-          tags:       ${{ env.BUILD_IMAGE }}
-          target:     with-targets
-          cache-from: type=gha
-          cache-to:   type=gha,mode=max
-          build-args: |
-            NUMPROC=6
-            USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
-            TARGETS_PRE=build/make/Makefile
-            TARGETS=${{ steps.build-targets.outputs.build_targets }}
-
-      - name: Start container
-        id: container
-        # Try to continue when "exporting to GitHub Actions Cache" failed with timeout
-        if: (success() || failure()) && steps.changed-files.outputs.doctests_all_changed_files
-        run: |
-          docker run --name BUILD -dit \
-                     --mount type=bind,src=$(pwd),dst=$(pwd) \
-                     --workdir $(pwd) \
-                     ${{ env.BUILD_IMAGE }} /bin/sh
-
-      # Testing
-
-      - name: Check that all modules can be imported
-        if: (success() || failure()) && steps.container.outcome == 'success' && steps.changed-files.outputs.doctests_all_changed_files
-        run: |
-          # Increase the length of the lines in the "short summary"
-          export COLUMNS=120
-          # The following command checks that all modules can be imported.
-          # The output also includes a long list of modules together with the number of tests in each module.
-          # This can be ignored.
-          ./sage -python -m pip install pytest-xdist
-          ./sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
-        shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
-
-      - name: Test changed files (sage -t --new)
-        if: (success() || failure()) && steps.container.outcome == 'success' && steps.changed-files.outputs.doctests_all_changed_files
-        run: |
-          export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
-          # https://github.com/tj-actions/changed-files?tab=readme-ov-file#outputs-
-          ./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.doctests_all_changed_files }}
-        shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
-
-  test-mod:
-    runs-on: ubuntu-latest
-    needs: [test-new]
-    services:
-      # https://docs.docker.com/build/ci/github-actions/local-registry/
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
-    strategy:
-      fail-fast: false
-      matrix:
-        targets:
-          - sagemath_categories-check
-    steps:
-      - name: Maximize build disk space
-        uses: easimon/maximize-build-space@v10
-        with:
-          # need space in /var for Docker images
-          root-reserve-mb:      30000
-          remove-dotnet:        true
-          remove-android:       true
-          remove-haskell:       true
-          remove-codeql:        true
-          remove-docker-images: true
-
-      - name: Checkout
-        id: checkout
-        uses: actions/checkout@v4
-
-      - name: Install test prerequisites
-        # From docker.yml
-        run: |
-          sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
-          sudo apt-get clean
-          df -h
-
-      - name: Merge CI fixes from sagemath/sage
-        # From docker.yml
-        # This step needs to happen after the commit sha is put in DOCKER_TAG
-        # so that multi-stage builds can work correctly.
-        run: |
-          .ci/merge-fixes.sh
-        env:
-          GH_TOKEN: ${{ github.token }}
-
-      # Building
-
-      - name: Generate Dockerfile
-        # From docker.yml
-        run: |
-          tox -e ${{ env.TOX_ENV }}
-          cp .tox/${{ env.TOX_ENV }}/Dockerfile .
-        env:
-          # Only generate the Dockerfile, do not run 'docker build' here
-          DOCKER_TARGETS: ""
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver-opts: network=host
-
-      - name: Build Docker image
-        id: image
-        uses: docker/build-push-action@v6
-        with:
-          push:       true
-          load:       false
-          context:    .
-          tags:       ${{ env.BUILD_IMAGE }}
-          target:     with-targets
-          cache-from: type=gha
-          cache-to:   type=gha,mode=max
-          build-args: |
-            NUMPROC=6
-            USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
-            TARGETS_PRE=build/make/Makefile
-            TARGETS=${{ needs.test-new.outputs.build_targets }}
-
-      - name: Start container
-        id: container
-        if: (success() || failure())
-        run: |
-          docker run --name BUILD -dit \
-                     --mount type=bind,src=$(pwd),dst=$(pwd) \
-                     --workdir $(pwd) \
-                     ${{ env.BUILD_IMAGE }} /bin/sh
-
-      # Testing
-
-      - name: Test modularized distributions
-        if: (success() || failure()) && steps.container.outcome == 'success'
-        run: |
-          export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
-          make V=0 tox-ensure && make ${{ matrix.targets }}
-        shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
-
   test-long:
     runs-on: ubuntu-latest
-    needs: [test-new]
     services:
       # https://docs.docker.com/build/ci/github-actions/local-registry/
       registry:
@@ -406,6 +129,8 @@ jobs:
 
       - name: Generate Dockerfile
         # From docker.yml
+        # tox -e <environment name> command runs commands in `tox.ini`,
+        # in particular the script `.ci/write-dockerfile.sh`, to generate `Dockerfile`.
         run: |
           tox -e ${{ env.TOX_ENV }}
           cp .tox/${{ env.TOX_ENV }}/Dockerfile .
@@ -427,8 +152,6 @@ jobs:
           context:    .
           tags:       ${{ env.BUILD_IMAGE }}
           target:     with-targets
-          cache-from: type=gha
-          cache-to:   type=gha,mode=max
           build-args: |
             NUMPROC=6
             USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
@@ -547,8 +270,6 @@ jobs:
           context:    .
           tags:       ${{ env.BUILD_IMAGE }}
           target:     with-targets
-          cache-from: type=gha
-          cache-to:   type=gha,mode=max
           build-args: |
             NUMPROC=6
             USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse