sagemath
diff --git a/‎.ci/create-changes-html.sh‎
Lines changed: 2 additions & 2 deletions b/‎.ci/create-changes-html.sh‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎.ci/docker-exec-script.sh‎
Lines changed: 14 additions & 0 deletions b/‎.ci/docker-exec-script.sh‎
Lines changed: 14 additions & 0 deletions
diff --git a/‎.ci/merge-fixes.sh‎
Lines changed: 8 additions & 5 deletions b/‎.ci/merge-fixes.sh‎
Lines changed: 8 additions & 5 deletions
diff --git a/‎.ci/retrofit-worktree.sh‎
Lines changed: 12 additions & 9 deletions b/‎.ci/retrofit-worktree.sh‎
Lines changed: 12 additions & 9 deletions
diff --git a/‎.ci/set_labels_by_changes.sh‎
Lines changed: 71 additions & 0 deletions b/‎.ci/set_labels_by_changes.sh‎
Lines changed: 71 additions & 0 deletions
diff --git a/‎build/bin/write-dockerfile.sh‎ renamed to ‎.ci/write-dockerfile.sh‎
Lines changed: 89 additions & 26 deletions b/‎build/bin/write-dockerfile.sh‎ renamed to ‎.ci/write-dockerfile.sh‎
Lines changed: 89 additions & 26 deletions
diff --git a/‎.devcontainer/onCreate-conda.sh‎
Lines changed: 0 additions & 2 deletions b/‎.devcontainer/onCreate-conda.sh‎
Lines changed: 0 additions & 2 deletions
diff --git a/‎.devcontainer/portability-centos-stream-8-python3.9-minimal/devcontainer.json‎
Lines changed: 0 additions & 25 deletions b/‎.devcontainer/portability-centos-stream-8-python3.9-minimal/devcontainer.json‎
Lines changed: 0 additions & 25 deletions
@@ -51,7 +51,7 @@ diffParagraphs.forEach(paragraph => {
 EOF
 echo '</head>' >> CHANGES.html
 echo '<body>' >> CHANGES.html
-(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- *.html) > diff.txt
+(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- "*.html") > diff.txt
 python3 - << EOF
 import os, re, html
 with open('diff.txt', 'r') as f:
@@ -82,7 +82,7 @@ for block in diff_blocks:
         if content:
             with open(file_path, 'w') as file:
                 file.writelines(content)
-        path = 'html/' + doc
+        path = doc
         hunks = '&nbsp;'.join(f'<a href="{path}#hunk{i+1}" class="hunk" target="_blank">#{i + 1}</a>' for i in range(count))
         out_blocks.append(f'<p class="diff"><a href="{path}">{doc}</a>&nbsp;' + hunks + '&emsp;</p>'
                             + '\n<pre><code class="language-diff">'
 
@@ -0,0 +1,14 @@
+#!/bin/sh -x
+if [ $# -lt 3 ]; then
+    echo >&2 "usage: docker-exec-script.sh CONTAINER WORKDIR [VAR=VALUE...] SCRIPT"
+    exit 1
+fi
+CONTAINER=$1
+WORKDIR=$2
+shift 2
+(echo "cd \"$WORKDIR\"";
+ while [ $# -gt 1 ]; do
+     echo "export \"$1\""
+     shift
+ done;
+ cat "$1") | docker exec -i $CONTAINER bash -ex
@@ -1,5 +1,8 @@
 #!/bin/sh
-# Apply open PRs labeled "blocker" from sagemath/sage as patches.
+# Apply open PRs labeled "p: CI Fix" from sagemath/sage as patches.
+# (policy set by vote in 2024-03,
+#  https://groups.google.com/g/sage-devel/c/OKwwUGyKveo/m/vpyCXYBqAAAJ)
+#
 # This script is invoked by various workflows in .github/workflows
 #
 # The repository variable SAGE_CI_FIXES_FROM_REPOS can be set
@@ -20,15 +23,15 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
             echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES"
             ;;
         */*)
-            echo "Getting open PRs with 'blocker' status from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+blocker+%2F+1%22"
+            echo "Getting open PRs with 'p: CI Fix' label from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+CI+Fix%22"
             GH="gh -R $REPO"
             REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}"
-            PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number' | tee $REPO_FILE)"
+            PRs="$($GH pr list --label "p: CI Fix" --json number --jq '.[].number' | tee $REPO_FILE)"
             date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date  # Record the date, for future reference
             if [ -z "$PRs" ]; then
-                echo "Nothing to do: Found no open PRs with 'blocker' status in $REPO."
+                echo "Nothing to do: Found no open PRs with 'p: CI Fix' label in $REPO."
             else
-                echo "Found open PRs with 'blocker' status in $REPO: $(echo $PRs)"
+                echo "Found open PRs with 'p: CI Fix' label in $REPO: $(echo $PRs)"
                 git tag -f test_base
                 git commit -q -m "Uncommitted changes" --no-allow-empty -a
                 for a in $PRs; do
 
@@ -12,32 +12,35 @@ export GIT_AUTHOR_EMAIL="[email protected]"
 export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
 export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
 
+set -e
+
 # Set globally for other parts of the workflow
 git config --global user.name "$GIT_AUTHOR_NAME"
 git config --global user.email "$GIT_AUTHOR_EMAIL"
 
-set -ex
+set -x
 
 # If actions/checkout downloaded our source tree using the GitHub REST API
 # instead of with git (because do not have git installed in our image),
 # we first make the source tree a repo.
-if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
+if [ ! -d .git ]; then git init; fi
 
-# Tag this state of the source tree "new". This is what we want to build and test.
+# Commit and tag this state of the source tree "new". This is what we want to build and test.
+git add -A && git commit --quiet --allow-empty -m "new"
 git tag -f new
 
 # Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
 # But $WORKTREE_DIRECTORY is not a git repository.
 # We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
 # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-# Then we update worktree and index with "git reset --hard new".
+# Then we update worktree and index with "git checkout new".
 # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-# The changed files now show up as uncommitted changes.
-# The final "git add -N" makes sure that files that were added in "new" do not show
-# as untracked files, which would be removed by "git clean -fx".
+if [ -L $WORKTREE_NAME ]; then
+    rm -f $WORKTREE_NAME
+fi
+git worktree prune --verbose
 git worktree add --detach $WORKTREE_NAME
 rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
 rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
 if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
-(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git checkout new && git status)
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# cd to the PR directory
+cd $PR_MOUNT_DIR
+PR_BASE_SHA=$(git merge-base $BASE_SHA $PR_HEAD_SHA)
+
+echo "set_labels_by_changes.sh called with environment:"
+echo "BASE SHA: $PR_BASE_SHA" 
+echo "HEAD SHA: $PR_HEAD_SHA" 
+echo "SMALL THRESHOLD $SMALL_THRESHOLD"
+echo "MODERATE THERESHOLD: $MODERATE_THRESHOLD"
+echo "LARGE THRESHOLD: $LARGE_THRESHOLD"
+
+# get all the changes made and changed files
+CHANGES=$(git diff --ignore-all-space $PR_BASE_SHA $PR_HEAD_SHA)
+
+# ignore blank lines
+CHANGES=$(echo "$CHANGES" | grep -vE '^[\+\-]\s*$')
+
+# ignore non necessary lines from git diff
+CHANGES=$(echo "$CHANGES" | grep -E '^[+\-]' | grep -vE '^\+\+\+|^\-\-\-')
+
+# count total no of lines
+CHANGES=$(echo "$CHANGES" | wc -l)
+
+echo "CHANGES MADE: $CHANGES"
+
+AUTH_HEADER="Authorization: Bearer $GITHUB_TOKEN"
+
+MINIMAL="v: minimal"
+SMALL="v: small"
+MODERATE="v: moderate"
+LARGE="v: large"
+
+DELETE_LABELS=("$MINIMAL" "$SMALL" "$MODERATE" "$LARGE")
+
+if [ "$CHANGES" -gt "$LARGE_THRESHOLD" ]; then
+    SIZE_LABEL="$LARGE"
+elif [ "$CHANGES" -gt "$MODERATE_THRESHOLD" ]; then
+    SIZE_LABEL="$MODERATE"
+elif [ "$CHANGES" -gt "$SMALL_THRESHOLD" ]; then
+    SIZE_LABEL="$SMALL"
+else
+    SIZE_LABEL="$MINIMAL"
+fi
+
+DELETE_LABELS=("${DELETE_LABELS[@]//${SIZE_LABEL}/}")
+
+# API for adding labels on the Pull Request 
+API_URL="https://api.github.com/repos/$REPOSITORY/issues/$PR_NUMBER/labels"
+
+echo "Adding label: ${SIZE_LABEL[@]}"
+for LABEL in "${SIZE_LABEL[@]}"; do
+    curl -X POST \
+        -H "$AUTH_HEADER" \
+        -H "Accept: application/vnd.github+json" \
+        -H "X-GitHub-Api-Version: 2022-11-28" \
+        -d "{\"labels\":[\"$LABEL\"]}" \
+        "$API_URL" >/dev/null
+done
+
+echo "Deleting Labels:"
+
+for DELETE_LABEL in "${DELETE_LABELS[@]}"; do
+    ENCODED_LABEL=$(echo "$DELETE_LABEL" | sed 's/ /%20/g')
+    curl -X DELETE \
+        -H "Accept: application/vnd.github+json" \
+        -H "$AUTH_HEADER" \
+        -H "X-GitHub-Api-Version: 2022-11-28" \
+        "$API_URL/$ENCODED_LABEL" >/dev/null
+done
@@ -1,20 +1,41 @@
 #! /usr/bin/env bash
-## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist
-## and satisfy the requirements tested by spkg-configure.m4
-## This is called by $SAGE_ROOT/tox.ini
 set -e
-SYSTEM="${1:-debian}"
 shopt -s extglob
+##
+## Write a Dockerfile for portability testing to stdout.
+##
+## This script needs to be run from SAGE_ROOT (root of the Sage repository).
+## It is called by $SAGE_ROOT/tox.ini for all environments 'tox -e docker-...'
+##
+## Positional arguments:
+##
+SYSTEM="${1:-debian}"
 SAGE_PACKAGE_LIST_ARGS="${2:-:standard:}"
 WITH_SYSTEM_SPKG="${3:-yes}"
 IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 EXTRA_SAGE_PACKAGES="${5:-_bootstrap}"
-#
+##
+## Environment variables that take influence:
+##
+## - BOOTSTRAP
+## - CONFIGURE_ARGS
+## - DEVTOOLSET
+## - DIST_UPGRADE
+## - DOCKER_BUILDKIT
+## - EXTRA_PATH
+## - EXTRA_REPOSITORIES
+## - EXTRA_SYSTEM_PACKAGES
+## - FULL_BASE_IMAGE_AND_TAG
+## - SKIP_SYSTEM_PKG_INSTALL
+## - USE_CONDARC
+## - __CHOWN
+## - __SUDO
+##
 STRIP_COMMENTS="sed s/#.*//;"
 SAGE_ROOT=.
 export PATH="$SAGE_ROOT"/build/bin:$PATH
 SYSTEM_PACKAGES=$EXTRA_SYSTEM_PACKAGES
-CONFIGURE_ARGS="--enable-option-checking "
+SYSTEM_CONFIGURE_ARGS="--enable-option-checking "
 for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do
     SYSTEM_PACKAGE=$(sage-get-system-packages $SYSTEM $SPKG)
     if [ -n "${SYSTEM_PACKAGE}" ]; then
@@ -24,16 +45,16 @@ for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_
             # shell-quote package if necessary
             SYSTEM_PACKAGES+=$(printf " %q" "$a")
         done
-        CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
+        SYSTEM_CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
     fi
 done
-echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
+echo "# Automatically generated by SAGE_ROOT/.ci/write-dockerfile.sh"
 echo "# the :comments: separate the generated file into sections"
 echo "# to simplify writing scripts that customize this file"
 ADD="ADD $__CHOWN"
 RUN=RUN
 cat <<EOF
-ARG BASE_IMAGE
+ARG BASE_IMAGE=$(eval echo "${FULL_BASE_IMAGE_AND_TAG}")
 FROM \${BASE_IMAGE} as with-system-packages
 EOF
 case $SYSTEM in
@@ -54,8 +75,21 @@ case $SYSTEM in
             1|y*|Y*)
                 ;;
             *)
+                #
+                # The Ubuntu Docker images are "minimized", meaning that some large
+                # bits such as documentation has been removed. We have to unminimize
+                # once (which reinstalls the full versions of some minimized packages),
+                # or e.g. the maxima documentation (which we depend on for correct operation)
+                # will be missing.
+                #
+                # But we only have to do this once. To save time in incremental builds,
+                # we remove the unminimize binary here after it has done its job.
+                #
                 cat <<EOF
-RUN (yes | unminimize) || echo "(ignored)"
+RUN if command -v unminimize > /dev/null; then  \
+        (yes | unminimize) || echo "(ignored)"; \
+        rm -f "\$(command -v unminimize)";       \
+    fi
 EOF
                 if [ -n "$DIST_UPGRADE" ]; then
                     cat <<EOF
@@ -143,7 +177,7 @@ EOF
                 ;;
             *)
                 cat <<EOF
-ARG USE_CONDARC=condarc.yml
+ARG USE_CONDARC=${USE_CONDARC-condarc.yml}
 ADD *condarc*.yml /tmp/
 RUN echo \${CONDARC}; cd /tmp && conda config --stdin < \${USE_CONDARC}
 RUN conda update -n base conda
@@ -230,32 +264,49 @@ cat <<EOF
 
 FROM with-system-packages as bootstrapped
 #:bootstrapping:
-RUN if [ -d /sage ]; then echo "### Incremental build from \$(cat /sage/VERSION.txt)" && mv /sage /sage-old && mkdir /sage && for a in local logs; do if [ -d /sage-old/\$a ]; then mv /sage-old/\$a /sage; fi; done; rm -rf /sage-old; else mkdir -p /sage; fi
+RUN rm -rf /new /sage/.git
+$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 .gitignore /new/
+$ADD config/config.rpath /new/config/config.rpath
+$ADD src/doc/bootstrap /new/src/doc/bootstrap
+$ADD src/bin /new/src/bin
+$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
+$ADD m4 /new/m4
+$ADD pkgs /new/pkgs
+$ADD build /new/build
+$ADD .upstream.d /new/.upstream.d
+ADD .ci /.ci
+RUN if [ -d /sage ]; then                                               \
+        echo "### Incremental build from \$(cat /sage/VERSION.txt)" &&  \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \
+        if ! (cd /new && /.ci/retrofit-worktree.sh worktree-image /sage); then \
+            echo "retrofit-worktree.sh failed, falling back to replacing /sage"; \
+            for a in local logs; do                                     \
+                if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi;       \
+            done;                                                       \
+            rm -rf /sage;                                               \
+            mv /new /sage;                                              \
+        fi;                                                             \
+    else                                                                \
+        mv /new /sage;                                                  \
+    fi
 WORKDIR /sage
-$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
-$ADD config/config.rpath config/config.rpath
-$ADD src/doc/bootstrap src/doc/bootstrap
-$ADD src/bin src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/VERSION.txt src/
-$ADD m4 ./m4
-$ADD pkgs pkgs
-$ADD build ./build
-$ADD .upstream.d ./.upstream.d
-ARG BOOTSTRAP=./bootstrap
+
+ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}"
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS
 
 FROM bootstrapped as configured
 #:configuring:
 RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log
-ARG EXTRA_CONFIGURE_ARGS=""
+ARG CONFIGURE_ARGS="${CONFIGURE_ARGS:---enable-build-as-root}"
 EOF
 if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "********** configuring without forcing ***********"; echo "::group::config.log"; cat config.log; echo "::endgroup::"; ./configure --enable-build-as-root; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; echo "********** configuring without forcing ***********"; ./configure \${CONFIGURE_ARGS}; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 else
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 fi
 cat <<EOF
@@ -286,7 +337,19 @@ ENV MAKE="make -j\${NUMPROC}"
 ARG USE_MAKEFLAGS="-k V=0"
 ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
-$ADD src src
+$ADD .gitignore /new/.gitignore
+$ADD src /new/src
+RUN cd /new && rm -rf .git && \
+    if /.ci/retrofit-worktree.sh worktree-pre /sage; then \
+        cd /sage && touch configure build/make/Makefile; \
+    else \
+        echo "retrofit-worktree.sh failed, falling back to replacing /sage/src"; \
+        rm -rf /sage/src;                                    \
+        mv src /sage/src;                                    \
+        cd /sage && ./bootstrap && ./config.status;          \
+    fi; \
+    cd /sage && rm -rf /new .git
+
 ARG TARGETS="build"
 $RUN $CHECK_STATUS_THEN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN $THEN_SAVE_STATUS
 
 
@@ -8,6 +8,4 @@ conda init bash
 
 # Build sage
 conda run -n sage-dev ./bootstrap
-conda run -n sage-dev ./configure --with-python=/opt/conda/envs/sage-dev/bin/python --prefix=/opt/conda/envs/sage-dev
-conda run -n sage-dev pip install --no-build-isolation -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup 
 conda run -n sage-dev pip install --no-build-isolation -v -v -e ./src