Merge tag '9.5.beta7' into t/29039/pip_installable_package_sage_bootstrap

SageMath version 9.5.beta7, Release Date: 2021-11-18

Author: Matthias Koeppe

.github/workflows/tox-experimental.yml

@@ -41,7 +41,7 @@ jobs:
         # This list is different from the one in tox.yml:
         # Trac #31526 switches gcc 4.x-based distributions to using the gcc_spkg configuration factor
         # Trac #32281 removes gcc 4.x-based distributions whose binutils are unusable
-        tox_system_factor: [ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie-gcc_spkg, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7-gcc_spkg, centos-8, gentoo, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-gcc_spkg]
+        tox_system_factor: [ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie-gcc_spkg, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7-gcc_spkg, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-gcc_spkg]
         tox_packages_factor: [maximal]
         targets_pattern: [0-g, h-o, p, q-z]
     env:

.github/workflows/tox-optional.yml

@@ -41,7 +41,7 @@ jobs:
         # This list is different from the one in tox.yml:
         # Trac #31526 switches gcc 4.x-based distributions to using the gcc_spkg configuration factor
         # Trac #32281 removes gcc 4.x-based distributions whose binutils are unusable
-        tox_system_factor: [ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie-gcc_spkg, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7-gcc_spkg, centos-8, gentoo, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-gcc_spkg]
+        tox_system_factor: [ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie-gcc_spkg, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7-gcc_spkg, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-gcc_spkg]
         tox_packages_factor: [maximal]
         targets_pattern: [0-g, h-o, p, q-z]
     env:

.github/workflows/tox.yml

@@ -38,7 +38,7 @@ jobs:
       fail-fast: false
       max-parallel: 20
       matrix:
-        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, ubuntu-impish, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, centos-7, centos-8, gentoo, gentoo-python3.7, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
+        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, ubuntu-impish, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, centos-7, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
         tox_packages_factor: [minimal, standard]
     env:
       TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}

.homebrew-build-env

@@ -2,7 +2,7 @@
 # that activate keg-only homebrew package installations
 
 HOMEBREW=`brew --prefix` || return 1
-for l in gettext bzip2 texinfo; do
+for l in gettext bzip2 texinfo polymake; do
     if [ -d "$HOMEBREW/opt/$l/bin" ]; then
         PATH="$HOMEBREW/opt/$l/bin:$PATH"
     fi
@@ -23,7 +23,7 @@ export PKG_CONFIG_PATH
 LIBRARY_PATH="$HOMEBREW/lib$LIBRARY_PATH"
 [ -z "$CPATH" ] || CPATH=":${CPATH}"
 CPATH="$HOMEBREW/include$CPATH"
-for l in readline bzip2 ntl; do
+for l in readline bzip2 ntl polymake; do
     if [ -d "$HOMEBREW/opt/$l/lib" ]; then
         LIBRARY_PATH="$HOMEBREW/opt/$l/lib:$LIBRARY_PATH"
     fi

.zenodo.json

@@ -1,10 +1,10 @@
 {
     "description": "Mirror of the Sage https://sagemath.org/ source tree", 
     "license": "other-open", 
-    "title": "sagemath/sage: 9.5.beta5", 
-    "version": "9.5.beta5", 
+    "title": "sagemath/sage: 9.5.beta7", 
+    "version": "9.5.beta7", 
     "upload_type": "software", 
-    "publication_date": "2021-10-28", 
+    "publication_date": "2021-11-18", 
     "creators": [
         {
             "affiliation": "SageMath.org", 
@@ -15,7 +15,7 @@
     "related_identifiers": [
         {
             "scheme": "url", 
-            "identifier": "https://github.com/sagemath/sage/tree/9.5.beta5", 
+            "identifier": "https://github.com/sagemath/sage/tree/9.5.beta7", 
             "relation": "isSupplementTo"
         }, 
         {

VERSION.txt

@@ -1 +1 @@
-SageMath version 9.5.beta5, Release Date: 2021-10-28
+SageMath version 9.5.beta7, Release Date: 2021-11-18

build/bin/sage-bootstrap-python

@@ -40,7 +40,19 @@ if [ "$LC_ALL" = "C" -o "$LANG" = "C" -o "$LC_CTYPE" = "C" ]; then
     export LANG
 fi
 
-PYTHONS="python python3 python3.8 python3.7 python2.7 python3.6 python2"
+PYTHONS="python python3 python3.10 python3.9 python3.8 python3.7 python2.7 python3.6 python2"
+# Trac #32405: Prefer a Python that provides ssl with SNI, which allows developers
+# to download from upstream URLs (configure --enable-download-from-upstream-url),
+# in particular from PyPI, which requires SNI.
+for PY in $PYTHONS; do
+    PYTHON="$(PATH="$SAGE_ORIG_PATH" command -v $PY)"
+    if [ -n "$PYTHON" ]; then
+        if "$PYTHON" -c "import argparse; import urllib; from hashlib import sha1; from ssl import HAS_SNI; assert HAS_SNI; from os import listdir; listdir(\"$(pwd)\");" 2>/dev/null; then
+            exec "$PYTHON" "$@"
+        fi
+    fi
+done
+# Second round, no ssl/SNI test.
 for PY in $PYTHONS; do
     PYTHON="$(PATH="$SAGE_ORIG_PATH" command -v $PY)"
     if [ -n "$PYTHON" ]; then

build/bin/sage-dist-helpers

@@ -56,11 +56,12 @@
 #    be given as arguments.  If $SAGE_DESTDIR is not set then the command is
 #    run with $SAGE_SUDO, if set.
 #
-# - sdh_pip_install [...]
+# - sdh_pip_install [--no-deps] [--build-isolation] [...]
 #
-#    Runs `pip install` with the given arguments, as well as additional
-#    default arguments used for installing packages into Sage with pip.
-#    Currently this is just a wrapper around the `sage-pip-install` command.
+#    Builds a wheel using `pip wheel` with the given options [...], then installs
+#    the wheel.  Unless the special option --build-isolation is given,
+#    the wheel is built using the option --no-build-isolation.
+#    If the special option --no-deps is given, it is passed to pip install.
 #    If $SAGE_DESTDIR is not set then the command is run with $SAGE_SUDO, if
 #    set.
 #
@@ -240,10 +241,37 @@ sdh_pip_install() {
     echo "Installing $PKG_NAME"
     mkdir -p dist
     rm -f dist/*.whl
-    python3 -m pip wheel --use-feature=in-tree-build --wheel-dir=dist --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation --ignore-requires-python "$@" || \
+    install_options=""
+    # pip has --no-build-isolation but no flag that turns the default back on...
+    build_isolation_option="--no-build-isolation --no-binary :all:"
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            --build-isolation)
+                # If a package requests build isolation, we allow it to provision
+                # its build environment using the stored wheels.
+                # We pass --find-links and remove the --no-binary option.
+                # The SPKG needs to declare "setuptools_wheel" as a dependency.
+                build_isolation_option="--find-links=$SAGE_SPKG_WHEELS"
+                ;;
+            --no-deps)
+                install_options="$install_options $1"
+                ;;
+            *)
+                break
+                ;;
+        esac
+        shift
+    done
+    python3 -m pip wheel --use-feature=in-tree-build --wheel-dir=dist --verbose --no-deps --no-index --isolated --ignore-requires-python $build_isolation_option "$@" || \
         sdh_die "Error building a wheel for $PKG_NAME"
 
-    sdh_store_and_pip_install_wheel .
+    sdh_store_and_pip_install_wheel $install_options .
+}
+
+sdh_pip_editable_install() {
+    echo "Installing $PKG_NAME (editable mode)"
+    python3 -m pip install --verbose --no-deps --no-index --no-build-isolation --isolated --editable "$@" || \
+        sdh_die "Error installing $PKG_NAME"
 }
 
 sdh_store_wheel() {

build/bin/sage-spkg

@@ -498,15 +498,14 @@ export PKG_NAME="$PKG_NAME"
 export PKG_BASE="$PKG_BASE"
 export PKG_VER="$PKG_VER"
 
-for lib in "\$SAGE_ROOT/build/bin/sage-dist-helpers" "\$SAGE_SRC/bin/sage-env-config" "\$SAGE_SRC/bin/sage-env" "\$SAGE_ROOT/build/bin/sage-build-env-config" "\$SAGE_ROOT/build/bin/sage-build-env"; do
+for lib in "\$SAGE_ROOT/build/bin/sage-dist-helpers" "\$SAGE_SRC/bin/sage-src-env-config" "\$SAGE_SRC/bin/sage-env-config" "\$SAGE_SRC/bin/sage-env" "\$SAGE_ROOT/build/bin/sage-build-env-config" "\$SAGE_ROOT/build/bin/sage-build-env"; do
     source "\$lib"
     if [ \$? -ne 0 ]; then
         echo >&2 "Error: failed to source \$lib"
         echo >&2 "Is \$SAGE_ROOT the correct SAGE_ROOT?"
         exit 1
     fi
 done
-export PATH="$SAGE_INST_LOCAL/bin:$PATH"
 
 export SAGE_INST_LOCAL="$SAGE_INST_LOCAL"
 
@@ -773,6 +772,7 @@ if [ "$SAGE_CHECK" = "yes" -o "$SAGE_CHECK" = "warn" ]; then
         if [ $? -ne 0 ]; then
             TEST_SUITE_RESULT="failed"
             if [ "$SAGE_CHECK" = "warn" ]; then
+                # The following warning message must be consistent with SAGE_ROOT/build/make/install (see trac:32781)
                 error_msg "Warning: Failures testing package $PKG_NAME (ignored)" "make check"
             else
                 error_msg "Error testing package $PKG_NAME" "make check"

build/make/Makefile.in

@@ -52,6 +52,10 @@ SAGE_SPKG = sage-spkg
 # These are added to SAGE_SPKG in the call
 SAGE_SPKG_OPTIONS = @SAGE_SPKG_OPTIONS@
 
+# Where the Sage distribution installs documentation.
+# set to empty if --disable-doc is used
+SAGE_DOCS = $(SAGE_LOCAL)
+
 # Where the Sage distribution installs Python packages.
 # This can be overridden by 'make SAGE_VENV=/some/venv'.
 SAGE_VENV = @SAGE_VENV@
@@ -142,7 +146,7 @@ SCRIPT_PACKAGES = @SAGE_SCRIPT_PACKAGES@
 # inst_git = $(INST)/.dummy
 
 $(foreach pkgname,$(BUILT_PACKAGES),\
-	$(eval inst_$(pkgname) = $(foreach tree, $(trees_$(pkgname)), $($(tree))/$(SPKG_INST_RELDIR)/$(pkgname)-$(vers_$(pkgname)))))
+	$(eval inst_$(pkgname) = $(foreach tree, $(trees_$(pkgname)), $(and $($(tree)), $($(tree))/$(SPKG_INST_RELDIR)/$(pkgname)-$(vers_$(pkgname))))))
 $(foreach pkgname,$(DUMMY_PACKAGES),\
 	$(eval inst_$(pkgname) = $(SAGE_LOCAL)/$(SPKG_INST_RELDIR)/.dummy))
 
@@ -157,7 +161,7 @@ $(SAGE_LOCAL)/$(SPKG_INST_RELDIR)/.dummy:
 
 
 # Filtered by installation tree
-$(foreach tree,SAGE_LOCAL SAGE_VENV, \
+$(foreach tree,SAGE_LOCAL SAGE_VENV SAGE_DOCS, \
     $(eval $(tree)_INSTALLED_PACKAGE_INSTS = \
                $(foreach pkgname,$(INSTALLED_PACKAGES), \
                          $(if $(findstring $(tree),$(trees_$(pkgname))), \
@@ -228,10 +232,10 @@ build-start: all-build
 base-toolchain: _clean-broken-gcc base
 	+$(MAKE_REC) toolchain
 
-# All targets except for the base packages
+# All targets except for the base packages and except the documentation
 all-sage: \
-		$(INSTALLED_PACKAGE_INSTS) \
-                $(UNINSTALLED_PACKAGES_CLEANS)
+		$(SAGE_LOCAL_INSTALLED_PACKAGE_INSTS) $(SAGE_LOCAL_UNINSTALLED_PACKAGES_CLEANS) \
+		$(SAGE_VENV_INSTALLED_PACKAGE_INSTS)  $(SAGE_VENV_UNINSTALLED_PACKAGES_CLEANS)
 
 # Same but filtered by installation trees:
 all-build-local: toolchain-deps
@@ -244,6 +248,11 @@ all-build-venv: toolchain-deps
 
 all-sage-venv:  $(SAGE_VENV_INSTALLED_PACKAGE_INSTS)  $(SAGE_VENV_UNINSTALLED_PACKAGES_CLEANS)
 
+all-build-docs: toolchain-deps
+	+$(MAKE_REC) all-sage-docs
+
+all-sage-docs:  $(SAGE_DOCS_INSTALLED_PACKAGE_INSTS) $(SAGE_DOCS_UNINSTALLED_PACKAGES_CLEANS)
+
 # Download all packages which should be inside an sdist tarball (the -B
 # option to make forces all targets to be built unconditionally)
 download-for-sdist:
@@ -315,35 +324,28 @@ base: $(inst_patch) $(inst_pkgconf)
 #
 # $ ./sage --docbuild -H
 
-# Building the documentation has many dependencies, because all
-# documented modules are imported and because we use matplotlib to
-# produce plots.
-DOC_DEPENDENCIES = sagelib sage_docbuild $(inst_sphinx) \
-	| $(SAGERUNTIME) $(inst_maxima) $(inst_networkx) $(inst_scipy) $(inst_sympy) \
-	$(inst_matplotlib) $(inst_pillow) $(inst_mathjax) $(inst_mpmath) \
-	$(inst_ipykernel) $(inst_jupyter_client) $(inst_conway_polynomials) \
-	$(inst_tachyon) $(inst_jmol) $(inst_thebe) $(inst_ipywidgets)
-
-doc: doc-html
+doc: $(SAGE_DOCS_INSTALLED_PACKAGE_INSTS) $(SAGE_DOCS_UNINSTALLED_PACKAGES_CLEANS)
 
-doc-html: $(DOC_DEPENDENCIES)
-	$(AM_V_at)cd "$(SAGE_SRC)/doc" && sage-logger -p "$(MAKE) doc-html" $(SAGE_ROOT)/logs/dochtml.log
+# All doc-building is delegated to the script packages
+# sagemath_doc_html, sagemath_doc_pdf
+doc-html: sagemath_doc_html
 
 # 'doc-html-no-plot': build docs without building the graphics coming
 # from the '.. plot' directive, in case you want to save a few
 # megabytes of disk space. 'doc-clean' is a prerequisite because the
 # presence of graphics is cached in src/doc/output.
 doc-html-no-plot: doc-clean
-	$(AM_V_at)sage-logger -p "$(MAKE) SAGE_DOCBUILD_OPTS=\"$(SAGE_DOCBUILD_OPTS) --no-plot\" doc-html" $(SAGE_ROOT)/logs/dochtml.log
+	+$(MAKE_REC) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-plot" doc-html
 
+# Using mathjax is actually the default, but this target can be used
+# to override an environment setting of SAGE_DOC_MATHJAX=no
 doc-html-mathjax:
-	$(AM_V_at)sage-logger -p "$(MAKE) SAGE_DOCBUILD_OPTS=\"$(SAGE_DOCBUILD_OPTS) -j\" doc-html" $(SAGE_ROOT)/logs/dochtml.log
+	+$(MAKE_REC) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) -j" doc-html
 
 # Keep target 'doc-html-jsmath' for backwards compatibility.
 doc-html-jsmath: doc-html-mathjax
 
-doc-pdf: $(DOC_DEPENDENCIES)
-	$(AM_V_at)cd "$(SAGE_SRC)/doc" && sage-logger -p "$(MAKE) doc-pdf" $(SAGE_ROOT)/logs/docpdf.log
+doc-pdf: sagemath_doc_pdf
 
 doc-clean: doc-src-clean doc-output-clean