Release Manager · Release Manager · commit 0405d07dde86 · 2023-10-04T23:51:26.000+02:00
gh-36367: Simplify experimental all-conda installation instructions via `pkgs/sage-conf_conda`   This is a version of sage-conf for the experimental all-conda installation method. It calls `configure` with the whole long set of options, which simplifies the instructions for the user. The new instructions use editable mode (PEP 660) for `pkgs/sage- conf_conda`, but it also works to install it in non-editable mode.    ### 📝 Checklist     - [x] The title is concise, informative, and self-explanatory. - [x] The description explains in detail what this PR is about. - [ ] I have linked a relevant issue or discussion. - [ ] I have created tests covering the changes. - [ ] I have updated the documentation accordingly. ### ⌛ Dependencies   URL: #36367 Reported by: Matthias Köppe Reviewer(s): Isuru Fernando, Tobias Diez
diff --git a/.gitignore b/.gitignore
@@ -175,6 +175,16 @@ build/bin/sage-build-env-config
 /pkgs/*/*.egg-info
 /pkgs/*/.tox
 
+/pkgs/sage-conf_pypi/sage_root/config.log
+/pkgs/sage-conf_pypi/sage_root/config.status
+/pkgs/sage-conf_pypi/sage_root/local/
+/pkgs/sage-conf_pypi/sage_root/logs/
+/pkgs/sage-conf_pypi/sage_root/prefix
+/pkgs/sage-conf_pypi/sage_root/src/bin/sage-env-config
+/pkgs/sage-conf_pypi/sage_root/src/bin/sage-src-env-config
+/pkgs/sage-conf_pypi/sage_root/upstream/
+/pkgs/sage-conf_pypi/sage_root/venv
+
 /pkgs/sagemath-objects/setup.cfg
 /pkgs/sagemath-bliss/setup.cfg
 /pkgs/sagemath-coxeter3/setup.cfg
diff --git a/pkgs/sage-conf/README.rst b/pkgs/sage-conf/README.rst
@@ -78,10 +78,20 @@ sage_conf in the SageMath distribution
 The original version of the distribution package ``sage_conf`` is used
 internally in the SageMath distribution.  It is provided in the directory
 `pkgs/sage-conf <https://github.com/sagemath/sage/tree/develop/pkgs/sage-conf/>`_.
-This version of the package is generated by the Sage distribution's ``configure``
+This version of the package is generated by the Sage distribution's ``./configure``
 script.
 
 
+sage_conf for conda
+-------------------
+
+The version of the distribution package in the directory
+`pkgs/sage-conf_conda <https://github.com/sagemath/sage/tree/develop/pkgs/sage-conf_conda/>`_
+is used in an experimental installation method of SageMath, where all packages
+are provided by conda.  This method is described in
+https://doc.sagemath.org/html/en/installation/conda.html#using-conda-to-provide-all-dependencies-for-the-sage-library-experimental
+
+
 sage_conf in downstream distributions
 -------------------------------------
 
diff --git a/pkgs/sage-conf_conda/.gitignore b/pkgs/sage-conf_conda/.gitignore
@@ -0,0 +1,6 @@
+/_sage_conf/_conf.py
+/build
+/dist
+/*.egg-info
+/.tox
+/bin/sage-env-config
diff --git a/pkgs/sage-conf_conda/MANIFEST.in b/pkgs/sage-conf_conda/MANIFEST.in
@@ -0,0 +1,53 @@
+prune           .tox
+include         VERSION.txt
+graft           bin
+exclude         bin/sage-env-config                             # generated by configure
+
+prune           sage_root
+include         sage_root/Makefile
+include         sage_root/README.md
+include         sage_root/VERSION.txt
+include         sage_root/bootstrap
+include         sage_root/bootstrap-conda
+graft           sage_root/build
+prune           sage_root/build/.tox
+exclude         sage_root/build/bin/sage-build-env-config       # generated by configure
+exclude         sage_root/build/make/Makefile-auto              # generated by configure
+exclude         sage_root/build/make/Makefile                   # generated by configure
+
+# These sources are not needed because individual distributions of these are made.
+prune           sage_root/build/pkgs/*/src*
+
+graft           sage_root/config
+include         sage_root/configure
+include         sage_root/configure.ac
+graft           sage_root/m4
+
+# Only these pkgs are needed (because of dependencies on source files,
+# see "git grep SAGE_ROOT build/pkgs/*/dependencies")
+graft           sage_root/pkgs/sage-conf
+prune           sage_root/pkgs/sage-conf/build
+prune           sage_root/pkgs/sage-conf/dist
+prune           sage_root/pkgs/sage-conf/*.egg-info
+exclude         sage_root/pkgs/sage-conf/_sage_conf/_conf.py    # generated by configure
+graft           sage_root/pkgs/sage-docbuild
+prune           sage_root/pkgs/sage-docbuild/build
+prune           sage_root/pkgs/sage-docbuild/dist
+prune           sage_root/pkgs/sage-docbuild/*.egg-info
+
+graft           sage_root/src/_sage_conf
+include         sage_root/src/bin/sage-env
+include         sage_root/src/bin/sage-env-config.in
+include         sage_root/src/bin/sage-src-env-config.in
+include         sage_root/src/bin/sage-venv-config
+include         sage_root/src/bin/sage-version.sh
+include         sage_root/src/doc/bootstrap                     # FIXME: should move to builds/pkgs/sagemath_doc_html/
+
+global-exclude  .tox
+global-exclude  *~*
+global-exclude  *.bak
+global-exclude  *.orig
+global-exclude  __pycache__
+global-exclude  *.py[co]
+global-exclude  *.so
+global-exclude  .DS_Store
diff --git a/pkgs/sage-conf_conda/README.rst b/pkgs/sage-conf_conda/README.rst
@@ -0,0 +1 @@
+../sage-conf/README.rst
diff --git a/pkgs/sage-conf_conda/VERSION.txt b/pkgs/sage-conf_conda/VERSION.txt
@@ -0,0 +1 @@
+10.2.beta5
diff --git a/pkgs/sage-conf_conda/_sage_conf b/pkgs/sage-conf_conda/_sage_conf
@@ -0,0 +1 @@
+../sage-conf/_sage_conf
diff --git a/pkgs/sage-conf_conda/bin b/pkgs/sage-conf_conda/bin
@@ -0,0 +1 @@
+../sage-conf_pypi/bin
diff --git a/pkgs/sage-conf_conda/pyproject.toml b/pkgs/sage-conf_conda/pyproject.toml
@@ -0,0 +1 @@
+../sage-conf/pyproject.toml
diff --git a/pkgs/sage-conf_conda/sage_conf.py b/pkgs/sage-conf_conda/sage_conf.py
@@ -0,0 +1 @@
+../sage-conf/sage_conf.py
diff --git a/pkgs/sage-conf_conda/sage_root b/pkgs/sage-conf_conda/sage_root
@@ -0,0 +1 @@
+../..
diff --git a/pkgs/sage-conf_conda/setup.cfg b/pkgs/sage-conf_conda/setup.cfg
@@ -0,0 +1 @@
+../sage-conf/setup.cfg
diff --git a/pkgs/sage-conf_conda/setup.py b/pkgs/sage-conf_conda/setup.py
@@ -0,0 +1,121 @@
+import os
+import sys
+import shutil
+import sysconfig
+import platform
+import fnmatch
+
+from pathlib import Path
+
+from setuptools import setup
+from distutils.command.build_scripts import build_scripts as distutils_build_scripts
+from setuptools.command.build_py import build_py as setuptools_build_py
+from setuptools.command.editable_wheel import editable_wheel as setuptools_editable_wheel
+from setuptools.errors import SetupError
+
+
+class build_py(setuptools_build_py):
+
+    def run(self):
+
+        HERE = os.path.dirname(__file__)
+        if self.editable_mode:
+            SAGE_ROOT = os.path.join(HERE, 'sage_root')
+        else:
+            SAGE_ROOT = self._create_writable_sage_root()
+
+        if not os.environ.get('CONDA_PREFIX', ''):
+            raise SetupError('No conda environment is active. '
+                             'See https://doc.sagemath.org/html/en/installation/conda.html on how to get started.')
+
+        if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
+            print(f'Reusing configured SAGE_ROOT={SAGE_ROOT}')
+        else:
+            cmd = f"cd {SAGE_ROOT} && ./configure --enable-build-as-root --with-system-python3=force --disable-notebook --disable-sagelib --disable-sage_conf --disable-doc"
+            cmd += ' --with-python=$CONDA_PREFIX/bin/python --prefix="$CONDA_PREFIX"'
+            cmd += ' $(for pkg in $(PATH="build/bin:$PATH" build/bin/sage-package list :standard: --exclude rpy2 --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)'
+            print(f"Running {cmd}")
+            sys.stdout.flush()
+            if os.system(cmd) != 0:
+                if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
+                    print("Warning: A configuration has been written, but the configure script has exited with an error. "
+                          "Carefully check any messages above before continuing.")
+                else:
+                    print(f"Error: The configure script has failed; this may be caused by missing build prerequisites.")
+                    sys.stdout.flush()
+                    PREREQ_SPKG = "_prereq bzip2 xz libffi"  # includes python3 SPKG_DEPCHECK packages
+                    os.system(f'cd {SAGE_ROOT} && export PACKAGES="$(build/bin/sage-get-system-packages conda {PREREQ_SPKG})" && [ -n "$PACKAGES" ] && echo "You can install the required build prerequisites using the following shell command" && echo "" && build/bin/sage-print-system-package-command conda --verbose --sudo install $PACKAGES && echo ""')
+                    raise SetupError("configure failed")
+
+        # In this mode, we never run "make".
+
+        # Copy over files generated by the configure script
+        # (see configure.ac AC_CONFIG_FILES)
+        if self.editable_mode:
+            pass  # same file
+        else:
+            shutil.copyfile(os.path.join(SAGE_ROOT, 'pkgs', 'sage-conf', '_sage_conf', '_conf.py'),
+                            os.path.join(HERE, '_sage_conf', '_conf.py'))
+        shutil.copyfile(os.path.join(SAGE_ROOT, 'src', 'bin', 'sage-env-config'),
+                        os.path.join(HERE, 'bin', 'sage-env-config'))
+
+        setuptools_build_py.run(self)
+
+    def _create_writable_sage_root(self):
+        HERE = os.path.dirname(__file__)
+        DOT_SAGE = os.environ.get('DOT_SAGE', os.path.join(os.environ.get('HOME'), '.sage'))
+        with open(os.path.join(HERE, 'VERSION.txt')) as f:
+            sage_version = f.read().strip()
+        # After #30534, SAGE_LOCAL no longer contains any Python.  So we key the SAGE_ROOT only to Sage version
+        # and architecture.
+        system = platform.system()
+        machine = platform.machine()
+        arch_tag = f'{system}-{machine}'
+        # TODO: Should be user-configurable with config settings
+        SAGE_ROOT = os.path.join(DOT_SAGE, f'sage-{sage_version}-{arch_tag}-conda')
+
+        def ignore(path, names):
+            # exclude all embedded src trees
+            if fnmatch.fnmatch(path, f'*/build/pkgs/*'):
+                return ['src']
+            ### ignore more stuff --- .tox etc.
+            return [name for name in names
+                    if name in ('.tox', '.git', '__pycache__',
+                                'prefix', 'local', 'venv', 'upstream',
+                                'config.status', 'config.log', 'logs')]
+
+        if not os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
+            # config.status and other configure output has to be writable.
+            # So (until the Sage distribution supports VPATH builds - #21469), we have to make a copy of sage_root.
+            try:
+                shutil.copytree('sage_root', SAGE_ROOT,
+                                ignore=ignore)  # will fail if already exists
+            except Exception as e:
+                raise SetupError(f"the directory SAGE_ROOT={SAGE_ROOT} already exists but it is not configured ({e}). "
+                                 "Please either remove it and try again, or install in editable mode (pip install -e).")
+
+        return SAGE_ROOT
+
+
+class build_scripts(distutils_build_scripts):
+
+    def run(self):
+        self.distribution.scripts.append(os.path.join('bin', 'sage-env-config'))
+        if not self.distribution.entry_points:
+            self.entry_points = self.distribution.entry_points = dict()
+        distutils_build_scripts.run(self)
+
+
+class editable_wheel(setuptools_editable_wheel):
+    r"""
+    Customized so that exceptions raised by our build_py
+    do not lead to the "Customization incompatible with editable install" message
+    """
+    _safely_run = setuptools_editable_wheel.run_command
+
+
+setup(
+    cmdclass=dict(build_py=build_py,
+                  build_scripts=build_scripts,
+                  editable_wheel=editable_wheel)
+)
diff --git a/pkgs/sage-conf_pypi/pyproject.toml b/pkgs/sage-conf_pypi/pyproject.toml
diff --git a/pkgs/sage-conf_pypi/pyproject.toml b/pkgs/sage-conf_pypi/pyproject.toml
@@ -0,0 +1 @@
+../sage-conf/pyproject.toml
diff --git a/pkgs/sage-conf_pypi/setup.py b/pkgs/sage-conf_pypi/setup.py
@@ -3,40 +3,35 @@
 import shutil
 import sysconfig
 import platform
+import fnmatch
 
 from setuptools import setup
 from distutils.command.build_scripts import build_scripts as distutils_build_scripts
 from setuptools.command.build_py import build_py as setuptools_build_py
-from setuptools.command.egg_info import egg_info as setuptools_egg_info
-from distutils.errors import (DistutilsSetupError, DistutilsModuleError,
-                              DistutilsOptionError)
+from setuptools.command.editable_wheel import editable_wheel as setuptools_editable_wheel
+from setuptools.errors import SetupError
+
 
 class build_py(setuptools_build_py):
 
     def run(self):
-        DOT_SAGE = os.environ.get('DOT_SAGE', os.path.join(os.environ.get('HOME'), '.sage'))
         HERE = os.path.dirname(__file__)
-        with open(os.path.join(HERE, 'VERSION.txt')) as f:
-            sage_version = f.read().strip()
+        if self.editable_mode:
+            SAGE_ROOT = os.path.join(HERE, 'sage_root')
+        else:
+            SAGE_ROOT = self._create_writable_sage_root()
+
         # For convenience, set up the homebrew env automatically. This is a no-op if homebrew is not present.
-        SETENV = '(. ./.homebrew-build-env 2> /dev/null || :)'
-        # After #30534, SAGE_LOCAL no longer contains any Python.  So we key the SAGE_ROOT only to Sage version
-        # and architecture.
-        system = platform.system()
-        machine = platform.machine()
-        arch_tag = f'{system}-{machine}'
-        # TODO: These two should be user-configurable with options passed to "setup.py install"
-        SAGE_ROOT = os.path.join(DOT_SAGE, f'sage-{sage_version}-{arch_tag}')
+        if os.environ.get('CONDA_PREFIX', ''):
+            SETENV = ':'
+        else:
+            SETENV = '(. ./.homebrew-build-env 2> /dev/null || :)'
+
         SAGE_LOCAL = os.path.join(SAGE_ROOT, 'local')
+
         if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
-            print(f'Reusing SAGE_ROOT={SAGE_ROOT}')
+            print(f'Reusing configured SAGE_ROOT={SAGE_ROOT}')
         else:
-            # config.status and other configure output has to be writable.
-            # So (until the Sage distribution supports VPATH builds - #21469), we have to make a copy of sage_root.
-            try:
-                shutil.copytree('sage_root', SAGE_ROOT)  # will fail if already exists
-            except Exception:
-                raise DistutilsSetupError(f"the directory SAGE_ROOT={SAGE_ROOT} already exists but it is not configured.  Please remove it and try again.")
             cmd = f"cd {SAGE_ROOT} && {SETENV} && ./configure --prefix={SAGE_LOCAL} --with-python={sys.executable} --enable-build-as-root --enable-download-from-upstream-url --with-system-python3=force --with-sage-venv --disable-notebook --disable-sagelib --disable-sage_conf --disable-doc"
             print(f"Running {cmd}")
             sys.stdout.flush()
@@ -45,7 +40,18 @@ def run(self):
                 sys.stdout.flush()
                 PREREQ_SPKG = "_prereq bzip2 xz libffi"  # includes python3 SPKG_DEPCHECK packages
                 os.system(f'cd {SAGE_ROOT} && export SYSTEM=$(build/bin/sage-guess-package-system 2>/dev/null) && export PACKAGES="$(build/bin/sage-get-system-packages $SYSTEM {PREREQ_SPKG})" && [ -n "$PACKAGES" ] && echo "You can install the required build prerequisites using the following shell command" && echo "" && build/bin/sage-print-system-package-command $SYSTEM --verbose --sudo install $PACKAGES && echo ""')
-                raise DistutilsSetupError("configure failed")
+                raise SetupError("configure failed")
+
+        # Copy over files generated by the configure script
+        # (see configure.ac AC_CONFIG_FILES)
+        if self.editable_mode:
+            pass  # same file
+        else:
+            shutil.copyfile(os.path.join(SAGE_ROOT, 'pkgs', 'sage-conf', '_sage_conf', '_conf.py'),
+                            os.path.join(HERE, '_sage_conf', '_conf.py'))
+        shutil.copyfile(os.path.join(SAGE_ROOT, 'src', 'bin', 'sage-env-config'),
+                        os.path.join(HERE, 'bin', 'sage-env-config'))
+
         # Here we run "make build" -- which builds everything except for sagelib because we
         # used configure --disable-sagelib
         # Alternative:
@@ -61,13 +67,44 @@ def run(self):
         if os.system(cmd) != 0:
             raise DistutilsSetupError(f"make {TARGETS} failed")
 
-        # Install configuration
-        shutil.copyfile(os.path.join(SAGE_ROOT, 'pkgs', 'sage-conf', '_sage_conf', '_conf.py'),
-                        os.path.join(HERE, '_sage_conf', '_conf.py'))
-        shutil.copyfile(os.path.join(SAGE_ROOT, 'src', 'bin', 'sage-env-config'),
-                        os.path.join(HERE, 'bin', 'sage-env-config'))
         setuptools_build_py.run(self)
 
+    def _create_writable_sage_root(self):
+        HERE = os.path.dirname(__file__)
+        DOT_SAGE = os.environ.get('DOT_SAGE', os.path.join(os.environ.get('HOME'), '.sage'))
+        with open(os.path.join(HERE, 'VERSION.txt')) as f:
+            sage_version = f.read().strip()
+        # After #30534, SAGE_LOCAL no longer contains any Python.  So we key the SAGE_ROOT only to Sage version
+        # and architecture.
+        system = platform.system()
+        machine = platform.machine()
+        arch_tag = f'{system}-{machine}'
+        # TODO: Should be user-configurable with config settings
+        SAGE_ROOT = os.path.join(DOT_SAGE, f'sage-{sage_version}-{arch_tag}')
+
+        def ignore(path, names):
+            # exclude all embedded src trees
+            if fnmatch.fnmatch(path, f'*/build/pkgs/*'):
+                return ['src']
+            ### ignore more stuff --- .tox etc.
+            return [name for name in names
+                    if name in ('.tox', '.git', '__pycache__',
+                                'prefix', 'local', 'venv', 'upstream',
+                                'config.status', 'config.log', 'logs')]
+
+        if not os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
+            # config.status and other configure output has to be writable.
+            # So (until the Sage distribution supports VPATH builds - #21469), we have to make a copy of sage_root.
+            try:
+                shutil.copytree('sage_root', SAGE_ROOT,
+                                ignore=ignore)  # will fail if already exists
+            except Exception as e:
+                raise SetupError(f"the directory SAGE_ROOT={SAGE_ROOT} already exists but it is not configured ({e}). "
+                                 "Please either remove it and try again, or install in editable mode (pip install -e).")
+
+        return SAGE_ROOT
+
+
 class build_scripts(distutils_build_scripts):
 
     def run(self):
@@ -76,6 +113,17 @@ def run(self):
             self.entry_points = self.distribution.entry_points = dict()
         distutils_build_scripts.run(self)
 
+
+class editable_wheel(setuptools_editable_wheel):
+    r"""
+    Customized so that exceptions raised by our build_py
+    do not lead to the "Customization incompatible with editable install" message
+    """
+    _safely_run = setuptools_editable_wheel.run_command
+
+
 setup(
-    cmdclass=dict(build_py=build_py, build_scripts=build_scripts)
+    cmdclass=dict(build_py=build_py,
+                  build_scripts=build_scripts,
+                  editable_wheel=editable_wheel)
 )
diff --git a/src/doc/en/installation/conda.rst b/src/doc/en/installation/conda.rst
