bootstrap-conda: Refactor using sage-get-system-packages

Author: Matthias Koeppe

bootstrap-conda

@@ -11,45 +11,75 @@ STRIP_COMMENTS="sed s/#.*//;"
 shopt -s extglob
 
 DEVELOP_SPKG_PATTERN="@(_develop$(for a in $(head -n 1 build/pkgs/_develop/dependencies); do echo -n "|"$a; done))"
-BOOTSTRAP_PACKAGES=$(echo $(${STRIP_COMMENTS} build/pkgs/_bootstrap/distros/conda.txt))
-SYSTEM_PACKAGES=
-OPTIONAL_SYSTEM_PACKAGES=
-SAGELIB_SYSTEM_PACKAGES=
-SAGELIB_OPTIONAL_SYSTEM_PACKAGES=
-DEVELOP_SYSTEM_PACKAGES=
+BOOTSTRAP_PACKAGES=_bootstrap
+PACKAGES=
+OPTIONAL_PACKAGES=
+SAGELIB_PACKAGES=
+SAGELIB_OPTIONAL_PACKAGES=
+DEVELOP_PACKAGES=
+
 for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sagemath); do
     PKG_SCRIPTS=build/pkgs/$PKG_BASE
     SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt
     PKG_TYPE=$(cat $PKG_SCRIPTS/type)
     PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
-    if [ -n "PKG_SYSTEM_PACKAGES" ]; then
+    if [ -n "$PKG_SYSTEM_PACKAGES" ]; then
         if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
+            if grep -q SAGE_PYTHON_PACKAGE_CHECK $PKG_SCRIPTS/spkg-configure.m4; then
+                # Python package that would need --enable-system-site-packages to be used
+                # with the Sage distribution, but we do not recommend that for conda.
+                PKG_SAGELIB_ONLY=yes
+            else
+                PKG_SAGELIB_ONLY=no
+            fi
+        else
+            # No spkg-configure, so the Sage distribution is not able to make use of this package.
+            PKG_SAGELIB_ONLY=yes
+        fi
+        [ -n "$BOOTSTRAP_VERBOSE" ] && echo "$PKG_BASE:$PKG_TYPE:$PKG_SAGELIB_ONLY"
+        if [ $PKG_SAGELIB_ONLY = no ]; then
             case "$PKG_BASE:$PKG_TYPE" in
                 *:standard)
-                    SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    PACKAGES+=" $PKG_BASE"
                     ;;
                 $DEVELOP_SPKG_PATTERN:*)
-                    DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    DEVELOP_PACKAGES+=" $PKG_BASE"
                     ;;
                 *)
-                    OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    OPTIONAL_PACKAGES+=" $PKG_BASE"
                     ;;
             esac
         else
             case "$PKG_BASE:$PKG_TYPE" in
                 *:standard)
-                    SAGELIB_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    SAGELIB_PACKAGES+=" $PKG_BASE"
                     ;;
                 $DEVELOP_SPKG_PATTERN:*)
-                    DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    DEVELOP_PACKAGES+=" $PKG_BASE"
                     ;;
                 *)
-                    SAGELIB_OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    SAGELIB_OPTIONAL_PACKAGES+=" $PKG_BASE"
                     ;;
             esac
         fi
     fi
 done
+unset PKG_SYSTEM_PACKAGES
+
+[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Collected:" && set | grep PACKAGES=
+
+# Translate to system packages
+export ENABLE_SYSTEM_SITE_PACKAGES=yes  # Disable filtering in sage-get-system-packages
+SYSTEM_PACKAGES=$(sage-get-system-packages conda $PACKAGES)
+BOOTSTRAP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $BOOTSTRAP_PACKAGES)
+OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $OPTIONAL_PACKAGES)
+SAGELIB_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_PACKAGES)
+SAGELIB_OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_OPTIONAL_PACKAGES)
+DEVELOP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $DEVELOP_PACKAGES)
+unset ENABLE_SYSTEM_SITE_PACKAGES
+
+[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Translated to system:" && set | grep SYSTEM_PACKAGES=
+
 echo >&2 $0:$LINENO: generate conda environment files
 for python_version in 3.9 3.10 3.11; do
     (
@@ -63,7 +93,7 @@ for python_version in 3.9 3.10 3.11; do
             echo "  - $pkg"
         done
         echo "  # Packages needed for ./bootstrap"
-        for pkg in $BOOTSTRAP_PACKAGES; do
+        for pkg in $BOOTSTRAP_SYSTEM_PACKAGES; do
             echo "  - $pkg"
         done
     ) > environment-$python_version.yml
@@ -84,23 +114,25 @@ for python_version in 3.9 3.10 3.11; do
     ) > environment-optional-$python_version.yml
 
     (
-        (
-            sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml
-            echo "  # Additional dev tools"
-            for pkg in $DEVELOP_SYSTEM_PACKAGES; do
-                echo "  - $pkg"
-            done
-        ) >&4
-        (
-            cat src/environment-$python_version.yml
-            echo "  # optional packages"
-            for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do
-                echo "  - $pkg"
-            done
-        ) >&5
+        sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml
+        echo "  # Additional dev tools"
+        for pkg in $DEVELOP_SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+    ) > src/environment-dev-$python_version.yml
+
+    (
+        cat src/environment-$python_version.yml
+        echo "  # optional packages"
+        for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do
+            echo "  - $pkg"
+        done
+    ) > src/environment-optional-$python_version.yml
+
+    (
         echo >&4 "  - pip:"
         echo >&5 "  - pip:"
-        for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do
+        for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do
             PKG_SCRIPTS=build/pkgs/$PKG_BASE
             SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt
             if [ ! -f $SYSTEM_PACKAGES_FILE ]; then