Trac #32698: Fix up builds without system python3 after #32442

URL: https://trac.sagemath.org/32698
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): John Palmieri

Author: Release Manager

.gitignore

@@ -182,6 +182,7 @@ src/*.egg-info/
 /src/bin/sage-src-env-config
 
 # Virtual environments
+/venv
 src/.env
 src/.venv
 src/env/

Makefile

@@ -109,6 +109,7 @@ distclean: build-clean
 	@echo "Deleting all remaining output from build system ..."
 	rm -rf local
 	rm -f src/bin/sage-env-config
+	rm -f prefix venv
 
 # Delete all auto-generated files which are distributed as part of the
 # source tarball

build/make/Makefile.in

@@ -535,8 +535,10 @@ $(1)-$(4)-no-deps:
 $(1)-no-deps: $(1)-$(4)-no-deps
 
 $(1)-$(4)-clean:
-	sage-spkg-uninstall $(if $(filter $(1),$(TOOLCHAIN_DEPS)),--keep-files) \
-		$(1) '$$($(4))'
+	if [ -d '$$($(4))' ]; then \
+	    sage-spkg-uninstall $(if $(filter $(1),$(TOOLCHAIN_DEPS)),--keep-files) \
+		$(1) '$$($(4))'; \
+	fi
 
 $(1)-clean: $(1)-$(4)-clean
 

build/pkgs/python3/spkg-build.in

@@ -84,8 +84,24 @@ sdh_configure --enable-shared $PYTHON_CONFIGURE
 # Make sure -L. is placed before -L$SAGE_LOCAL/lib so that python and extension
 # modules are linked with the right libpython; we pass this in at make time
 # only, since we don't want -L. to be saved as one of the default LDFLAGS
-# used for building third-party extension modules
-sdh_make LDFLAGS="-L. $LDFLAGS"
+# used for building third-party extension modules.
+#
+# Trac #32442: As we now install python in SAGE_VENV, not SAGE_LOCAL,
+# we need to provide paths into $SAGE_LOCAL, so that setup.py finds
+# the libraries needed for the extension modules - in particular sqlite3.
+# (The search code there does not know about CPATH and LIBRARY_PATH.)
+make_LDFLAGS="-L. -L$SAGE_LOCAL/lib $LDFLAGS"
+make_CPPFLAGS="-I$SAGE_LOCAL/include $CPPFLAGS"
+
+# Also, we need to add an rpath, like we do for SAGE_LOCAL in src/bin/sage-env.
+# SAGE_INST_LOCAL is the installation hierarchy for the current package
+# -- for python3, this is SAGE_VENV.
+make_LDFLAGS="-Wl,-rpath,$SAGE_INST_LOCAL/lib $make_LDFLAGS"
+if [ "$UNAME" = "Linux" ]; then
+    make_LDFLAGS="-Wl,-rpath-link,$SAGE_INST_LOCAL/lib $make_LDFLAGS"
+fi
+
+sdh_make LDFLAGS="$make_LDFLAGS" CPPFLAGS="$make_CPPFLAGS"
 
 if [ "$UNAME" = "Darwin" ]; then
     export DYLD_LIBRARY_PATH="."

build/pkgs/python3/spkg-configure.m4

@@ -119,6 +119,17 @@ To build Sage with a different system python, use ./configure --with-python=/pat
     ])
     AC_SUBST([PYTHON_FOR_VENV])
 
+    AS_VAR_IF([SAGE_VENV], [auto], [SAGE_VENV=$SAGE_VENV_AUTO])
+    AS_CASE([$SAGE_VENV],
+        [no],  [SAGE_VENV='${SAGE_LOCAL}'],dnl Quoted so that it is resolved at build time by shell/Makefile
+        [yes], [AS_IF([test -n "$PYTHON_FOR_VENV"], [
+                   PYTHON_VERSION=$("$PYTHON_FOR_VENV" -c "import sysconfig; print(sysconfig.get_python_version())")
+                ], [
+                   PYTHON_VERSION=$(echo $(cat build/pkgs/python3/package-version.txt))
+                ])
+                SAGE_VENV='${SAGE_LOCAL}'/var/lib/sage/venv-python$PYTHON_VERSION]
+    )
+
     dnl These temporary directories are created by the check above
     dnl and need to be cleaned up to prevent the "rm -f conftest*"
     dnl (that a bunch of other checks do) from emitting warnings about

configure.ac

@@ -70,15 +70,24 @@ SAGE_LOCAL="$prefix"
 # This is nonstandard.
 if test "$SAGE_LOCAL" = NONE; then
     SAGE_LOCAL=local
+    if test -x "$SAGE_LOCAL"/bin/python3; then
+        # Incremental build with an existing installation of python3 spkg
+        # in SAGE_LOCAL or venv in SAGE_LOCAL.  Keep old behavior.
+        SAGE_VENV_AUTO=no
+    else
+        SAGE_VENV_AUTO=yes
+    fi
+else
+    SAGE_VENV_AUTO=no
 fi
 SAGE_SRC="$SAGE_ROOT/src"
 SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed"
 
 AC_ARG_WITH([sage-venv],
-            [AS_HELP_STRING([--with-sage-venv=SAGE_VENV],
+            [AS_HELP_STRING([--with-sage-venv={auto (default),yes,no,SAGE_VENV}],
                             [put Python packages into an installation hierarchy separate from prefix])],
             [SAGE_VENV="$withval"],
-            [SAGE_VENV='${SAGE_LOCAL}'])dnl Quoted so that it is resolved at build time by shell/Makefile
+            [SAGE_VENV="auto"])
 AC_SUBST([SAGE_VENV])
 
 #---------------------------------------------------------
@@ -493,6 +502,29 @@ AC_CONFIG_COMMANDS(mkdirs,
         SAGE_INST="$SAGE_SPKG_INST"
     ])
 
+AC_CONFIG_COMMANDS(links, [
+    dnl Create links for the convenience of users
+    SYMLINK="${ac_top_build_prefix}prefix"
+    AS_IF([test -L "$SYMLINK" -o ! -e "$SYMLINK"], [
+        AC_MSG_NOTICE([creating convenience symlink $SYMLINK -> $SAGE_LOCAL])
+        rm -f "$SYMLINK"
+        ln -sf "$SAGE_LOCAL" "$SYMLINK"
+    ], [
+        AC_MSG_NOTICE([cannot create convenience symlink $SYMLINK -> $SAGE_LOCAL because the file exists and is not a symlink; this is harmless])
+    ])
+    SYMLINK="${ac_top_build_prefix}venv"
+    AS_IF([test -L "$SYMLINK" -o ! -e "$SYMLINK"], [
+        AC_MSG_NOTICE([creating convenience symlink $SYMLINK -> $SAGE_VENV])
+        rm -f "$SYMLINK"
+        ln -sf "$SAGE_VENV" "$SYMLINK"
+    ], [
+        AC_MSG_NOTICE([cannot create convenience symlink $SYMLINK -> $SAGE_VENV because the file exists and is not a symlink; this is harmless])
+    ])
+], [
+    SAGE_LOCAL="$SAGE_LOCAL"
+    eval SAGE_VENV="$SAGE_VENV"dnl eval so as to evaluate the embedded ${SAGE_LOCAL}
+])
+
 AC_OUTPUT()
 
 SAGE_SYSTEM_PACKAGE_NOTICE()

m4/sage_spkg_collect.m4

@@ -182,8 +182,8 @@ for DIR in $SAGE_ROOT/build/pkgs/*; do
     dnl Jupyter notebook, then packages such as jupyter_core would have to be installed into
     dnl two trees.
     SPKG_TREE_VAR=SAGE_LOCAL
-    if test -f "$DIR/requirements.txt" -o -f "$DIR/install-requires.txt"; then
-        dnl A Python package
+    if test -f "$DIR/requirements.txt" -o -f "$DIR/install-requires.txt" -o "$SPKG_NAME" = python3; then
+        dnl A Python package or spkg installation of python3 itself
         SPKG_TREE_VAR=SAGE_VENV
     fi
     SAGE_PACKAGE_TREES="${SAGE_PACKAGE_TREES}$(printf '\ntrees_')${SPKG_NAME} = ${SPKG_TREE_VAR}"