[Python] Statically link cppyy_backend inside libcppyy

Separating the libraries made sense in the past when the ROOT Python
interfaces still supported PyPy, where only the backend needed to be
loaded and there was no CPython extension. But nowadays, the
clingwrapper in the `cppyy_backend` library is only used for the CPython
extension `libcppyy`, so we might as well link statically.

This avoids some overhead and boilerplate code:

  * we don't need to set the `CPPYY_BACKEND_LIBRARY` path

  * we don't need system links for `libcppyy_backend.so` if the Python
    install directory differs from the library install directory

  * we don't need Python code to load the backend library (installing
    the `cppyy_backend` Python code can be completely skipped, and
    patching it is not needed anymore)

The change implies that the `rpath` code also needs to be moved from the
backend to the CPyCppyy CMake code.

Author: guitargeek

bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt

@@ -55,14 +55,15 @@ add_library(${libname} SHARED ${headers} ${sources})
 # Set the suffix to '.so' and the prefix to 'lib'
 set_target_properties(${libname} PROPERTIES  ${ROOT_LIBRARY_PROPERTIES})
 if(MSVC)
-  target_link_libraries(${libname} PUBLIC cppyy_backend Python3::Python)
+  target_link_libraries(${libname} PRIVATE cppyy_backend)
+  target_link_libraries(${libname} PUBLIC Python3::Python)
   set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
   set_target_properties(${libname} PROPERTIES PREFIX "lib")
   set_target_properties(${libname} PROPERTIES SUFFIX ".pyd")
 elseif(APPLE)
-  target_link_libraries(${libname} PUBLIC -Wl,-bind_at_load -Wl,-w -Wl,-undefined -Wl,dynamic_lookup cppyy_backend)
+  target_link_libraries(${libname} PRIVATE -Wl,-bind_at_load -Wl,-w -Wl,-undefined -Wl,dynamic_lookup cppyy_backend)
 else()
-  target_link_libraries(${libname} PUBLIC -Wl,--unresolved-symbols=ignore-all cppyy_backend)
+  target_link_libraries(${libname} PRIVATE -Wl,--unresolved-symbols=ignore-all cppyy_backend)
 endif()
 
 if(NOT MSVC)
@@ -104,6 +105,26 @@ target_include_directories(${libname}
 
 set_property(GLOBAL APPEND PROPERTY ROOT_EXPORTED_TARGETS ${libname})
 
+if(NOT MSVC)
+  # Make sure that relative RUNPATH to main ROOT libraries is always correct.
+
+  file(RELATIVE_PATH pymoduledir_to_libdir_build ${localruntimedir} "${localruntimedir}")
+  file(RELATIVE_PATH pymoduledir_to_libdir_install ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_PYTHONDIR} "${CMAKE_INSTALL_FULL_LIBDIR}")
+
+  if(APPLE)
+    set_target_properties(${libname} PROPERTIES
+        BUILD_RPATH "@loader_path/${pymoduledir_to_libdir_build}"
+        INSTALL_RPATH "@loader_path/${pymoduledir_to_libdir_install}"
+    )
+  else()
+    set_target_properties(${libname} PROPERTIES
+        BUILD_RPATH "$ORIGIN/${pymoduledir_to_libdir_build}"
+        INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}"
+    )
+  endif()
+
+endif()
+
 # Install library
 install(TARGETS ${libname} EXPORT ${CMAKE_PROJECT_NAME}Exports
                             RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT libraries

bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/CommonDefs.h

@@ -6,29 +6,29 @@
 #ifdef _MSC_VER
 // Windows requires symbols to be explicitly exported
 #define CPYCPPYY_EXPORT extern __declspec(dllexport)
-#define CPYCPPYY_IMPORT extern __declspec(dllimport)
 #define CPYCPPYY_CLASS_EXPORT __declspec(dllexport)
 
 // CPYCPPYY_EXTERN is dual use in the public API
 #ifndef CPYCPPYY_INTERNAL
 #define CPYCPPYY_EXTERN extern __declspec(dllexport)
 #define CPYCPPYY_CLASS_EXTERN __declspec(dllexport)
 #else
-#define CPYCPPYY_EXTERN extern __declspec(dllimport)
-#define CPYCPPYY_CLASS_EXTERN __declspec(dllimport)
+#define CPYCPPYY_EXTERN extern
+#define CPYCPPYY_CLASS_EXTERN
 #endif
 
 #define CPYCPPYY_STATIC
 
 #else
 // Linux, Mac, etc.
 #define CPYCPPYY_EXPORT extern
-#define CPYCPPYY_IMPORT extern
 #define CPYCPPYY_CLASS_EXPORT
 #define CPYCPPYY_EXTERN extern
 #define CPYCPPYY_CLASS_EXTERN
 #define CPYCPPYY_STATIC static
 
 #endif
 
+#define CPYCPPYY_IMPORT extern
+
 #endif // !CPYCPPYY_COMMONDEFS_H

bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h

@@ -126,7 +126,12 @@ inline Cppyy::TCppType_t CPPInstance::ObjectIsA(bool check_smart) const
 
 
 //- object proxy type and type verification ----------------------------------
-CPYCPPYY_IMPORT PyTypeObject CPPInstance_Type;
+// Needs to be extern because the libROOTPythonizations is secretly using it
+#ifdef _MSC_VER
+extern __declspec(dllimport) PyTypeObject CPPInstance_Type;
+#else
+extern PyTypeObject CPPInstance_Type;
+#endif
 
 template<typename T>
 inline bool CPPInstance_Check(T* object)

bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h

@@ -9,11 +9,7 @@
 #include <stdint.h>
 
 // import/export (after precommondefs.h from PyPy)
-#ifdef _MSC_VER
-#define CPPYY_IMPORT extern __declspec(dllimport)
-#else
 #define CPPYY_IMPORT extern
-#endif
 
 // some more types; assumes Cppyy.h follows Python.h
 #ifndef PY_LONG_LONG

bindings/pyroot/cppyy/CPyCppyy/src/SignalTryCatch.h

@@ -71,6 +71,11 @@ using CppyyExceptionContext_t = ExceptionContext_t;
         gException = R__old; \
     }
 
-CPYCPPYY_IMPORT CppyyExceptionContext_t *gException;
+// extern, defined in ROOT Core
+#ifdef _MSC_VER
+extern __declspec(dllimport) CppyyExceptionContext_t *gException;
+#else
+extern CppyyExceptionContext_t *gException;
+#endif
 
 #endif

bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt

@@ -4,80 +4,6 @@
 # For the licensing terms see $ROOTSYS/LICENSE.
 # For the list of contributors see $ROOTSYS/README/CREDITS.
 
-set(py_sources
-  cppyy_backend/__init__.py
-  cppyy_backend/_cling_config.py
-  cppyy_backend/_cppyy_generator.py
-  cppyy_backend/_genreflex.py
-  cppyy_backend/_rootcling.py
-  cppyy_backend/bindings_utils.py
-  cppyy_backend/loader.py
-  cppyy_backend/_get_cppflags.py
-)
-
-set(cppyy_backendPySrcDir cling/python/cppyy_backend)
-file(COPY ${cppyy_backendPySrcDir}
-     DESTINATION ${localruntimedir}
-     PATTERN "cmake" EXCLUDE
-     PATTERN "pkg_templates" EXCLUDE)
-
-file(RELATIVE_PATH PYTHONDIR_TO_LIBDIR "${CMAKE_INSTALL_FULL_PYTHONDIR}" "${CMAKE_INSTALL_FULL_LIBDIR}")
-
-set(libname cppyy_backend)
-
-add_library(${libname} SHARED clingwrapper/src/clingwrapper.cxx)
-if(MSVC)
-  set_target_properties(${libname} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
-endif()
-# Set the suffix to '.so' and the prefix to 'lib'
-set_target_properties(${libname} PROPERTIES  ${ROOT_LIBRARY_PROPERTIES})
-target_link_libraries(${libname} Core ${CMAKE_DL_LIBS})
-
-# cppyy uses ROOT headers from binary directory
-add_dependencies(${libname} move_headers)
-
-set_property(GLOBAL APPEND PROPERTY ROOT_EXPORTED_TARGETS ${libname})
-
-if(NOT MSVC)
-  # Make sure that relative RUNPATH to main ROOT libraries is always correct.
-
-  file(RELATIVE_PATH pymoduledir_to_libdir_build ${localruntimedir} "${localruntimedir}")
-  file(RELATIVE_PATH pymoduledir_to_libdir_install ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_PYTHONDIR} "${CMAKE_INSTALL_FULL_LIBDIR}")
-
-  if(APPLE)
-    set_target_properties(${libname} PROPERTIES
-        BUILD_RPATH "@loader_path/${pymoduledir_to_libdir_build}"
-        INSTALL_RPATH "@loader_path/${pymoduledir_to_libdir_install}"
-    )
-  else()
-    set_target_properties(${libname} PROPERTIES
-        BUILD_RPATH "$ORIGIN/${pymoduledir_to_libdir_build}"
-        INSTALL_RPATH "$ORIGIN/${pymoduledir_to_libdir_install}"
-    )
-  endif()
-
-endif()
-
-# Install library
-install(TARGETS ${libname} EXPORT ${CMAKE_PROJECT_NAME}Exports
-                            RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT libraries
-                            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT libraries
-                            ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT libraries)
-if (NOT MSVC AND NOT CMAKE_INSTALL_LIBDIR STREQUAL CMAKE_INSTALL_PYTHONDIR)
-  # add a symlink to ${libname} in CMAKE_INSTALL_PYTHONDIR
-  set(LIB_FILE_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${libname}.so)
-  install(CODE "file(CREATE_LINK ${PYTHONDIR_TO_LIBDIR}/${LIB_FILE_NAME}
-    \$ENV{DESTDIR}${CMAKE_INSTALL_FULL_PYTHONDIR}/${LIB_FILE_NAME} SYMBOLIC)")
-endif()
-
-# Compile .py files
-foreach(py_source ${py_sources})
-  install(CODE "execute_process(COMMAND ${Python3_EXECUTABLE} -m py_compile ${localruntimedir}/${py_source})")
-  install(CODE "execute_process(COMMAND ${Python3_EXECUTABLE} -O -m py_compile ${localruntimedir}/${py_source})")
-endforeach()
-
-
-# Install Python sources and bytecode
-install(DIRECTORY ${localruntimedir}/cppyy_backend
-        DESTINATION ${CMAKE_INSTALL_PYTHONDIR}
-        COMPONENT libraries)
+add_library(cppyy_backend STATIC clingwrapper/src/clingwrapper.cxx)
+target_link_libraries(cppyy_backend Core)
+target_compile_options(cppyy_backend PRIVATE -fPIC)

bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py

@@ -5,7 +5,6 @@
 import sys
 
 from . import _stdcpp_fix
-from cppyy_backend import loader
 
 __all__ = [
     'gbl',
@@ -19,11 +18,19 @@
     '_end_capture_stderr'
     ]
 
-# first load the dependency libraries of the backend, then pull in the
-# libcppyy extension module
-c = loader.load_cpp_backend()
+# First load the dependency libraries of the backend, then pull in the libcppyy
+# extension module. If the backed can't be loaded, it was probably linked
+# statically into the extension module, so we don't error out at this point.
+try:
+    from cppyy_backend import loader
+    c = loader.load_cpp_backend()
+except ImportError:
+    c = None
+
 import libcppyy as _backend
-_backend._cpp_backend = c
+
+if c is not None:
+    _backend._cpp_backend = c
 
 # explicitly expose APIs from libcppyy
 _w = ctypes.CDLL(_backend.__file__, ctypes.RTLD_GLOBAL)

bindings/pyroot/cppyy/patches/cppyy-backend-Fix-backend-loading-on-macOS.patch

@@ -21,19 +21,14 @@
 # Prevent cppyy from filtering ROOT libraries
 os.environ["CPPYY_NO_ROOT_FILTER"] = "1"
 
-root_module_path = os.path.dirname(__file__) # expected to be ${CMAKE_INSTALL_PYTHONDIR}/ROOT
-root_install_pythondir = os.path.dirname(root_module_path) # expected to be ${CMAKE_INSTALL_PYTHONDIR}
-
-# Directly tell cppyy where to find the backend library so we don't have to
-# rely on the LD_LIBRARY_PATH being set
-os.environ["CPPYY_BACKEND_LIBRARY"] = os.path.join(root_install_pythondir, "libcppyy_backend")
-
 # The libROOTPythonizations CPython extension is in the same directory as the
 # ROOT Python module, but to find the other ROOT libraries we need to also add
 # the path of the ROOT library directory (only needed on Windows). For example,
 # if the ROOT Python module is in $ROOTSYS/bin/ROOT/__init__.py, the libraries
 # are usually in $ROOTSYS/bin.
 if 'win32' in sys.platform:
+    root_module_path = os.path.dirname(__file__) # expected to be ${CMAKE_INSTALL_PYTHONDIR}/ROOT
+    root_install_pythondir = os.path.dirname(root_module_path) # expected to be ${CMAKE_INSTALL_PYTHONDIR}
     os.add_dll_directory(root_install_pythondir)
 
 # Do setup specific to AddressSanitizer environments