[WIP] migrate ROOT's cppyy to leverage InterOp for bindings

bindings/distrdf/python/DistRDF/Backends/Base.py

@@ -49,7 +49,7 @@ def setup_mapper(initialization_fn: Callable, code_to_declare: str) -> None:
     initialization_fn()
 
     # Declare all user code in one call
-    ROOT.gInterpreter.Declare(code_to_declare)
+    ROOT.gCling.Declare(code_to_declare)
 
 
 def get_mergeable_values(starting_node: ROOT.RDF.RNode, range_id: int,
@@ -215,7 +215,7 @@ def register_declaration(cls, code_to_declare):
             code_with_guard = f"#ifndef {hex}\n#define {hex}\n{stripped}\n#endif"
             cls.strings_to_declare[hex] = code_with_guard
 
-        ROOT.gInterpreter.Declare(cls.strings_to_declare[hex])
+        ROOT.gCling.Declare(cls.strings_to_declare[hex])
 
     @classmethod
     def register_shared_lib(cls, paths_to_shared_libraries):

bindings/distrdf/python/DistRDF/Backends/Utils.py

@@ -39,10 +39,10 @@ def extend_include_path(include_path: str) -> None:
             needed for the analysis.
     """
     root_path = "-I{}".format(include_path)
-    ROOT.gInterpreter.AddIncludePath(root_path)
+    ROOT.gCling.AddIncludePath(root_path)
 
     # Retrieve ROOT internal list of include paths and add debug statement
-    root_includepath = ROOT.gInterpreter.GetIncludePath()
+    root_includepath = ROOT.gCling.GetIncludePath()
     logger.debug("ROOT include paths:\n{}".format(root_includepath))
 
 
@@ -62,7 +62,7 @@ def distribute_headers(headers_to_include: Iterable[str]) -> None:
         # Create C++ include code
         include_code = "#include \"{}\"\n".format(header)
         try:
-            ROOT.gInterpreter.Declare(include_code)
+            ROOT.gCling.Declare(include_code)
         except Exception as e:
             msg = "There was an error in including \"{}\" !".format(header)
             raise e(msg)

bindings/distrdf/test/backend/test_common.py

@@ -70,7 +70,7 @@ def test_initialization_runs_in_current_environment(self):
         """
         def defineIntVariable(name, value):
             import ROOT
-            ROOT.gInterpreter.ProcessLine("int %s = %s;" % (name, value))
+            ROOT.gCling.ProcessLine("int %s = %s;" % (name, value))
 
         varvalue = 2
         DistRDF.initialize(defineIntVariable, "myInt", varvalue)

bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/API.h

@@ -25,15 +25,15 @@
 #endif
 #include "Python.h"
 
-#define CPYCPPYY_VERSION_HEX 0x010c10
+#define CPYCPPYY_VERSION_HEX 0x011200
 
 // Cppyy types
 namespace Cppyy {
-    typedef size_t      TCppScope_t;
+    typedef void*       TCppScope_t;
     typedef TCppScope_t TCppType_t;
     typedef void*       TCppEnum_t;
     typedef void*       TCppObject_t;
-    typedef intptr_t    TCppMethod_t;
+    typedef void*       TCppMethod_t;
 
     typedef size_t      TCppIndex_t;
     typedef void*       TCppFuncAddr_t;
@@ -124,6 +124,7 @@ class CPYCPPYY_CLASS_EXTERN Converter {
 
 // create a converter based on its full type name and dimensions
 CPYCPPYY_EXTERN Converter* CreateConverter(const std::string& name, cdims_t = 0);
+CPYCPPYY_EXTERN Converter* CreateConverter(Cppyy::TCppType_t type, cdims_t = 0);
 
 // delete a previously created converter
 CPYCPPYY_EXTERN void DestroyConverter(Converter* p);
@@ -154,6 +155,7 @@ class CPYCPPYY_CLASS_EXTERN Executor {
 
 // create an executor based on its full type name
 CPYCPPYY_EXTERN Executor* CreateExecutor(const std::string& name, cdims_t = 0);
+CPYCPPYY_EXTERN Executor* CreateExecutor(Cppyy::TCppType_t type, cdims_t = 0);
 
 // delete a previously created executor
 CPYCPPYY_EXTERN void DestroyConverter(Converter* p);
@@ -180,7 +182,8 @@ CPYCPPYY_EXTERN void* Instance_AsVoidPtr(PyObject* pyobject);
 // void* to C++ Instance (python object proxy) conversion, returns a new reference
 CPYCPPYY_EXTERN PyObject* Instance_FromVoidPtr(
     void* addr, const std::string& classname, bool python_owns = false);
-
+CPYCPPYY_EXTERN PyObject* Instance_FromVoidPtr(
+    void* addr, Cppyy::TCppScope_t klass_scope, bool python_owns = false);
 // type verifiers for C++ Scope
 CPYCPPYY_EXTERN bool Scope_Check(PyObject* pyobject);
 CPYCPPYY_EXTERN bool Scope_CheckExact(PyObject* pyobject);

bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/CommonDefs.h

@@ -6,29 +6,29 @@
 #ifdef _MSC_VER
 // Windows requires symbols to be explicitly exported
 #define CPYCPPYY_EXPORT extern __declspec(dllexport)
+#define CPYCPPYY_IMPORT extern __declspec(dllimport)
 #define CPYCPPYY_CLASS_EXPORT __declspec(dllexport)
 
 // CPYCPPYY_EXTERN is dual use in the public API
 #ifndef CPYCPPYY_INTERNAL
 #define CPYCPPYY_EXTERN extern __declspec(dllexport)
 #define CPYCPPYY_CLASS_EXTERN __declspec(dllexport)
 #else
-#define CPYCPPYY_EXTERN extern
-#define CPYCPPYY_CLASS_EXTERN
+#define CPYCPPYY_EXTERN extern __declspec(dllimport)
+#define CPYCPPYY_CLASS_EXTERN __declspec(dllimport)
 #endif
 
 #define CPYCPPYY_STATIC
 
 #else
 // Linux, Mac, etc.
 #define CPYCPPYY_EXPORT extern
+#define CPYCPPYY_IMPORT extern
 #define CPYCPPYY_CLASS_EXPORT
 #define CPYCPPYY_EXTERN extern
 #define CPYCPPYY_CLASS_EXTERN
 #define CPYCPPYY_STATIC static
 
 #endif
 
-#define CPYCPPYY_IMPORT extern
-
 #endif // !CPYCPPYY_COMMONDEFS_H

bindings/pyroot/cppyy/CPyCppyy/src/API.cxx

@@ -135,6 +135,23 @@ PyObject* CPyCppyy::Instance_FromVoidPtr(
     return pyobject;
 }
 
+//-----------------------------------------------------------------------------
+PyObject* CPyCppyy::Instance_FromVoidPtr(
+    void* addr, Cppyy::TCppScope_t klass_scope, bool python_owns)
+{
+// Bind the addr to a python object of class defined by classname.
+    if (!Initialize())
+        return nullptr;
+
+// perform cast (the call will check TClass and addr, and set python errors)
+    PyObject* pyobject = BindCppObjectNoCast(addr, klass_scope, false);
+
+// give ownership, for ref-counting, to the python side, if so requested
+    if (python_owns && CPPInstance_Check(pyobject))
+        ((CPPInstance*)pyobject)->PythonOwns();
+
+    return pyobject;
+}
 namespace CPyCppyy {
 // version with C type arguments only for use with Numba
 PyObject* Instance_FromVoidPtr(void* addr, const char* classname, int python_owns) {
@@ -226,7 +243,7 @@ bool CPyCppyy::Instance_IsLively(PyObject* pyobject)
 
 // the instance fails the lively test if it owns the C++ object while having a
 // reference count of 1 (meaning: it could delete the C++ instance any moment)
-    if (Py_REFCNT(pyobject) <= 1 && (((CPPInstance*)pyobject)->fFlags & CPPInstance::kIsOwner))
+    if (pyobject->ob_refcnt <= 1 && (((CPPInstance*)pyobject)->fFlags & CPPInstance::kIsOwner))
         return false;
 
     return true;

bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.cxx

@@ -15,6 +15,8 @@
 //- data _____________________________________________________________________
 namespace CPyCppyy {
     extern PyObject* gNullPtrObject;
+    void* Instance_AsVoidPtr(PyObject* pyobject);
+    PyObject* Instance_FromVoidPtr(void* addr, Cppyy::TCppScope_t klass_scope, bool python_owns);
 }
 
 
@@ -44,7 +46,7 @@ PyObject* CPyCppyy::CPPConstructor::Reflex(
     if (request == Cppyy::Reflex::RETURN_TYPE) {
         std::string fn = Cppyy::GetScopedFinalName(this->GetScope());
         if (format == Cppyy::Reflex::OPTIMAL || format == Cppyy::Reflex::AS_TYPE)
-            return CreateScopeProxy(fn);
+            return CreateScopeProxy(this->GetScope());
         else if (format == Cppyy::Reflex::AS_STRING)
             return CPyCppyy_PyText_FromString(fn.c_str());
     }
@@ -56,7 +58,6 @@ PyObject* CPyCppyy::CPPConstructor::Reflex(
 PyObject* CPyCppyy::CPPConstructor::Call(CPPInstance*& self,
     CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt)
 {
-
 // setup as necessary
     if (fArgsRequired == -1 && !this->Initialize(ctxt))
         return nullptr;                     // important: 0, not Py_None
@@ -78,15 +79,6 @@ PyObject* CPyCppyy::CPPConstructor::Call(CPPInstance*& self,
         return nullptr;
     }
 
-    const auto cppScopeFlags = ((CPPScope*)Py_TYPE(self))->fFlags;
-
-// Do nothing if the constructor is explicit and we are in an implicit
-// conversion context. We recognize this by checking the CPPScope::kNoImplicit
-// flag, as further implicit conversions are disabled to prevent infinite
-// recursion. See also the ConvertImplicit() helper in Converters.cxx.
-    if((cppScopeFlags & CPPScope::kNoImplicit) && Cppyy::IsExplicit(GetMethod()))
-        return nullptr;
-
 // self provides the python context for lifelines
     if (!ctxt->fPyContext)
         ctxt->fPyContext = (PyObject*)cargs.fSelf;    // no Py_INCREF as no ownership
@@ -135,7 +127,7 @@ PyObject* CPyCppyy::CPPConstructor::Call(CPPInstance*& self,
 
     } else {
     // translate the arguments
-        if (cppScopeFlags & CPPScope::kNoImplicit)
+        if (((CPPClass*)Py_TYPE(self))->fFlags & CPPScope::kNoImplicit)
             ctxt->fFlags |= CallContext::kNoImplicit;
         if (!this->ConvertAndSetArgs(cargs.fArgs, cargs.fNArgsf, ctxt))
             return nullptr;
@@ -182,6 +174,7 @@ PyObject* CPyCppyy::CPPConstructor::Call(CPPInstance*& self,
     return nullptr;
 }
 
+
 //----------------------------------------------------------------------------
 CPyCppyy::CPPMultiConstructor::CPPMultiConstructor(Cppyy::TCppScope_t scope, Cppyy::TCppMethod_t method) :
     CPPConstructor(scope, method)

bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.cxx

@@ -4,6 +4,7 @@
 #include "PyStrings.h"
 #include "CPPDataMember.h"
 #include "CPPInstance.h"
+#include "CPPEnum.h"
 #include "Dimensions.h"
 #include "LowLevelViews.h"
 #include "ProxyWrappers.h"
@@ -47,30 +48,23 @@ static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls
         }
     }
 
-// non-initialized or public data accesses through class (e.g. by help())
-    void* address = dm->GetAddress(pyobj);
-    if (!address || (intptr_t)address == -1 /* Cling error */)
-        return nullptr;
 
     if (dm->fFlags & (kIsEnumPrep | kIsEnumType)) {
         if (dm->fFlags & kIsEnumPrep) {
         // still need to do lookup; only ever try this once, then fallback on converter
             dm->fFlags &= ~kIsEnumPrep;
 
         // fDescription contains the full name of the actual enum value object
-            const std::string& lookup = CPyCppyy_PyText_AsString(dm->fDescription);
-            const std::string& enum_type  = TypeManip::extract_namespace(lookup);
-            const std::string& enum_scope = TypeManip::extract_namespace(enum_type);
+            const Cppyy::TCppScope_t enum_type  = Cppyy::GetParentScope(dm->fScope);
+            const Cppyy::TCppScope_t enum_scope = Cppyy::GetParentScope(enum_type);
 
-            PyObject* pyscope = nullptr;
-            if (enum_scope.empty()) pyscope = GetScopeProxy(Cppyy::gGlobalScope);
-            else pyscope = CreateScopeProxy(enum_scope);
+            PyObject* pyscope = CreateScopeProxy(enum_scope);
             if (pyscope) {
-                PyObject* pyEnumType = PyObject_GetAttrString(pyscope,
-                    enum_type.substr(enum_scope.size() ? enum_scope.size()+2 : 0, std::string::npos).c_str());
+                PyObject* pyEnumType = 
+                    PyObject_GetAttrString(pyscope, Cppyy::GetFinalName(enum_type).c_str());
                 if (pyEnumType) {
-                    PyObject* pyval = PyObject_GetAttrString(pyEnumType,
-                        lookup.substr(enum_type.size()+2, std::string::npos).c_str());
+                    PyObject* pyval =
+                        PyObject_GetAttrString(pyEnumType, Cppyy::GetFinalName(dm->fScope).c_str());
                     Py_DECREF(pyEnumType);
                     if (pyval) {
                         Py_DECREF(dm->fDescription);
@@ -88,7 +82,16 @@ static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls
             Py_INCREF(dm->fDescription);
             return dm->fDescription;
         }
+
+        if (Cppyy::IsEnumConstant(dm->fScope)) {
+            // anonymous enum
+            return pyval_from_enum(Cppyy::ResolveEnum(dm->fScope), nullptr, nullptr, dm->fScope);
+        }
     }
+// non-initialized or public data accesses through class (e.g. by help())
+    void* address = dm->GetAddress(pyobj);
+    if (!address || (intptr_t)address == -1 /* Cling error */)
+        return nullptr;
 
     if (dm->fConverter != 0) {
         PyObject* result = dm->fConverter->FromMemory((dm->fFlags & kIsArrayType) ? &address : address);
@@ -319,53 +322,54 @@ PyTypeObject CPPDataMember_Type = {
 
 
 //- public members -----------------------------------------------------------
-void CPyCppyy::CPPDataMember::Set(Cppyy::TCppScope_t scope, Cppyy::TCppIndex_t idata)
+void CPyCppyy::CPPDataMember::Set(Cppyy::TCppScope_t scope, Cppyy::TCppScope_t data)
 {
-    fEnclosingScope = scope;
-    fOffset         = Cppyy::GetDatamemberOffset(scope, idata); // TODO: make lazy
-    fFlags          = Cppyy::IsStaticData(scope, idata) ? kIsStaticData : 0;
-
-    std::vector<dim_t> dims;
-    int ndim = 0; Py_ssize_t size = 0;
-    while (0 < (size = Cppyy::GetDimensionSize(scope, idata, ndim))) {
-         ndim += 1;
-         if (size == INT_MAX)      // meaning: incomplete array type
-             size = UNKNOWN_SIZE;
-         if (ndim == 1) dims.reserve(4);
-         dims.push_back((dim_t)size);
+    if (Cppyy::IsLambdaClass(Cppyy::GetDatamemberType(data))) {
+        fScope = Cppyy::WrapLambdaFromVariable(data);
+    } else {
+        fScope = data;
     }
-    if (!dims.empty())
-        fFlags |= kIsArrayType;
 
-    const std::string name = Cppyy::GetDatamemberName(scope, idata);
-    fFullType = Cppyy::GetDatamemberType(scope, idata);
-    if (Cppyy::IsEnumData(scope, idata)) {
+    fEnclosingScope = scope;
+    fOffset         = Cppyy::GetDatamemberOffset(fScope, fScope == data ? scope : Cppyy::GetScope("__cppyy_internal_wrap_g")); // XXX: Check back here // TODO: make lazy
+    fFlags          = Cppyy::IsStaticDatamember(fScope) ? kIsStaticData : 0;
+
+    const std::string name = Cppyy::GetFinalName(fScope);
+    Cppyy::TCppType_t type;
+
+
+    if (Cppyy::IsEnumConstant(fScope)) {
+        type = Cppyy::GetEnumConstantType(fScope);
+        fFullType = Cppyy::GetTypeAsString(type);
         if (fFullType.find("(anonymous)") == std::string::npos &&
             fFullType.find("(unnamed)")   == std::string::npos) {
         // repurpose fDescription for lazy lookup of the enum later
             fDescription = CPyCppyy_PyText_FromString((fFullType + "::" + name).c_str());
             fFlags |= kIsEnumPrep;
         }
-        fFullType = Cppyy::ResolveEnum(fFullType);
-        fFlags |= kIsConstData;
-    } else if (Cppyy::IsConstData(scope, idata)) {
+        type = Cppyy::ResolveType(type);
         fFlags |= kIsConstData;
+    } else {
+        type = Cppyy::GetDatamemberType(fScope);
+        fFullType = Cppyy::GetTypeAsString(type);
+
+        // Get the integer type if it's an enum
+        if (Cppyy::IsEnumType(type))
+            type = Cppyy::ResolveType(type);
+
+        if (Cppyy::IsConstVar(fScope))
+            fFlags |= kIsConstData;
     }
 
-// if this data member is an array, the conversion needs to be pointer to object for instances,
-// to prevent the need for copying in the conversion; furthermore, fixed arrays' full type for
-// builtins are not declared as such if more than 1-dim (TODO: fix in clingwrapper)
-    if (!dims.empty() && fFullType.back() != '*') {
-        if (Cppyy::GetScope(fFullType)) fFullType += '*';
-        else if (fFullType.back() != ']') {
-            for (auto d: dims) fFullType += d == UNKNOWN_SIZE ? "*" : "[]";
-        }
-    }
+    std::vector<dim_t> dims = Cppyy::GetDimensions(type);
+
+    if (!dims.empty())
+        fFlags |= kIsArrayType;
 
     if (dims.empty())
-        fConverter = CreateConverter(fFullType);
+        fConverter = CreateConverter(type, 0);
     else
-        fConverter = CreateConverter(fFullType, {(dim_t)dims.size(), dims.data()});
+        fConverter = CreateConverter(type, {(dim_t)dims.size(), dims.data()});
 
     if (!(fFlags & kIsEnumPrep))
         fDescription = CPyCppyy_PyText_FromString(name.c_str());