Merge pull request #1 from Vipul-Cariappa/mcbarton/Add-llvm-21-support

Mcbarton/add llvm 21 support

Author: mcbarton

docs/DevelopersDocumentation.rst

@@ -423,12 +423,54 @@ library files and run pytest:
    python -m pip install pytest
    python -m pytest -sv
 
-***********************************
+###################################
  CppInterOp Internal Documentation
-***********************************
+###################################
 
 CppInterOp maintains an internal Doxygen documentation of its components.
 Internal documentation aims to capture intrinsic details and overall usage of
 code components. The goal of internal documentation is to make the codebase
 easier to understand for the new developers. Internal documentation can be
 visited : `here <build/html/index.html>`_
+
+**************************************
+ Multiple Interpreter & Thread-Safety
+**************************************
+
+CppInterOp allows the user to create multiple interpreters at a time and
+use those interpreters. The interpreters that are created are stored in a
+stack and a map. The stack is used to enable the model where the user
+wants to create a temporary interpreter and destroy it after performing a
+few operations. In such a use case, the top of the stack is the only
+interpreter in use at any given point in time.
+
+The map is used to store the mapping from :code:`clang::ASTContext` to
+:code:`Cpp::InterpreterInfo`. This is required to figure out which
+interpreter an object belongs to. Say the library user performs the
+following operations:
+
+1. Create an Interpreter
+2. Compile some code with variable :code:`a`
+3. Create another Interpreter
+4. Performs :code:`Cpp::GetVariableOffset(a)`
+
+In step 4, the top of the stack is an interpreter without the definition of
+:code:`a`. And we cannot use it to figure out the address of :code:`a`.
+The :code:`clang::Decl` passed to :code:`Cpp::GetVariableOffset` is used to
+retrieve the :code:`clang::ASTContext`, using
+:code:`clang::Decl::getASTContext`. We then use the map to figure out the
+exact Interpreter Instance this :code:`clang::Decl` belongs to and perform
+the operation.
+
+A shortcoming of this is that if the CppInterOp accepts a
+:code:`clang::QualType` instead of :code:`clang::Decl`, then it is not
+possible to get the :code:`clang::ASTContext` from the :code:`clang::QualType`.
+In such cases, we iterate over the Allocator of all the Interpreters in our
+stack and figure out which :code:`clang::ASTContext` allocated this
+:code:`clang::QualType`. This is a very expensive operation. But there is no
+alternative to this.
+
+For **thread-safety**, we introduce a lock for each of the interpreters we
+create. And lock only that one specific interpreter when required. We also
+have 2 global locks, one for LLVM, and another is used to lock operations
+performed on the interpreter stack and the map itself.

include/CppInterOp/CppInterOp.h

@@ -389,7 +389,7 @@ CPPINTEROP_API std::string GetQualifiedCompleteName(TCppScope_t klass);
 CPPINTEROP_API std::vector<TCppScope_t> GetUsingNamespaces(TCppScope_t scope);
 
 /// Gets the global scope of the whole C++  instance.
-CPPINTEROP_API TCppScope_t GetGlobalScope();
+CPPINTEROP_API TCppScope_t GetGlobalScope(TInterp_t interp = nullptr);
 
 /// Strips the typedef and returns the underlying class, and if the
 /// underlying decl is not a class it returns the input unchanged.
@@ -398,18 +398,28 @@ CPPINTEROP_API TCppScope_t GetUnderlyingScope(TCppScope_t scope);
 /// Gets the namespace or class (by stripping typedefs) for the name
 /// passed as a parameter, and if the parent is not passed,
 /// then global scope will be assumed.
+/// Looks up the name in parent, if parent is nullptr,
+/// interp is used to select the interpreter if multiple in-use.
+/// interp is ignored if parent is non-null.
 CPPINTEROP_API TCppScope_t GetScope(const std::string& name,
-                                    TCppScope_t parent = nullptr);
+                                    TCppScope_t parent = nullptr,
+                                    TInterp_t interp = nullptr);
 
 /// When the namespace is known, then the parent doesn't need
 /// to be specified. This will probably be phased-out in
 /// future versions of the interop library.
-CPPINTEROP_API TCppScope_t GetScopeFromCompleteName(const std::string& name);
+/// interp is used to select the interpreter if multiple in-use.
+CPPINTEROP_API TCppScope_t GetScopeFromCompleteName(const std::string& name,
+                                                    TInterp_t interp = nullptr);
 
 /// This function performs a lookup within the specified parent,
 /// a specific named entity (functions, enums, etcetera).
+/// Looks up the name in parent, if parent is nullptr,
+/// interp is used to select the interpreter if multiple in-use.
+/// interp is ignored if parent is non-null.
 CPPINTEROP_API TCppScope_t GetNamed(const std::string& name,
-                                    TCppScope_t parent = nullptr);
+                                    TCppScope_t parent = nullptr,
+                                    TInterp_t interp = nullptr);
 
 /// Gets the parent of the scope that is passed as a parameter.
 CPPINTEROP_API TCppScope_t GetParentScope(TCppScope_t scope);
@@ -493,8 +503,12 @@ CPPINTEROP_API bool IsTemplatedFunction(TCppFunction_t func);
 
 /// This function performs a lookup to check if there is a
 /// templated function of that type.
+/// Looks up the name in parent, if parent is nullptr,
+/// interp is used to select the interpreter if multiple in-use.
+/// interp is ignored if parent is non-null.
 CPPINTEROP_API bool ExistsFunctionTemplate(const std::string& name,
-                                           TCppScope_t parent = nullptr);
+                                           TCppScope_t parent = nullptr,
+                                           TInterp_t interp = nullptr);
 
 /// Sets a list of all the constructor for a scope/class that is
 /// supplied as a parameter.
@@ -542,7 +556,8 @@ CPPINTEROP_API bool IsDestructor(TCppConstFunction_t method);
 CPPINTEROP_API bool IsStaticMethod(TCppConstFunction_t method);
 
 ///\returns the address of the function given its potentially mangled name.
-CPPINTEROP_API TCppFuncAddr_t GetFunctionAddress(const char* mangled_name);
+CPPINTEROP_API TCppFuncAddr_t GetFunctionAddress(const char* mangled_name,
+                                                 TInterp_t interp = nullptr);
 
 ///\returns the address of the function given its function declaration.
 CPPINTEROP_API TCppFuncAddr_t GetFunctionAddress(TCppFunction_t method);
@@ -643,7 +658,9 @@ CPPINTEROP_API TCppType_t GetCanonicalType(TCppType_t type);
 
 /// Used to either get the built-in type of the provided string, or
 /// use the name to lookup the actual type.
-CPPINTEROP_API TCppType_t GetType(const std::string& type);
+/// interp is used to select the interpreter if multiple in-use.
+CPPINTEROP_API TCppType_t GetType(const std::string& type,
+                                  TInterp_t interp = nullptr);
 
 ///\returns the complex of the provided type.
 CPPINTEROP_API TCppType_t GetComplexType(TCppType_t element_type);
@@ -702,6 +719,11 @@ CreateInterpreter(const std::vector<const char*>& Args = {},
 ///\returns false on failure or if \c I is not tracked in the stack.
 CPPINTEROP_API bool DeleteInterpreter(TInterp_t I = nullptr);
 
+/// Take ownership of an interpreter instance.
+///\param[in] I - the interpreter to be taken, if nullptr, returns the last.
+///\returns nullptr on failure or if \c I is not tracked in the stack.
+CPPINTEROP_API TInterp_t TakeInterpreter(TInterp_t I = nullptr);
+
 /// Activates an instance of an interpreter to handle subsequent API requests
 ///\param[in] I - the interpreter to be activated.
 ///\returns false on failure.
@@ -722,10 +744,10 @@ CPPINTEROP_API void UseExternalInterpreter(TInterp_t I);
 
 /// Adds a Search Path for the Interpreter to get the libraries.
 CPPINTEROP_API void AddSearchPath(const char* dir, bool isUser = true,
-                                  bool prepend = false);
+                                  bool prepend = false, TInterp_t I = nullptr);
 
 /// Returns the resource-dir path (for headers).
-CPPINTEROP_API const char* GetResourceDir();
+CPPINTEROP_API const char* GetResourceDir(TInterp_t I = nullptr);
 
 /// Uses the underlying clang compiler to detect the resource directory.
 /// In essence calling clang -print-resource-dir and checks if it ends with
@@ -746,46 +768,52 @@ DetectSystemCompilerIncludePaths(std::vector<std::string>& Paths,
 
 /// Secondary search path for headers, if not found using the
 /// GetResourceDir() function.
-CPPINTEROP_API void AddIncludePath(const char* dir);
+CPPINTEROP_API void AddIncludePath(const char* dir, TInterp_t I = nullptr);
 
 // Gets the currently used include paths
 ///\param[out] IncludePaths - the list of include paths
 ///
 CPPINTEROP_API void GetIncludePaths(std::vector<std::string>& IncludePaths,
                                     bool withSystem = false,
-                                    bool withFlags = false);
+                                    bool withFlags = false,
+                                    TInterp_t I = nullptr);
 
 /// Only Declares a code snippet in \c code and does not execute it.
 ///\returns 0 on success
-CPPINTEROP_API int Declare(const char* code, bool silent = false);
+CPPINTEROP_API int Declare(const char* code, bool silent = false,
+                           TInterp_t I = nullptr);
 
 /// Declares and executes a code snippet in \c code.
 ///\returns 0 on success
-CPPINTEROP_API int Process(const char* code);
+CPPINTEROP_API int Process(const char* code, TInterp_t I = nullptr);
 
 /// Declares, executes and returns the execution result as a intptr_t.
 ///\returns the expression results as a intptr_t.
-CPPINTEROP_API intptr_t Evaluate(const char* code, bool* HadError = nullptr);
+CPPINTEROP_API intptr_t Evaluate(const char* code, bool* HadError = nullptr,
+                                 TInterp_t I = nullptr);
 
 /// Looks up the library if access is enabled.
 ///\returns the path to the library.
-CPPINTEROP_API std::string LookupLibrary(const char* lib_name);
+CPPINTEROP_API std::string LookupLibrary(const char* lib_name,
+                                         TInterp_t I = nullptr);
 
 /// Finds \c lib_stem considering the list of search paths and loads it by
 /// calling dlopen.
 /// \returns true on success.
-CPPINTEROP_API bool LoadLibrary(const char* lib_stem, bool lookup = true);
+CPPINTEROP_API bool LoadLibrary(const char* lib_stem, bool lookup = true,
+                                TInterp_t I = nullptr);
 
 /// Finds \c lib_stem considering the list of search paths and unloads it by
 /// calling dlclose.
 /// function.
-CPPINTEROP_API void UnloadLibrary(const char* lib_stem);
+CPPINTEROP_API void UnloadLibrary(const char* lib_stem, TInterp_t I = nullptr);
 
 /// Scans all libraries on the library search path for a given potentially
 /// mangled symbol name.
 ///\returns the path to the first library that contains the symbol definition.
-CPPINTEROP_API std::string
-SearchLibrariesForSymbol(const char* mangled_name, bool search_system /*true*/);
+CPPINTEROP_API std::string SearchLibrariesForSymbol(const char* mangled_name,
+                                                    bool search_system /*true*/,
+                                                    TInterp_t I = nullptr);
 
 /// Inserts or replaces a symbol in the JIT with the one provided. This is
 /// useful for providing our own implementations of facilities such as printf.
@@ -796,7 +824,8 @@ SearchLibrariesForSymbol(const char* mangled_name, bool search_system /*true*/);
 ///
 ///\returns true on failure.
 CPPINTEROP_API bool InsertOrReplaceJitSymbol(const char* linker_mangled_name,
-                                             uint64_t address);
+                                             uint64_t address,
+                                             TInterp_t I = nullptr);
 
 /// Tries to load provided objects in a string format (prettyprint).
 CPPINTEROP_API std::string ObjToString(const char* type, void* obj);
@@ -833,9 +862,10 @@ GetClassTemplateInstantiationArgs(TCppScope_t templ_instance,
 
 /// Instantiates a function template from a given string representation. This
 /// function also does overload resolution.
+///\param[in] interp - is used to select the interpreter if multiple in-use.
 ///\returns the instantiated function template declaration.
-CPPINTEROP_API TCppFunction_t
-InstantiateTemplateFunctionFromString(const char* function_template);
+CPPINTEROP_API TCppFunction_t InstantiateTemplateFunctionFromString(
+    const char* function_template, TInterp_t interp = nullptr);
 
 /// Finds best overload match based on explicit template parameters (if any)
 /// and argument types.
@@ -932,7 +962,7 @@ CPPINTEROP_API void CodeComplete(std::vector<std::string>& Results,
 /// Reverts the last N operations performed by the interpreter.
 ///\param[in] N The number of operations to undo. Defaults to 1.
 ///\returns 0 on success, non-zero on failure.
-CPPINTEROP_API int Undo(unsigned N = 1);
+CPPINTEROP_API int Undo(unsigned N = 1, TInterp_t interp = nullptr);
 
 } // end namespace Cpp
 

lib/CppInterOp/Compatibility.h

@@ -429,6 +429,7 @@ inline void InstantiateClassTemplateSpecialization(
 #if CLANG_VERSION_MAJOR < 20
   interp.getSema().InstantiateClassTemplateSpecialization(
       clang::SourceLocation::getFromRawEncoding(1), CTSD,
+
       clang::TemplateSpecializationKind::TSK_ExplicitInstantiationDefinition,
       /*Complain=*/true);
 #else