pipes for redirection in oop jit

Author: kr-2003

clang/include/clang/Interpreter/RemoteJITUtils.h

@@ -26,7 +26,8 @@
 
 llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
 launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory,
-               llvm::StringRef SlabAllocateSizeString);
+               llvm::StringRef SlabAllocateSizeString,
+               std::function<void()> CustomizeFork = nullptr);
 
 /// Create a JITLinkExecutor that connects to the given network address
 /// through a TCP socket. A valid NetworkAddress provides hostname and port,
@@ -35,4 +36,13 @@ llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
 connectTCPSocket(llvm::StringRef NetworkAddress, bool UseSharedMemory,
                  llvm::StringRef SlabAllocateSizeString);
 
+#ifdef LLVM_ON_UNIX
+/// Returns PID of last launched executor.
+pid_t getLastLaunchedExecutorPID();
+
+/// Returns PID of nth launched executor.
+/// 1-based indexing.
+pid_t getNthLaunchedExecutorPID(int n);
+#endif
+
 #endif // LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H

clang/lib/Interpreter/RemoteJITUtils.cpp

@@ -33,6 +33,10 @@
 using namespace llvm;
 using namespace llvm::orc;
 
+#if LLVM_ON_UNIX
+static std::vector<pid_t> LaunchedExecutorPID;
+#endif
+
 Expected<uint64_t> getSlabAllocSize(StringRef SizeString) {
   SizeString = SizeString.trim();
 
@@ -91,7 +95,8 @@ createSharedMemoryManager(SimpleRemoteEPC &SREPC,
 
 Expected<std::unique_ptr<SimpleRemoteEPC>>
 launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
-               llvm::StringRef SlabAllocateSizeString) {
+               llvm::StringRef SlabAllocateSizeString,
+               std::function<void()> CustomizeFork) {
 #ifndef LLVM_ON_UNIX
   // FIXME: Add support for Windows.
   return make_error<StringError>("-" + ExecutablePath +
@@ -134,6 +139,9 @@ launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
     close(ToExecutor[WriteEnd]);
     close(FromExecutor[ReadEnd]);
 
+    if (CustomizeFork)
+      CustomizeFork();
+
     // Execute the child process.
     std::unique_ptr<char[]> ExecutorPath, FDSpecifier;
     {
@@ -155,6 +163,8 @@ launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
              << ExecutorPath.get() << "\"\n";
       exit(1);
     }
+  } else {
+    LaunchedExecutorPID.push_back(ChildPID);
   }
   // else we're the parent...
 
@@ -265,3 +275,18 @@ connectTCPSocket(StringRef NetworkAddress, bool UseSharedMemory,
       std::move(S), *SockFD, *SockFD);
 #endif
 }
+
+#if LLVM_ON_UNIX
+
+pid_t getLastLaunchedExecutorPID() {
+  if (!LaunchedExecutorPID.size())
+    return -1;
+  return LaunchedExecutorPID.back();
+}
+
+pid_t getNthLaunchedExecutorPID(int n) {
+  if (n - 1 < 0 || n - 1 >= static_cast<int>(LaunchedExecutorPID.size()))
+    return -1;
+  return LaunchedExecutorPID.at(n - 1);
+}
+#endif

clang/unittests/Interpreter/CMakeLists.txt

@@ -26,6 +26,13 @@ add_distinct_clang_unittest(ClangReplInterpreterTests
   TargetParser
   )
 
+if(TARGET llvm-jitlink-executor AND TARGET compiler-rt)
+  add_dependencies(ClangReplInterpreterTests
+    llvm-jitlink-executor
+    compiler-rt
+  )
+endif()
+
 # Exceptions on Windows are not yet supported.
 if(NOT WIN32)
   add_subdirectory(ExceptionTests)

clang/unittests/Interpreter/InterpreterTest.cpp

@@ -15,23 +15,36 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclGroup.h"
 #include "clang/AST/Mangle.h"
+#include "clang/Basic/Version.h"
+#include "clang/Config/config.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Interpreter/Interpreter.h"
+#include "clang/Interpreter/RemoteJITUtils.h"
 #include "clang/Interpreter/Value.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Sema.h"
+#include "llvm/Support/Error.h"
+#include "llvm/TargetParser/Host.h"
 
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 using namespace clang;
 
+llvm::ExitOnError ExitOnError;
+
 int Global = 42;
 // JIT reports symbol not found on Windows without the visibility attribute.
 REPL_EXTERNAL_VISIBILITY int getGlobal() { return Global; }
 REPL_EXTERNAL_VISIBILITY void setGlobal(int val) { Global = val; }
 
+#ifdef _WIN32
+#define STDIN_FILENO 0
+#define STDOUT_FILENO 1
+#define STDERR_FILENO 2
+#endif
+
 namespace {
 
 class InterpreterTest : public InterpreterTestBase {
@@ -52,6 +65,93 @@ createInterpreter(const Args &ExtraArgs = {},
   return cantFail(clang::Interpreter::create(std::move(CI)));
 }
 
+static std::string getExecutorPath() {
+  llvm::SmallString<256> ExecutorPath(llvm::sys::fs::getMainExecutable(
+      nullptr, reinterpret_cast<void *>(&getExecutorPath)));
+  llvm::sys::path::remove_filename(ExecutorPath);
+
+  llvm::sys::path::remove_filename(ExecutorPath); // Remove "Interpreter"
+  llvm::sys::path::remove_filename(ExecutorPath); // Remove "unittests"
+  llvm::sys::path::remove_filename(ExecutorPath); // Remove "clang"
+  llvm::sys::path::remove_filename(ExecutorPath); // Remove "tools"
+
+  llvm::sys::path::append(ExecutorPath, "bin", "llvm-jitlink-executor");
+  return ExecutorPath.str().str();
+}
+
+static std::string getOrcRuntimePath() {
+  llvm::SmallString<256> RuntimePath(llvm::sys::fs::getMainExecutable(
+      nullptr, reinterpret_cast<void *>(&getOrcRuntimePath)));
+
+  llvm::sys::path::remove_filename(RuntimePath);
+
+  llvm::sys::path::remove_filename(RuntimePath); // Remove "Interpreter"
+  llvm::sys::path::remove_filename(RuntimePath); // Remove "unittests"
+  llvm::sys::path::remove_filename(RuntimePath); // Remove "clang"
+  llvm::sys::path::remove_filename(RuntimePath); // Remove "tools"
+
+  llvm::sys::path::append(RuntimePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang",
+                          CLANG_VERSION_MAJOR_STRING, "lib");
+
+  llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
+  if (SystemTriple.isOSBinFormatMachO()) {
+    llvm::sys::path::append(RuntimePath, "darwin", "liborc_rt_osx.a");
+  } else if (SystemTriple.isOSBinFormatELF()) {
+    llvm::sys::path::append(RuntimePath, "x86_64-unknown-linux-gnu",
+                            "liborc_rt.a");
+  }
+
+  return RuntimePath.str().str();
+}
+
+static std::unique_ptr<Interpreter>
+createInterpreterWithRemoteExecution(const Args &ExtraArgs = {},
+                                     DiagnosticConsumer *Client = nullptr) {
+  Args ClangArgs = {"-Xclang", "-emit-llvm-only"};
+  llvm::append_range(ClangArgs, ExtraArgs);
+  auto CB = clang::IncrementalCompilerBuilder();
+  CB.SetCompilerArgs(ClangArgs);
+  auto CI = cantFail(CB.CreateCpp());
+  if (Client)
+    CI->getDiagnostics().setClient(Client, /*ShouldOwnClient=*/false);
+
+  std::unique_ptr<llvm::orc::LLJITBuilder> JB;
+
+  llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
+
+  if ((SystemTriple.isOSBinFormatELF() || SystemTriple.isOSBinFormatMachO())) {
+    std::string OOPExecutor = getExecutorPath();
+    std::string OrcRuntimePath = getOrcRuntimePath();
+    bool UseSharedMemory = false;
+    std::string SlabAllocateSizeString = "";
+    std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC;
+    EPC = ExitOnError(launchExecutor(OOPExecutor, UseSharedMemory,
+                                     SlabAllocateSizeString,
+                                     [=] { // Lambda defined inline
+                                       auto redirect = [](int from, int to) {
+                                         if (from != to) {
+                                           dup2(from, to);
+                                           close(from);
+                                         }
+                                       };
+
+                                       redirect(0, STDIN_FILENO);
+                                       redirect(1, STDOUT_FILENO);
+                                       redirect(2, STDERR_FILENO);
+
+                                       setvbuf(stdout, nullptr, _IONBF, 0);
+                                       setvbuf(stderr, nullptr, _IONBF, 0);
+                                     }));
+    if (EPC) {
+      CB.SetTargetTriple(EPC->getTargetTriple().getTriple());
+      JB = ExitOnError(clang::Interpreter::createLLJITBuilder(std::move(EPC),
+                                                              OrcRuntimePath));
+    }
+  }
+
+  return cantFail(clang::Interpreter::create(std::move(CI), std::move(JB)));
+}
+
 static size_t DeclsSize(TranslationUnitDecl *PTUDecl) {
   return std::distance(PTUDecl->decls().begin(), PTUDecl->decls().end());
 }
@@ -68,6 +168,23 @@ TEST_F(InterpreterTest, Sanity) {
   EXPECT_EQ(1U, DeclsSize(R2.TUPart));
 }
 
+TEST_F(InterpreterTest, SanityWithRemoteExecution) {
+  if (!HostSupportsJIT())
+    GTEST_SKIP();
+
+  std::string OrcRuntimePath = getOrcRuntimePath();
+
+  std::unique_ptr<Interpreter> Interp = createInterpreterWithRemoteExecution();
+
+  using PTU = PartialTranslationUnit;
+
+  PTU &R1(cantFail(Interp->Parse("void g(); void g() {}")));
+  EXPECT_EQ(2U, DeclsSize(R1.TUPart));
+
+  PTU &R2(cantFail(Interp->Parse("int i;")));
+  EXPECT_EQ(1U, DeclsSize(R2.TUPart));
+}
+
 static std::string DeclToString(Decl *D) {
   return llvm::cast<NamedDecl>(D)->getQualifiedNameAsString();
 }