Skip to content

Commit c642e2a

Browse files
kr-2003kr-2003
andauthored
[clang-repl] Add support for running custom code in Remote JIT executor (llvm#157358)
Introduce a custom lambda mechanism that allows injecting user-defined code into the Remote JIT’s executor. --------- Co-authored-by: kr-2003 <[email protected]>
1 parent 0c3cf20 commit c642e2a

File tree

6 files changed

+237
-5
lines changed

6 files changed

+237
-5
lines changed

clang/include/clang/Interpreter/Interpreter.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,13 +135,15 @@ class Interpreter {
135135
std::string OrcRuntimePath = "";
136136
/// PID of the out-of-process JIT executor.
137137
uint32_t ExecutorPID = 0;
138+
/// Custom lambda to be executed inside child process/executor
139+
std::function<void()> CustomizeFork = nullptr;
138140
/// An optional code model to provide to the JITTargetMachineBuilder
139141
std::optional<llvm::CodeModel::Model> CM = std::nullopt;
140142

141143
JITConfig()
142144
: IsOutOfProcess(false), OOPExecutor(""), OOPExecutorConnect(""),
143145
UseSharedMemory(false), SlabAllocateSize(0), OrcRuntimePath(""),
144-
ExecutorPID(0), CM(std::nullopt) {}
146+
ExecutorPID(0), CustomizeFork(nullptr), CM(std::nullopt) {}
145147
};
146148

147149
protected:

clang/lib/Interpreter/IncrementalExecutor.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,8 @@ createSharedMemoryManager(llvm::orc::SimpleRemoteEPC &SREPC,
172172
llvm::Expected<std::pair<std::unique_ptr<llvm::orc::SimpleRemoteEPC>, uint32_t>>
173173
IncrementalExecutor::launchExecutor(llvm::StringRef ExecutablePath,
174174
bool UseSharedMemory,
175-
unsigned SlabAllocateSize) {
175+
unsigned SlabAllocateSize,
176+
std::function<void()> CustomizeFork) {
176177
#ifndef LLVM_ON_UNIX
177178
// FIXME: Add support for Windows.
178179
return llvm::make_error<llvm::StringError>(
@@ -215,6 +216,9 @@ IncrementalExecutor::launchExecutor(llvm::StringRef ExecutablePath,
215216
close(ToExecutor[WriteEnd]);
216217
close(FromExecutor[ReadEnd]);
217218

219+
if (CustomizeFork)
220+
CustomizeFork();
221+
218222
// Execute the child process.
219223
std::unique_ptr<char[]> ExecutorPath, FDSpecifier;
220224
{

clang/lib/Interpreter/IncrementalExecutor.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ class IncrementalExecutor {
7979
static llvm::Expected<
8080
std::pair<std::unique_ptr<llvm::orc::SimpleRemoteEPC>, uint32_t>>
8181
launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory,
82-
unsigned SlabAllocateSize);
82+
unsigned SlabAllocateSize,
83+
std::function<void()> CustomizeFork = nullptr);
8384

8485
#if LLVM_ON_UNIX && LLVM_ENABLE_THREADS
8586
static llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>

clang/lib/Interpreter/Interpreter.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -355,7 +355,8 @@ Interpreter::outOfProcessJITBuilder(JITConfig Config) {
355355
if (!Config.OOPExecutor.empty()) {
356356
// Launch an out-of-process executor locally in a child process.
357357
auto ResultOrErr = IncrementalExecutor::launchExecutor(
358-
Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize);
358+
Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize,
359+
Config.CustomizeFork);
359360
if (!ResultOrErr)
360361
return ResultOrErr.takeError();
361362
childPid = ResultOrErr->second;

clang/unittests/Interpreter/CMakeLists.txt

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,25 @@ set(CLANG_LIBS_TO_LINK
2929
)
3030
endif()
3131

32-
add_distinct_clang_unittest(ClangReplInterpreterTests
32+
set(CLANG_REPL_TEST_SOURCES
3333
IncrementalCompilerBuilderTest.cpp
3434
IncrementalProcessingTest.cpp
3535
InterpreterTest.cpp
3636
InterpreterExtensionsTest.cpp
3737
CodeCompletionTest.cpp
38+
)
39+
40+
if(TARGET compiler-rt)
41+
list(APPEND CLANG_REPL_TEST_SOURCES
42+
OutOfProcessInterpreterTests.cpp
43+
)
44+
message(STATUS "Compiler-RT found, enabling out of process JIT tests")
45+
endif()
46+
47+
add_distinct_clang_unittest(ClangReplInterpreterTests
48+
${CLANG_REPL_TEST_SOURCES}
49+
50+
PARTIAL_SOURCES_INTENDED
3851

3952
EXPORT_SYMBOLS
4053

@@ -48,6 +61,14 @@ add_distinct_clang_unittest(ClangReplInterpreterTests
4861
${LLVM_COMPONENTS_TO_LINK}
4962
)
5063

64+
if(TARGET compiler-rt)
65+
add_dependencies(ClangReplInterpreterTests
66+
llvm-jitlink-executor
67+
compiler-rt
68+
)
69+
message(STATUS "Adding dependency on compiler-rt for out of process JIT tests")
70+
endif()
71+
5172
if(EMSCRIPTEN)
5273
# Without the above you try to link to LLVMSupport twice, and end
5374
# up with a duplicate symbol error when creating the main module
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
//===- unittests/Interpreter/OutOfProcessInterpreterTest.cpp --- Interpreter
2+
// tests when Out-of-Process ----===//
3+
//
4+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5+
// See https://llvm.org/LICENSE.txt for license information.
6+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7+
//
8+
//===----------------------------------------------------------------------===//
9+
//
10+
// Unit tests for Clang's Interpreter library.
11+
//
12+
//===----------------------------------------------------------------------===//
13+
14+
#include "InterpreterTestFixture.h"
15+
#include "clang/AST/Decl.h"
16+
#include "clang/AST/DeclGroup.h"
17+
#include "clang/AST/Mangle.h"
18+
#include "clang/Basic/Version.h"
19+
#include "clang/Config/config.h"
20+
#include "clang/Frontend/CompilerInstance.h"
21+
#include "clang/Frontend/TextDiagnosticPrinter.h"
22+
#include "clang/Interpreter/Interpreter.h"
23+
#include "clang/Interpreter/Value.h"
24+
#include "clang/Sema/Lookup.h"
25+
#include "clang/Sema/Sema.h"
26+
#include "llvm/Support/Error.h"
27+
#include "llvm/TargetParser/Host.h"
28+
#include "gmock/gmock.h"
29+
#include "gtest/gtest.h"
30+
#include <memory>
31+
#include <signal.h>
32+
#include <sstream>
33+
#include <unistd.h>
34+
35+
using namespace clang;
36+
37+
llvm::ExitOnError ExitOnError;
38+
39+
namespace {
40+
41+
using Args = std::vector<const char *>;
42+
43+
struct FileDeleter {
44+
void operator()(FILE *f) {
45+
if (f)
46+
fclose(f);
47+
}
48+
};
49+
50+
struct IOContext {
51+
std::unique_ptr<FILE, FileDeleter> stdin_file;
52+
std::unique_ptr<FILE, FileDeleter> stdout_file;
53+
std::unique_ptr<FILE, FileDeleter> stderr_file;
54+
55+
bool initializeTempFiles() {
56+
stdin_file.reset(tmpfile());
57+
stdout_file.reset(tmpfile());
58+
stderr_file.reset(tmpfile());
59+
return stdin_file && stdout_file && stderr_file;
60+
}
61+
62+
std::string readStdoutContent() {
63+
if (!stdout_file)
64+
return "";
65+
rewind(stdout_file.get());
66+
std::ostringstream content;
67+
char buffer[1024];
68+
size_t bytes_read;
69+
while ((bytes_read = fread(buffer, 1, sizeof(buffer), stdout_file.get())) >
70+
0) {
71+
content.write(buffer, bytes_read);
72+
}
73+
return content.str();
74+
}
75+
76+
std::string readStderrContent() {
77+
if (!stderr_file)
78+
return "";
79+
rewind(stderr_file.get());
80+
std::ostringstream content;
81+
char buffer[1024];
82+
size_t bytes_read;
83+
while ((bytes_read = fread(buffer, 1, sizeof(buffer), stderr_file.get())) >
84+
0) {
85+
content.write(buffer, bytes_read);
86+
}
87+
return content.str();
88+
}
89+
};
90+
91+
static void removePathComponent(unsigned N, llvm::SmallString<256> &Path) {
92+
for (unsigned i = 0; i < N; ++i)
93+
llvm::sys::path::remove_filename(Path);
94+
}
95+
96+
static std::string getExecutorPath() {
97+
llvm::SmallString<256> ExecutorPath(llvm::sys::fs::getMainExecutable(
98+
nullptr, reinterpret_cast<void *>(&getExecutorPath)));
99+
removePathComponent(5, ExecutorPath);
100+
llvm::sys::path::append(ExecutorPath, "bin", "llvm-jitlink-executor");
101+
return ExecutorPath.str().str();
102+
}
103+
104+
static std::string getOrcRuntimePath() {
105+
llvm::SmallString<256> RuntimePath(llvm::sys::fs::getMainExecutable(
106+
nullptr, reinterpret_cast<void *>(&getOrcRuntimePath)));
107+
removePathComponent(5, RuntimePath);
108+
llvm::sys::path::append(RuntimePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang",
109+
CLANG_VERSION_MAJOR_STRING, "lib");
110+
111+
llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
112+
if (SystemTriple.isOSBinFormatMachO()) {
113+
llvm::sys::path::append(RuntimePath, "darwin", "liborc_rt_osx.a");
114+
} else if (SystemTriple.isOSBinFormatELF()) {
115+
llvm::sys::path::append(RuntimePath, "x86_64-unknown-linux-gnu",
116+
"liborc_rt.a");
117+
}
118+
return RuntimePath.str().str();
119+
}
120+
121+
static std::unique_ptr<Interpreter>
122+
createInterpreterWithRemoteExecution(std::shared_ptr<IOContext> io_ctx,
123+
const Args &ExtraArgs = {}) {
124+
Args ClangArgs = {"-Xclang", "-emit-llvm-only"};
125+
llvm::append_range(ClangArgs, ExtraArgs);
126+
auto CB = clang::IncrementalCompilerBuilder();
127+
CB.SetCompilerArgs(ClangArgs);
128+
auto CI = cantFail(CB.CreateCpp());
129+
130+
clang::Interpreter::JITConfig Config;
131+
llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
132+
133+
if (SystemTriple.isOSBinFormatELF() || SystemTriple.isOSBinFormatMachO()) {
134+
Config.IsOutOfProcess = true;
135+
Config.OOPExecutor = getExecutorPath();
136+
Config.UseSharedMemory = false;
137+
Config.SlabAllocateSize = 0;
138+
Config.OrcRuntimePath = getOrcRuntimePath();
139+
140+
int stdin_fd = fileno(io_ctx->stdin_file.get());
141+
int stdout_fd = fileno(io_ctx->stdout_file.get());
142+
int stderr_fd = fileno(io_ctx->stderr_file.get());
143+
144+
Config.CustomizeFork = [=] {
145+
auto redirect = [](int from, int to) {
146+
if (from != to) {
147+
dup2(from, to);
148+
close(from);
149+
}
150+
};
151+
152+
redirect(stdin_fd, STDIN_FILENO);
153+
redirect(stdout_fd, STDOUT_FILENO);
154+
redirect(stderr_fd, STDERR_FILENO);
155+
156+
setvbuf(stdout, nullptr, _IONBF, 0);
157+
setvbuf(stderr, nullptr, _IONBF, 0);
158+
159+
printf("CustomizeFork executed\n");
160+
fflush(stdout);
161+
};
162+
}
163+
164+
return cantFail(clang::Interpreter::create(std::move(CI), Config));
165+
}
166+
167+
static size_t DeclsSize(TranslationUnitDecl *PTUDecl) {
168+
return std::distance(PTUDecl->decls().begin(), PTUDecl->decls().end());
169+
}
170+
171+
TEST_F(InterpreterTestBase, SanityWithRemoteExecution) {
172+
if (!HostSupportsJIT())
173+
GTEST_SKIP();
174+
175+
std::string OrcRuntimePath = getOrcRuntimePath();
176+
std::string ExecutorPath = getExecutorPath();
177+
178+
if (!llvm::sys::fs::exists(OrcRuntimePath) ||
179+
!llvm::sys::fs::exists(ExecutorPath))
180+
GTEST_SKIP();
181+
182+
auto io_ctx = std::make_shared<IOContext>();
183+
ASSERT_TRUE(io_ctx->initializeTempFiles());
184+
185+
std::unique_ptr<Interpreter> Interp =
186+
createInterpreterWithRemoteExecution(io_ctx);
187+
ASSERT_TRUE(Interp);
188+
189+
using PTU = PartialTranslationUnit;
190+
PTU &R1(cantFail(Interp->Parse("void g(); void g() {}")));
191+
EXPECT_EQ(2U, DeclsSize(R1.TUPart));
192+
193+
PTU &R2(cantFail(Interp->Parse("int i = 42;")));
194+
EXPECT_EQ(1U, DeclsSize(R2.TUPart));
195+
196+
std::string captured_stdout = io_ctx->readStdoutContent();
197+
std::string captured_stderr = io_ctx->readStderrContent();
198+
199+
EXPECT_TRUE(captured_stdout.find("CustomizeFork executed") !=
200+
std::string::npos);
201+
}
202+
203+
} // end anonymous namespace

0 commit comments

Comments
 (0)