diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 07bcff755c5..1ad08724443 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -394,6 +394,7 @@ jobs: - run: | # Build Go test apps. cd mirrord/layer/tests ../../../scripts/build_go_apps.sh 24 + - run: ./mirrord/layer/tests/apps/dlopen_cgo/build_test_app.sh - uses: actions/setup-go@v5 with: go-version: "1.25" @@ -454,7 +455,6 @@ jobs: - run: | cd mirrord/layer/tests/apps/double_listen cargo build - - run: ./mirrord/layer/tests/apps/dlopen_cgo/build_test_app.sh - run: ./scripts/build_c_apps.sh - run: cargo build --target x86_64-unknown-linux-gnu -p mirrord-layer - run: cargo build --target x86_64-unknown-linux-gnu -p mirrord diff --git a/changelog.d/+dlopen-many.added.md b/changelog.d/+dlopen-many.added.md new file mode 100644 index 00000000000..6d9b5f75483 --- /dev/null +++ b/changelog.d/+dlopen-many.added.md @@ -0,0 +1,2 @@ +Added an experimental feature for supporting dlopen multiple c-shared go libraries +for amd64 Linux and go v1.24.* only. diff --git a/mirrord/layer/src/go/c_shared/mod.rs b/mirrord/layer/src/go/c_shared/mod.rs new file mode 100644 index 00000000000..74a0a5fe653 --- /dev/null +++ b/mirrord/layer/src/go/c_shared/mod.rs @@ -0,0 +1,251 @@ +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +pub(crate) mod go_1_24 { + use std::arch::naked_asm; + + use crate::{hooks::HookManager, macros::hook_symbol}; + + pub(crate) fn hook(hook_manager: &mut HookManager, module_name: &str) { + hook_symbol!( + hook_manager, + module_name, + "internal/runtime/syscall.Syscall6", + internal_runtime_syscall_syscall6_detour + ); + } + + /// Detour of `internal/runtime/syscall.Syscall6`. + /// + /// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) + /// + /// + /// Function arguments are mapped as the following: + /// rax = num + /// rbx = a1 + /// rcx = a2 + /// rdi = a3 + /// rsi = a4 + /// r8 = a5 + /// r9 = a6 + /// r14 = g + #[unsafe(naked)] + unsafe extern "C" fn internal_runtime_syscall_syscall6_detour() { + naked_asm!( + // If the syscall is SYS_EXIT or SYS_EXIT_GROUP, + // skip our logic and just execute it. + "cmp rax,60", // SYS_EXIT + "je 2f", + "cmp rax,231", // SYS_EXIT_GROUP + "je 2f", + "push rbp", + "mov rbp,rsp", + // Reservce space and store args. + "sub rsp,0x40", + "mov [rsp+0x38],r9", + "mov [rsp+0x30],r8", + "mov [rsp+0x28],rsi", + "mov [rsp+0x20],rdi", + "mov [rsp+0x18],rcx", + "mov [rsp+0x10],rbx", + "mov [rsp],rax", + // function args starting at rdi. + "mov rdi,rsp", + "call {c_abi_wrapper_on_systemstack}", + // Check failure + "cmp rax,-0xfff", + "jbe 1f", + // Syscall failed. + // Save errno in rcx. + "neg rax", + "mov rcx,rax", + // Fill -1 in rax. + "mov rax,-0x1", + // Clear result register. + "mov rbx,0x0", + // Drop space reserved for locals. + "add rsp,0x40", + // Restore rbp and return. + "pop rbp", + "ret", + // Syscall did not fail. Result is in rax. Clear other result registers. + "1:", + "mov rbx,0x0", + "mov rcx,0x0", + // Drop space for storing args locally. + "add rsp,0x40", + // Restore rbp and return. + "pop rbp", + "ret", + // Move the first syscall argument to the correct register (rdx), + // and execute the syscall. + "2:", + "mov rdx,rdi", + "syscall", + + c_abi_wrapper_on_systemstack = sym c_abi_wrapper_on_systemstack, + ); + } + + /// Calls [`c_abi_wrapper`] on systemstack. + /// + /// Implemented based on [`runtime.systemstack`](https://github.com/golang/go/blob/go1.24.11/src/runtime/asm_amd64.s#L483) + #[unsafe(naked)] + unsafe extern "C" fn c_abi_wrapper_on_systemstack() { + naked_asm!( + // This is required, as we call `gosave_systemstack_switch` later. + // Not having any local variables in this function is required as well. + "push rbp", + "mov rbp,rsp", + // We assume that r14 stores the address of g. + // Load address of current m to rbx. + // https://github.com/golang/go/blob/go1.24.11/src/runtime/runtime2.go#L410 + "mov rbx,[r14+0x30]", + // Check if g is m->gsignal. If so, do not switch stack. + // https://github.com/golang/go/blob/go1.24.11/src/runtime/runtime2.go#L536 + "cmp r14,[rbx+0x50]", + "je 1f", + // Load address of m->g0 to rdx. + // Check if g is g0. If so, do not switch stack. + "mov rdx,[rbx]", + "cmp r14,rdx", + "je 1f", + // We expect g is m->curg now. If not, abort with `ud2`. + // https://github.com/golang/go/blob/go1.24.11/src/runtime/runtime2.go#L541 + "cmp r14,[rbx+0xc0]", + "jne 2f", + // Switch to system stack. + "call {gosave_systemstack_switch}", + // rdx holds the address of m->g0. Store it also in TLS and r14. + "mov fs:0xfffffffffffffff8,rdx", + "mov r14,rdx", + // Fill rsp with g0's g->sched->sp. After this, we are on system stack. + "mov rsp,[rdx+0x38]", + // We assume rdi still has the original syscall args stored on user g stack. + "call {c_abi_wrapper}", + // Switch back to g stack. + // https://github.com/golang/go/blob/go1.24.11/src/runtime/asm_amd64.s#L516-L526 + // We assume r14 still has g0. Store g0's m in rbx. + "mov rbx,[r14+0x30]", + // Store m->curg in rsi. + "mov rsi,[rbx+0xc0]", + // Store user g in TLS + "mov fs:0xfffffffffffffff8,rsi", + // Store user g in r14 + "mov r14,rsi", + // Fill rsp with g->sched->sp + "mov rsp,[rsi+0x38]", + // Fill rbp with g->sched->bp + // https://github.com/golang/go/blob/go1.24.11/src/runtime/runtime2.go#L317 + "mov rbp,[rsi+0x68]", + "mov QWORD PTR [rsi+0x38],0x0", + "mov QWORD PTR [rsi+0x68],0x0", + // Restore rbp and return. + "pop rbp", + "ret", + // Already on system stack. Tail call `c_abi_wrapper`. + // https://github.com/golang/go/blob/go1.24.11/src/runtime/asm_amd64.s#L528-L537 + "1:", + "pop rbp", + "jmp {c_abi_wrapper}", + // Abort the program. + "2:", + "ud2", + gosave_systemstack_switch = sym gosave_systemstack_switch, + c_abi_wrapper = sym c_abi_wrapper, + ); + } + + /// Exact copy of the `go_1_25::c_abi_wrapper` function. + /// + /// Move function args from stack to registers so we can call + /// [`c_abi_syscall6_handler`](crate::go::c_abi_syscall6_handler). + /// + /// We expect rdi stores the address of the start of function args on the stack. + /// + /// C ABI: fn(rdi, rsi, rdx, rcx, r8, r9, stack) + #[unsafe(naked)] + unsafe extern "C" fn c_abi_wrapper() { + naked_asm!( + "push rbp", + "mov rbp,rsp", + "sub rsp,0x8", + "and rsp,-0x10", + "mov rax,[rdi]", + "mov rsi,[rdi+0x10]", + "mov rdx,[rdi+0x18]", + "mov rcx,[rdi+0x20]", + "mov r8,[rdi+0x28]", + "mov r9,[rdi+0x30]", + "mov r10,[rdi+0x38]", + "mov [rsp],r10", + "mov rdi,rax", + "call c_abi_syscall6_handler", + "mov rsp, rbp", + "pop rbp", + "ret" + ); + } + + /// Implemented based on [`gosave_systemstack_switch`](https://github.com/golang/go/blob/go1.24.11/src/runtime/asm_amd64.s#L823) + /// + /// Smashes r9. + #[unsafe(naked)] + unsafe extern "C" fn gosave_systemstack_switch() { + naked_asm!( + // TODO: In Go's implementation, it loads the address of `runtime.systemstack_switch` + 8 bytes. + // [`runtime.systemstack_switch`](https://github.com/golang/go/blob/go1.24.11/src/runtime/asm_amd64.s#L475) + // is a dummy marker function. If it is directly called, it will hit a `ud2` trap. + // Go only cares that g->sched->pc is set to an address between the prologue and epilogue. + // + // I don't know if it matters for the address to be between the actual `systemstack_switch` function. + // Or a simple dummy function will work? + "lea r9, [rip + {dummy} + 0x8]", + // Store r9 in g->sched->pc. + "mov QWORD PTR [r14+0x40],r9", + "lea r9, [rsp+0x8]", + // Store r9 in g->sched->sp. + "mov QWORD PTR [r14+0x38],r9", + // Store 0 in g->sched->ret. + "mov QWORD PTR [r14+0x58],0x0", + // Store rbp in g->sched->bp. + "mov QWORD PTR [r14+0x68],rbp", + // Store g->sched->ctxt in r9. + "mov r9, QWORD PTR [r14+0x50]", + // If g->sched->ctxt == 0 abort runtime, otherwise return. + "test r9, r9", + "jz 1f", + "call {runtime_abort}", + "1:", + "ret", + + runtime_abort = sym runtime_abort, + dummy = sym dummy, + ); + } + + /// Implemented based on [`runtime.abort`](https://github.com/golang/go/blob/fed3b0a298464457c58d1150bdb3942f22bd6220/src/runtime/asm_amd64.s#L1237) + /// + /// This function crashes the runtime. `int3` is recognized by debuggers. + /// The rest of the function is an infinite trap loop. + #[unsafe(naked)] + unsafe extern "C" fn runtime_abort() { + naked_asm!("int3", "1:", "jmp 1b",); + } + + /// A dummy function that wraps some `nop` between prologue and epilogue. + #[unsafe(naked)] + unsafe extern "C" fn dummy() { + naked_asm!( + "push rbp", + "mov rbp,rsp", + "nop", + "nop", + "nop", + "nop", + "nop", // this is the address we save in g->sched->pc + "nop", + "nop", + "pop rbp", + "ret", + ); + } +} diff --git a/mirrord/layer/src/go/linux_x64.rs b/mirrord/layer/src/go/linux_x64.rs index 0533ae10959..0ef2c370e4f 100644 --- a/mirrord/layer/src/go/linux_x64.rs +++ b/mirrord/layer/src/go/linux_x64.rs @@ -640,8 +640,14 @@ pub(crate) fn enable_hooks_in_loaded_module(hook_manager: &mut HookManager, modu return; }; + tracing::trace!(version, module_name, "Detected Go"); if version >= 1.25 { post_go_1_25::hook_in_module(hook_manager, version, module_name.as_str()); + } else if version >= 1.24 { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + crate::go::c_shared::go_1_24::hook(hook_manager, module_name.as_str()); + #[cfg(all(target_os = "linux", target_arch = "aarch64"))] + post_go1_23(hook_manager, Some(module_name.as_str())); } else if version >= 1.23 { post_go1_23(hook_manager, Some(module_name.as_str())); } else if version >= 1.19 { diff --git a/mirrord/layer/src/go/mod.rs b/mirrord/layer/src/go/mod.rs index 2da44adcbf4..1f3c402db97 100644 --- a/mirrord/layer/src/go/mod.rs +++ b/mirrord/layer/src/go/mod.rs @@ -10,6 +10,11 @@ use tracing::trace; use crate::{close_detour, file::hooks::*, hooks::HookManager, socket::hooks::*}; +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + target_os = "linux" +))] +pub(crate) mod c_shared; #[cfg_attr( all(target_os = "linux", target_arch = "x86_64"), path = "linux_x64.rs" diff --git a/mirrord/layer/tests/apps/dlopen_cgo/.gitignore b/mirrord/layer/tests/apps/dlopen_cgo/.gitignore index a59918867be..9463d0e52e4 100644 --- a/mirrord/layer/tests/apps/dlopen_cgo/.gitignore +++ b/mirrord/layer/tests/apps/dlopen_cgo/.gitignore @@ -1,3 +1,4 @@ *.so -*.h -out.cpp_dlopen_cgo +*.a +libgo*.h +out.* diff --git a/mirrord/layer/tests/apps/dlopen_cgo/build_test_app.sh b/mirrord/layer/tests/apps/dlopen_cgo/build_test_app.sh index d92a0bf7f12..668c74a1939 100755 --- a/mirrord/layer/tests/apps/dlopen_cgo/build_test_app.sh +++ b/mirrord/layer/tests/apps/dlopen_cgo/build_test_app.sh @@ -4,31 +4,89 @@ set -euo pipefail # Resolve directory where this script is located SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_FILE="$SCRIPT_DIR/main.go" -CPP_FILE="$SCRIPT_DIR/main.cpp" -SO_FILE="$SCRIPT_DIR/libgo_server.so" -OUT_BIN="$SCRIPT_DIR/out.cpp_dlopen_cgo" +# Go library source that export C server APIs +SERVER_GO_FILE="$SCRIPT_DIR/server/main.go" +# Dynamic library (c-shared) +SERVER_C_SHARED_LIB="$SCRIPT_DIR/server/libgo_server_c_shared.so" +# Static library (c-archive) +SERVER_C_ARCHIVE_LIB="$SCRIPT_DIR/server/libgo_server_c_archive.a" + +# CPP library that wraps exported server APIs from static CGO library +SERVER_CPP_FILE="$SCRIPT_DIR/server/lib.cpp" +SERVER_CPP_WRAPPER_LIB="$SCRIPT_DIR/server/libcpp_server.so" + + +# Go library source that export C fileops APIs +FILEOPS_GO_FILE="$SCRIPT_DIR/fileops/main.go" +# Dynamic library (c-shared) +FILEOPS_C_SHARED_LIB="$SCRIPT_DIR/fileops/libgo_fileops_c_shared.so" +# Static library (c-archive) +FILEOPS_C_ARCHIVE_LIB="$SCRIPT_DIR/fileops/libgo_fileops_c_archive.a" + +# CPP library that wraps exported fileops APIs from static CGO library +FILEOPS_CPP_FILE="$SCRIPT_DIR/fileops/lib.cpp" +FILEOPS_CPP_WRAPPER_LIB="$SCRIPT_DIR/fileops/libcpp_fileops.so" + +# CPP app that dlopen server and fileops dynamic CGO library +C_SHARED_CPP_FILE="$SCRIPT_DIR/main_c_shared.cpp" +C_SHARED_OUT_BIN="$SCRIPT_DIR/out.dlopen_cgo_c_shared" + +# CPP app that dlopen the CPP wrapper libraries of static CGO libraries +C_ARCHIVE_CPP_FILE="$SCRIPT_DIR/main_c_archive.cpp" +C_ARCHIVE_OUT_BIN="$SCRIPT_DIR/out.dlopen_cpp_wrapper_cgo_c_archive" echo "Script directory: $SCRIPT_DIR" # Ensure files exist -if [[ ! -f "$GO_FILE" ]]; then - echo "ERROR: main.go not found in $SCRIPT_DIR" +if [[ ! -f "$SERVER_GO_FILE" ]]; then + echo "ERROR: server/main.go not found in $SCRIPT_DIR" + exit 1 +fi + +if [[ ! -f "$FILEOPS_GO_FILE" ]]; then + echo "ERROR: fileops/main.go not found in $SCRIPT_DIR" exit 1 fi -if [[ ! -f "$CPP_FILE" ]]; then +if [[ ! -f "$C_SHARED_CPP_FILE" ]]; then echo "ERROR: main.cpp not found in $SCRIPT_DIR" exit 1 fi -echo "Building Go shared library..." -go build -buildmode=c-shared -o "$SO_FILE" "$GO_FILE" +go version + +echo "Building Go c-shared server library..." +go build -buildmode=c-shared -o "$SERVER_C_SHARED_LIB" "$SERVER_GO_FILE" + +echo "Building Go c-shared file ops library..." +go build -buildmode=c-shared -o "$FILEOPS_C_SHARED_LIB" "$FILEOPS_GO_FILE" + +echo "Building C++ c-shared loader app..." +g++ "$C_SHARED_CPP_FILE" -o "$C_SHARED_OUT_BIN" -ldl + +echo "Done! c-shared artifacts:" +echo " - $SERVER_C_SHARED_LIB" +echo " - $FILEOPS_C_SHARED_LIB" +echo " - $C_SHARED_OUT_BIN" + +echo "Building Go c-archive server library..." +go build -buildmode=c-archive -o "$SERVER_C_ARCHIVE_LIB" "$SERVER_GO_FILE" + +echo "Building C++ server dynamic wrapper library..." +g++ -fPIC -shared $SERVER_CPP_FILE $SERVER_C_ARCHIVE_LIB -o $SERVER_CPP_WRAPPER_LIB -lpthread -ldl + +echo "Building Go c-archive file ops library..." +go build -buildmode=c-archive -o "$FILEOPS_C_ARCHIVE_LIB" "$FILEOPS_GO_FILE" + +echo "Building C++ file ops dynamic wrapper library..." +g++ -fPIC -shared $FILEOPS_CPP_FILE $FILEOPS_C_ARCHIVE_LIB -o $FILEOPS_CPP_WRAPPER_LIB -lpthread -ldl -echo "Building C++ loader app..." -g++ "$CPP_FILE" -o "$OUT_BIN" -ldl +echo "Building C++ c-archive loaded app..." +g++ $C_ARCHIVE_CPP_FILE -o $C_ARCHIVE_OUT_BIN -ldl -echo "Done!" -echo "Artifacts:" -echo " - $SO_FILE" -echo " - $OUT_BIN" +echo "Done! c-archive artifacts:" +echo " - $SERVER_C_ARCHIVE_LIB" +echo " - $SERVER_CPP_WRAPPER_LIB" +echo " - $FILEOPS_C_ARCHIVE_LIB" +echo " - $FILEOPS_CPP_WRAPPER_LIB" +echo " - $C_ARCHIVE_OUT_BIN" diff --git a/mirrord/layer/tests/apps/dlopen_cgo/fileops/fileops.h b/mirrord/layer/tests/apps/dlopen_cgo/fileops/fileops.h new file mode 100644 index 00000000000..8a5c81b7aa6 --- /dev/null +++ b/mirrord/layer/tests/apps/dlopen_cgo/fileops/fileops.h @@ -0,0 +1,12 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +char* cppReadFileToStringWrapper(const char* path); + +#ifdef __cplusplus +} +#endif + diff --git a/mirrord/layer/tests/apps/dlopen_cgo/fileops/lib.cpp b/mirrord/layer/tests/apps/dlopen_cgo/fileops/lib.cpp new file mode 100644 index 00000000000..26d3f208d85 --- /dev/null +++ b/mirrord/layer/tests/apps/dlopen_cgo/fileops/lib.cpp @@ -0,0 +1,8 @@ +#include "fileops.h" + +#include "libgo_fileops_c_archive.h" + +extern "C" char* cppReadFileToStringWrapper(const char* path) { + return ReadFileToString((char*)path); +} + diff --git a/mirrord/layer/tests/apps/dlopen_cgo/fileops/main.go b/mirrord/layer/tests/apps/dlopen_cgo/fileops/main.go new file mode 100644 index 00000000000..35bed0c0712 --- /dev/null +++ b/mirrord/layer/tests/apps/dlopen_cgo/fileops/main.go @@ -0,0 +1,24 @@ +package main + +import "C" + +import ( + "fmt" + "os" +) + +//export ReadFileToString +func ReadFileToString(cpath *C.char) *C.char { + path := C.GoString(cpath) + + data, err := os.ReadFile(path) + if err != nil { + msg := fmt.Sprintf("ReadFileToString error: %v", err) + return C.CString(msg) + } + + return C.CString(string(data)) +} + +func main() {} + diff --git a/mirrord/layer/tests/apps/dlopen_cgo/main.cpp b/mirrord/layer/tests/apps/dlopen_cgo/main.cpp deleted file mode 100644 index 5ee2e214e50..00000000000 --- a/mirrord/layer/tests/apps/dlopen_cgo/main.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "libgo_server.h" - -using namespace std; - -static atomic running{true}; - -void signal_handler(int signum) { - cout << "\nReceived signal " << signum << ", shutting down...\n"; - running = false; -} - -string get_exe_dir() { - char exe_path[PATH_MAX]; - ssize_t len = readlink("/proc/self/exe", exe_path, sizeof(exe_path)-1); - exe_path[len] = '\0'; - return string(dirname(exe_path)); -} - -/** - * This test app calls `dlopen()` to load a c-shared cgo library. - * To compile the app: - * 1. build the go c-shared library: `go build -buildmode=c-shared -o libgo_server.so server.go`. - * 2. build the cpp app: `g++ main.cpp -o out.cpp_dlopen_cgo -ldl`. -**/ -int main() { - // Install signal handlers - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); - - string exe_dir = get_exe_dir(); - string so_path = exe_dir + "/libgo_server.so"; - - // Load library - void* handle = dlopen(so_path.c_str(), RTLD_LAZY); - if (!handle) { - cerr << "dlopen error: " << dlerror() << "\n"; - return 1; - } - - typedef int (*StartServerFn)(char*, int); - typedef void (*RunFileOpsFn)(); - typedef void (*StopServerFn)(); - - StartServerFn StartServer = (StartServerFn)dlsym(handle, "StartServer"); - StopServerFn StopServer = (StopServerFn)dlsym(handle, "StopServer"); - - if (!StartServer || !StopServer) { - cerr << "dlsym error: " << dlerror() << "\n"; - return 1; - } - - // Start Go server - int rc = StartServer((char*)"127.0.0.1", 23333); - if (rc != 0) { - cerr << "StartServer returned " << rc << "\n"; - return 1; - } - - cout << "Server started. Press Ctrl-C to stop.\n"; - - // Main thread waits until a signal is received - while (running) { - this_thread::sleep_for(std::chrono::milliseconds(200)); - } - - // Graceful shutdown - StopServer(); - - // Let the server clean up - this_thread::sleep_for(std::chrono::milliseconds(200)); - - // DO NOT call dlclose() for Go shared libraries - return 0; -} diff --git a/mirrord/layer/tests/apps/dlopen_cgo/main_c_archive.cpp b/mirrord/layer/tests/apps/dlopen_cgo/main_c_archive.cpp new file mode 100644 index 00000000000..ad36a297806 --- /dev/null +++ b/mirrord/layer/tests/apps/dlopen_cgo/main_c_archive.cpp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +static atomic running{true}; + +void signal_handler(int signum) { + cout << "\nReceived signal " << signum << ", shutting down...\n"; + running = false; +} + +string get_exe_dir() { + char exe_path[PATH_MAX]; + ssize_t len = readlink("/proc/self/exe", exe_path, sizeof(exe_path) - 1); + exe_path[len] = '\0'; + return string(dirname(exe_path)); +} + +int main() { + // Install signal handlers + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + string exe_dir = get_exe_dir(); + + string server_so_path = exe_dir + "/server/libcpp_server.so"; + string fileops_so_path = exe_dir + "/fileops/libcpp_fileops.so"; + + // Load C++ server library + void* server_handle = dlopen( + server_so_path.c_str(), + RTLD_LAZY | RTLD_NODELETE + ); + if (!server_handle) { + cerr << "dlopen(server) error: " << dlerror() << "\n"; + return 1; + } + + // Load C++ fileops library + void* fileops_handle = dlopen( + fileops_so_path.c_str(), + RTLD_LAZY | RTLD_NODELETE + ); + if (!fileops_handle) { + cerr << "dlopen(fileops) error: " << dlerror() << "\n"; + return 1; + } + + // Resolve server symbols + typedef int (*StartServerFn)(const char*, int); + typedef void (*StopServerFn)(); + + StartServerFn StartServer = + (StartServerFn)dlsym(server_handle, "cppStartServerWrapper"); + if (!StartServer) { + cerr << "dlsym(cppStartServerWrapper) error: " << dlerror() << "\n"; + return 1; + } + + StopServerFn StopServer = + (StopServerFn)dlsym(server_handle, "cppStopServerWrapper"); + if (!StopServer) { + cerr << "dlsym(cppStopServerWrapper) error: " << dlerror() << "\n"; + return 1; + } + + // Resolve fileops symbol + typedef char* (*ReadFileToStringFn)(const char*); + + ReadFileToStringFn ReadFileToString = + (ReadFileToStringFn)dlsym(fileops_handle, "cppReadFileToStringWrapper"); + if (!ReadFileToString) { + cerr << "dlsym(cppReadFileToStringWrapper) error: " << dlerror() << "\n"; + return 1; + } + + // Start Go-backed server + int rc = StartServer("127.0.0.1", 23333); + if (rc != 0) { + cerr << "StartServer returned " << rc << "\n"; + return 1; + } + + cout << "Server started. Press Ctrl-C to stop.\n"; + + // Main loop + while (running) { + this_thread::sleep_for(chrono::seconds(1)); + + char* file_content = ReadFileToString("/app/test.txt"); + if (!file_content) { + cerr << "ReadFileToString returned null\n"; + continue; + } + + cout << "\n--- test.txt (via Go → C++ → dlopen) ---\n"; + cout << file_content; + cout << "\n--- EOF ---\n"; + + free(file_content); + } + + // Graceful shutdown + StopServer(); + this_thread::sleep_for(chrono::milliseconds(200)); + + // IMPORTANT: + // Do NOT dlclose() Go-backed C++ libraries + return 0; +} diff --git a/mirrord/layer/tests/apps/dlopen_cgo/main_c_shared.cpp b/mirrord/layer/tests/apps/dlopen_cgo/main_c_shared.cpp new file mode 100644 index 00000000000..f509b84e69d --- /dev/null +++ b/mirrord/layer/tests/apps/dlopen_cgo/main_c_shared.cpp @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "server/libgo_server_c_shared.h" +#include "fileops/libgo_fileops_c_shared.h" + +using namespace std; + +static atomic running{true}; + +void signal_handler(int signum) { + cout << "\nReceived signal " << signum << ", shutting down...\n"; + running = false; +} + +string get_exe_dir() { + char exe_path[PATH_MAX]; + ssize_t len = readlink("/proc/self/exe", exe_path, sizeof(exe_path)-1); + exe_path[len] = '\0'; + return string(dirname(exe_path)); +} + +/** + * This test app calls `dlopen()` to load two different c-shared cgo libraries. + * To compile the app, run the script `./build_test_app.sh`, which does the following: + * 1. build the go c-shared server library: `go build -buildmode=c-shared -o server/libgo_server.so server/main.go`. + * 2. build the go c-shared file ops library: `go build -buildmode=c-shared -o fileops/libgo_fileops.so fileops/main.go`. + * 3. build the cpp app: `g++ main.cpp -o out.cpp_dlopen_cgo -ldl`. +**/ +int main() { + // Install signal handlers + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + string exe_dir = get_exe_dir(); + string server_so_path = exe_dir + "/server/libgo_server_c_shared.so"; + string fileops_so_path = exe_dir + "/fileops/libgo_fileops_c_shared.so"; + + // These dlopen() flags are provided by the user. + void* server_so_handle = dlopen(server_so_path.c_str(), RTLD_LAZY | RTLD_NODELETE); + if (!server_so_handle) { + cerr << "dlopen error: " << dlerror() << "\n"; + return 1; + } + void* fileops_so_handle = dlopen(fileops_so_path.c_str(), RTLD_LAZY | RTLD_NODELETE); + if (!fileops_so_handle) { + cerr << "dlopen error: " << dlerror() << "\n"; + return 1; + } + + // Exported functions from the server library + typedef int (*StartServerFn)(char*, int); + typedef void (*StopServerFn)(); + StartServerFn StartServer = (StartServerFn)dlsym(server_so_handle, "StartServer"); + if (!StartServer) { + cerr << "dlsym error: " << dlerror() << "\n"; + return 1; + } + StopServerFn StopServer = (StopServerFn)dlsym(server_so_handle, "StopServer"); + if (!StopServer) { + cerr << "dlsym error: " << dlerror() << "\n"; + return 1; + } + + // Exported function from the file ops library + typedef char* (*ReadFileToStringFn)(char*); + ReadFileToStringFn ReadFileToString = (ReadFileToStringFn)dlsym(fileops_so_handle, "ReadFileToString"); + if (!ReadFileToString) { + cerr << "dlsym error: " << dlerror() << "\n"; + return 1; + } + + // Start Go server + int rc = StartServer((char*)"127.0.0.1", 23333); + if (rc != 0) { + cerr << "StartServer returned " << rc << "\n"; + return 1; + } + + cout << "Server started. Press Ctrl-C to stop.\n"; + + // Main thread waits until a signal is received while reading + // the same file over and over. + while (running) { + this_thread::sleep_for(std::chrono::milliseconds(1000)); + char* file_content = ReadFileToString((char*)"/app/test.txt"); + if (!file_content) { + cerr << "ReadFileToString returned null\n"; + } else { + cout << "\n--- test.txt (via Go → dlopen) ---\n"; + cout << file_content; + cout << "\n--- EOF ---\n"; + } + free(file_content); + } + + // Graceful shutdown + StopServer(); + + // Let the server clean up + this_thread::sleep_for(std::chrono::milliseconds(200)); + + // DO NOT call dlclose() for Go shared libraries + return 0; +} diff --git a/mirrord/layer/tests/apps/dlopen_cgo/server/lib.cpp b/mirrord/layer/tests/apps/dlopen_cgo/server/lib.cpp new file mode 100644 index 00000000000..9c235db6364 --- /dev/null +++ b/mirrord/layer/tests/apps/dlopen_cgo/server/lib.cpp @@ -0,0 +1,12 @@ +#include "server.h" + +#include "libgo_server_c_archive.h" + +extern "C" int cppStartServerWrapper(const char* host, int port) { + return StartServer((char*)host, port); +} + +extern "C" void cppStopServerWrapper() { + StopServer(); +} + diff --git a/mirrord/layer/tests/apps/dlopen_cgo/main.go b/mirrord/layer/tests/apps/dlopen_cgo/server/main.go similarity index 100% rename from mirrord/layer/tests/apps/dlopen_cgo/main.go rename to mirrord/layer/tests/apps/dlopen_cgo/server/main.go diff --git a/mirrord/layer/tests/apps/dlopen_cgo/server/server.h b/mirrord/layer/tests/apps/dlopen_cgo/server/server.h new file mode 100644 index 00000000000..c64d8a2f5ab --- /dev/null +++ b/mirrord/layer/tests/apps/dlopen_cgo/server/server.h @@ -0,0 +1,13 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +int cppStartServerWrapper(const char* host, int port); +void cppStopServerWrapper(); + +#ifdef __cplusplus +} +#endif + diff --git a/mirrord/layer/tests/common/mod.rs b/mirrord/layer/tests/common/mod.rs index a82c32aa047..06b844bd5cb 100644 --- a/mirrord/layer/tests/common/mod.rs +++ b/mirrord/layer/tests/common/mod.rs @@ -186,7 +186,9 @@ pub enum Application { NodeMakeConnections, NodeIssue3456, /// C++ app that dlopen c-shared go library. - DlopenCgo, + DlopenCgoCShared, + /// C++ app that dlopen C++ library that wraps static CGO library. + DlopenCppCgoCArchive, /// C app that calls BSD connectx(2). Connectx, /// Rust app that closes a clone socket. @@ -373,7 +375,12 @@ impl Application { Application::GoIssue2988(version) => { format!("tests/apps/issue2988/{version}.go_test_app") } - Application::DlopenCgo => String::from("tests/apps/dlopen_cgo/out.cpp_dlopen_cgo"), + Application::DlopenCgoCShared => { + String::from("tests/apps/dlopen_cgo/out.dlopen_cgo_c_shared") + } + Application::DlopenCppCgoCArchive => { + String::from("tests/apps/dlopen_cgo/out.dlopen_cpp_wrapper_cgo_c_archive") + } Application::Connectx => String::from("tests/apps/connectx/out.c_test_app"), Application::DupListen => { format!( @@ -508,7 +515,8 @@ impl Application { | Application::RustIssue2438 | Application::RustIssue3248 | Application::GoIssue2988(..) - | Application::DlopenCgo + | Application::DlopenCgoCShared + | Application::DlopenCppCgoCArchive | Application::Connectx | Application::DoubleListen | Application::DupListen => vec![], @@ -607,10 +615,11 @@ impl Application { | Application::GoIssue2988(..) | Application::NodeMakeConnections | Application::DoubleListen + | Application::DlopenCgoCShared + | Application::DlopenCppCgoCArchive | Application::Connectx => unimplemented!("shouldn't get here"), Application::PythonSelfConnect => 1337, Application::RustIssue2058 => 1234, - Application::DlopenCgo => 23333, } } diff --git a/mirrord/layer/tests/dlopen_cgo.rs b/mirrord/layer/tests/dlopen_cgo.rs index 283917ffc69..eaca0f60913 100644 --- a/mirrord/layer/tests/dlopen_cgo.rs +++ b/mirrord/layer/tests/dlopen_cgo.rs @@ -17,7 +17,8 @@ use mirrord_protocol::tcp::{LayerTcpSteal, StealType}; #[tokio::test] #[timeout(Duration::from_secs(60))] async fn test_dlopen_cgo( - #[values(Application::DlopenCgo)] application: Application, + #[values(Application::DlopenCgoCShared, Application::DlopenCppCgoCArchive)] + application: Application, dylib_path: &Path, ) { use mirrord_protocol::ClientMessage;