jl_dlfind: do not find symbols in library dependencies (#58815)

A `ccall` without a specific library finds one with `jl_dlfind`, which
calls `jl_dlsym` on each of the following libraries, in order:
 - `libjulia-internal`
 - `libjulia`
 - The current executable
 - (On Windows): `kernel32`, `crtdll`, `ntdll`, `ws2_32`

The semantics of using `dlsym` (this does not apply to
`GetProcAddress`) with a specific handle are a little weird: the
library and its dependencies are searched, even if the provided symbol
would resolve somewhere else if used in the library.

On macOS and Linux, this causes all of the calls to libc functions in
Base go through the handle for `libjulia-internal`, making it difficult
to hook functions. For example, `-fsanitize=thread` on clang intercepts
malloc and free by defining them in the executable; calls to malloc
from Julia code would go to the original libc version while calls to
free in `libjulia-internal` would go to the hooked version.


This change makes `jl_dlfind` return a handle only if the symbol is
found in that library and not one of its dependencies.

## Example

`a.c`:
```c
void b_func(void);

void c_func(void) {
  puts("c override from a");
}

int main() {
  puts("a");
  b_func();
}
```

`b.c`:
```c
define _GNU_SOURCE
#include <stdio.h>
#include <dlfcn.h>

void c_func(void);

void b_func(void) {
  puts("b");
  c_func();

  /* How jl_libjulia_internal_handle is set by jl_find_dynamic_library_by_addr */
  Dl_info info;
  dladdr(&b_func, &info);
  void *hdl = dlopen(info.dli_fname, RTLD_NOW | RTLD_NOLOAD | RTLD_LOCAL);
  void *(*dl_c_func)(void) = dlsym(hdl, "c_func");
  dl_c_func();
}
```

`c.c`:
```c
#include <stdio.h>

void c_func(void) {
  puts("c");
}
```

```
$ cc -g -fPIC -shared -o libc.so c.c
$ cc -g -fPIC -shared -o libb.so b.c libc.so
$ cc -Wl,-rpath,. -g -fPIC -o main a.c libb.so libc.so
$ ./main
a
b
c override from a
c
```

Author: xal-0

base/libdl.jl

@@ -60,8 +60,8 @@ function dlsym(hnd::Ptr, s::Union{Symbol,AbstractString}; throw_error::Bool = tr
     hnd == C_NULL && throw(ArgumentError("NULL library handle"))
     val = Ref(Ptr{Cvoid}(0))
     symbol_found = ccall(:jl_dlsym, Cint,
-        (Ptr{Cvoid}, Cstring, Ref{Ptr{Cvoid}}, Cint),
-        hnd, s, val, Int64(throw_error)
+        (Ptr{Cvoid}, Cstring, Ref{Ptr{Cvoid}}, Cint, Cint),
+        hnd, s, val, Int64(throw_error), Int64(1)
     )
     if symbol_found == 0
         return nothing

src/ccall.cpp

@@ -627,7 +627,7 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va
                 void *symaddr;
                 std::string iname("i");
                 iname += f_name;
-                if (jl_dlsym(jl_libjulia_internal_handle, iname.c_str(), &symaddr, 0)) {
+                if (jl_dlsym(jl_libjulia_internal_handle, iname.c_str(), &symaddr, 0, 0)) {
                     f_lib = JL_LIBJULIA_INTERNAL_DL_LIBNAME;
                     f_name = jl_symbol_name(jl_symbol(iname.c_str()));
                 }

src/dlload.c

@@ -238,7 +238,8 @@ JL_DLLEXPORT int jl_dlclose(void *handle) JL_NOTSAFEPOINT
 #endif
 }
 
-void *jl_find_dynamic_library_by_addr(void *symbol, int throw_err) {
+void *jl_find_dynamic_library_by_addr(void *symbol, int throw_err, int close) JL_NOTSAFEPOINT
+{
     void *handle;
 #ifdef _OS_WINDOWS_
     if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
@@ -255,14 +256,21 @@ void *jl_find_dynamic_library_by_addr(void *symbol, int throw_err) {
             jl_error("could not load base module");
         return NULL;
     }
+    dlerror();
     handle = dlopen(info.dli_fname, RTLD_NOW | RTLD_NOLOAD | RTLD_LOCAL);
-#if !defined(__APPLE__)
+#if defined(_OS_FREEBSD_)
+    // FreeBSD will not give you a handle for the executable if you dlopen() it
+    // with RTLD_NOLOAD, so check jl_exe_handle.
+    if (handle == NULL && dlerror() == NULL) {
+        handle = jl_exe_handle;
+    }
+#elif !defined(__APPLE__)
     if (handle == RTLD_DEFAULT && (RTLD_DEFAULT != NULL || dlerror() == NULL)) {
         // We loaded the executable but got RTLD_DEFAULT back, ask for a real handle instead
         handle = dlopen("", RTLD_NOW | RTLD_NOLOAD | RTLD_LOCAL);
     }
 #endif
-    if (handle != NULL)
+    if (handle != NULL && close)
         dlclose(handle); // Undo ref count increment from `dlopen`
 #endif
     return handle;
@@ -285,7 +293,7 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags,
 
     // modname == NULL is a sentinel value requesting the handle of libjulia-internal
     if (modname == NULL)
-        return jl_find_dynamic_library_by_addr(&jl_load_dynamic_library, throw_err);
+        return jl_libjulia_internal_handle;
 
     abspath = jl_isabspath(modname);
     is_atpath = 0;
@@ -407,13 +415,26 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags,
     return handle;
 }
 
-JL_DLLEXPORT int jl_dlsym(void *handle, const char *symbol, void ** value, int throw_err) JL_NOTSAFEPOINT
+/*
+ * When search_deps is 1, act like dlsym and search both the library for the
+ * handle and all its dependencies.  Use this option only when compatibility
+ * with dlsym(3) is required, thought this behaviour is not possible on Windows.
+ *
+ * At time of writing, only Base.dlsym() uses search_deps = 1.
+ */
+JL_DLLEXPORT int jl_dlsym(void *handle, const char *symbol, void ** value, int throw_err, int search_deps) JL_NOTSAFEPOINT
 {
     int symbol_found = 0;
 
     /* First, get the symbol value */
-#ifdef _OS_WINDOWS_
+#if defined(_OS_WINDOWS_)
     *value = GetProcAddress((HMODULE) handle, symbol);
+#elif defined(_OS_DARWIN_)
+    /* When !search_deps and the handle isn't special, force RTLD_FIRST. */
+    if (!search_deps && handle != RTLD_NEXT && handle != RTLD_DEFAULT &&
+        handle != RTLD_SELF && handle != RTLD_MAIN_ONLY)
+        handle = (void *)((uintptr_t)handle | 1);
+    *value = dlsym(handle, symbol);
 #else
     *value = dlsym(handle, symbol);
 #endif
@@ -437,14 +458,29 @@ JL_DLLEXPORT int jl_dlsym(void *handle, const char *symbol, void ** value, int t
     }
 #endif
 
-    if (!symbol_found && throw_err) {
+#if !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_)
+    /*
+     * Unlike GetProcAddress, dlsym will search the dependencies of the given
+     * library, so we must check where the symbol came from.
+     */
+    if (symbol_found && !search_deps && handle != jl_RTLD_DEFAULT_handle) {
+        void *symbol_handle = jl_find_dynamic_library_by_addr(*value, 0, 1);
+        symbol_found = handle == symbol_handle;
+    }
+#endif
+
+    if (!symbol_found) {
+        if (throw_err) {
 #ifdef _OS_WINDOWS_
-        char err[256];
-        win32_formatmessage(GetLastError(), err, sizeof(err));
+            char err[256];
+            win32_formatmessage(GetLastError(), err, sizeof(err));
 #endif
-        jl_errorf("could not load symbol \"%s\":\n%s", symbol, err);
+            jl_errorf("could not load symbol \"%s\":\n%s", symbol, err);
+        }
+        return 0;
     }
-    return symbol_found;
+
+    return 1;
 }
 
 // Look for symbols in internal libraries
@@ -455,23 +491,23 @@ JL_DLLEXPORT const char *jl_dlfind(const char *f_name)
     // https://cgit.freebsd.org/src/commit/?id=21a52f99440c9bec7679f3b0c5c9d888901c3694
     // (See https://github.com/JuliaLang/julia/issues/50846)
     if (strcmp(f_name, "dl_iterate_phdr") == 0)
-        return JL_EXE_LIBNAME;
+        return NULL;
 #endif
     void * dummy;
-    if (jl_dlsym(jl_libjulia_internal_handle, f_name, &dummy, 0))
+    if (jl_dlsym(jl_libjulia_internal_handle, f_name, &dummy, 0, 0))
         return JL_LIBJULIA_INTERNAL_DL_LIBNAME;
-    if (jl_dlsym(jl_libjulia_handle, f_name, &dummy, 0))
+    if (jl_dlsym(jl_libjulia_handle, f_name, &dummy, 0, 0))
         return JL_LIBJULIA_DL_LIBNAME;
-    if (jl_dlsym(jl_exe_handle, f_name, &dummy, 0))
+    if (jl_dlsym(jl_exe_handle, f_name, &dummy, 0, 0))
         return JL_EXE_LIBNAME;
 #ifdef _OS_WINDOWS_
-    if (jl_dlsym(jl_kernel32_handle, f_name, &dummy, 0))
+    if (jl_dlsym(jl_kernel32_handle, f_name, &dummy, 0, 0))
         return "kernel32";
-    if (jl_dlsym(jl_crtdll_handle, f_name, &dummy, 0)) // Prefer crtdll over ntdll
+    if (jl_dlsym(jl_crtdll_handle, f_name, &dummy, 0, 0)) // Prefer crtdll over ntdll
         return jl_crtdll_basename;
-    if (jl_dlsym(jl_ntdll_handle, f_name, &dummy, 0))
+    if (jl_dlsym(jl_ntdll_handle, f_name, &dummy, 0, 0))
         return "ntdll";
-    if (jl_dlsym(jl_winsock_handle, f_name, &dummy, 0))
+    if (jl_dlsym(jl_winsock_handle, f_name, &dummy, 0, 0))
         return "ws2_32";
 #endif
     // additional common libraries (libc?) could be added here, but in general,

src/init.c

@@ -21,6 +21,10 @@
 #include <pthread_np.h>
 #endif
 
+#if !defined(_OS_WINDOWS_)
+#include <dlfcn.h>
+#endif
+
 #include "julia.h"
 #include "julia_internal.h"
 #include "builtin_proto.h"
@@ -723,26 +727,29 @@ JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage)
     void *stack_lo, *stack_hi;
     jl_init_stack_limits(1, &stack_lo, &stack_hi);
 
-    jl_libjulia_internal_handle = jl_find_dynamic_library_by_addr(&jl_load_dynamic_library, /* throw_err */ 1);
-    jl_libjulia_handle = jl_find_dynamic_library_by_addr(&jl_any_type, /* throw_err */ 1);
+    // Note that if we ever want to be able to unload Julia entirely, we will
+    // have to dlclose() these handles.
+    jl_libjulia_internal_handle = jl_find_dynamic_library_by_addr(&jl_load_dynamic_library, /* throw_err */ 1, 0);
+    jl_libjulia_handle = jl_find_dynamic_library_by_addr(&jl_any_type, /* throw_err */ 1, 0);
 #ifdef _OS_WINDOWS_
+    /* If this parameter is NULL, GetModuleHandle returns a handle to the file
+       used to create the calling process (.exe file). */
     jl_exe_handle = GetModuleHandleA(NULL);
-    jl_RTLD_DEFAULT_handle = jl_libjulia_internal_handle;
+    jl_RTLD_DEFAULT_handle = NULL;
     jl_ntdll_handle = jl_dlopen("ntdll.dll", JL_RTLD_NOLOAD); // bypass julia's pathchecking for system dlls
     jl_kernel32_handle = jl_dlopen("kernel32.dll", JL_RTLD_NOLOAD);
     jl_crtdll_handle = jl_dlopen(jl_crtdll_name, JL_RTLD_NOLOAD);
     jl_winsock_handle = jl_dlopen("ws2_32.dll", JL_RTLD_NOLOAD);
     HMODULE jl_dbghelp = (HMODULE) jl_dlopen("dbghelp.dll", JL_RTLD_NOLOAD);
     needsSymRefreshModuleList = 0;
     if (jl_dbghelp)
-        jl_dlsym(jl_dbghelp, "SymRefreshModuleList", (void **)&hSymRefreshModuleList, 1);
+        jl_dlsym(jl_dbghelp, "SymRefreshModuleList", (void **)&hSymRefreshModuleList, 1, 0);
 #else
-    jl_exe_handle = jl_dlopen(NULL, JL_RTLD_NOW);
-#ifdef RTLD_DEFAULT
+    /* macOS dlopen(3): If path is NULL and the option RTLD_FIRST is used, the
+       handle returned will only search the main executable. */
+    jl_exe_handle = jl_dlopen(NULL, JL_RTLD_NOW | JL_RTLD_NOLOAD | JL_RTLD_LOCAL | JL_RTLD_FIRST);
+    // RTLD_DEFAULT is mandatory on POSIX
     jl_RTLD_DEFAULT_handle = RTLD_DEFAULT;
-#else
-    jl_RTLD_DEFAULT_handle = jl_exe_handle;
-#endif
 #endif
 
     jl_init_rand();

src/jitlayers.cpp

@@ -1722,7 +1722,7 @@ struct JuliaOJIT::DLSymOptimizer {
 
     void *lookup_symbol(void *libhandle, const char *fname) JL_NOTSAFEPOINT {
         void *addr;
-        jl_dlsym(libhandle, fname, &addr, 0);
+        jl_dlsym(libhandle, fname, &addr, 0, 0);
         return addr;
     }
 

src/julia.h

@@ -2311,13 +2311,17 @@ enum JL_RTLD_CONSTANT {
      /* MacOS X 10.5+: */
      JL_RTLD_FIRST=128U
 };
+#ifdef _OS_DARWIN_
+#define JL_RTLD_DEFAULT (JL_RTLD_LAZY | JL_RTLD_DEEPBIND | JL_RTLD_FIRST)
+#else
 #define JL_RTLD_DEFAULT (JL_RTLD_LAZY | JL_RTLD_DEEPBIND)
+#endif
 
 typedef void *jl_libhandle; // compatible with dlopen (void*) / LoadLibrary (HMODULE)
 JL_DLLEXPORT jl_libhandle jl_load_dynamic_library(const char *fname, unsigned flags, int throw_err);
 JL_DLLEXPORT jl_libhandle jl_dlopen(const char *filename, unsigned flags) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int jl_dlclose(jl_libhandle handle) JL_NOTSAFEPOINT;
-JL_DLLEXPORT int jl_dlsym(jl_libhandle handle, const char *symbol, void ** value, int throw_err) JL_NOTSAFEPOINT;
+JL_DLLEXPORT int jl_dlsym(jl_libhandle handle, const char *symbol, void ** value, int throw_err, int search_deps) JL_NOTSAFEPOINT;
 
 // evaluation
 JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_module_t *m, jl_value_t *v);

src/julia_internal.h

@@ -1630,7 +1630,7 @@ void win32_formatmessage(DWORD code, char *reason, int len) JL_NOTSAFEPOINT;
 #endif
 
 JL_DLLEXPORT void *jl_get_library_(const char *f_lib, int throw_err);
-void *jl_find_dynamic_library_by_addr(void *symbol, int throw_err);
+void *jl_find_dynamic_library_by_addr(void *symbol, int throw_err, int close) JL_NOTSAFEPOINT;
 #define jl_get_library(f_lib) jl_get_library_(f_lib, 1)
 JL_DLLEXPORT void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) *hnd);
 JL_DLLEXPORT void *jl_lazy_load_and_lookup(jl_value_t *lib_val, const char *f_name);

src/processor_arm.cpp

@@ -779,7 +779,7 @@ static inline unsigned long jl_getauxval(unsigned long type)
     // First, try resolving getauxval in libc
     auto libc = jl_dlopen(nullptr, JL_RTLD_LOCAL);
     static unsigned long (*getauxval_p)(unsigned long) = NULL;
-    if (getauxval_p == NULL && jl_dlsym(libc, "getauxval", (void **)&getauxval_p, 0)) {
+    if (getauxval_p == NULL && jl_dlsym(libc, "getauxval", (void **)&getauxval_p, 0, 0)) {
         return getauxval_p(type);
     }
 

src/runtime_ccall.cpp

@@ -58,7 +58,7 @@ void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) *
     if (!handle)
         jl_atomic_store_release(hnd, (handle = jl_get_library(f_lib)));
     void * ptr;
-    jl_dlsym(handle, f_name, &ptr, 1);
+    jl_dlsym(handle, f_name, &ptr, 1, 0);
     return ptr;
 }
 
@@ -79,7 +79,7 @@ void *jl_lazy_load_and_lookup(jl_value_t *lib_val, const char *f_name)
     } else
         jl_type_error("ccall", (jl_value_t*)jl_symbol_type, lib_val);
     void *ptr;
-    jl_dlsym(lib_ptr, f_name, &ptr, 1);
+    jl_dlsym(lib_ptr, f_name, &ptr, 1, 0);
     return ptr;
 }
 

src/runtime_intrinsics.c

@@ -669,7 +669,7 @@ JL_DLLEXPORT jl_value_t *jl_cglobal(jl_value_t *v, jl_value_t *ty)
     }
     else {
         void *handle = jl_get_library((char*)jl_dlfind(f_name));
-        jl_dlsym(handle, f_name, &ptr, 1);
+        jl_dlsym(handle, f_name, &ptr, 1, 0);
     }
     JL_GC_POP();