Add support for ILP64 Accelerate

The new Accelerate released in macOS v13.3 provides two new interfaces;
an upgraded LAPACK for the LP64 interface, and an all-new ILP64
interface (that uses the same upgraded LAPACK).  These symbols are
available from Accelerate with the suffix `$NEWLAPACK` and
`$NEWLAPACK$ILP64`, respectively.

Unfortunately, this is not a "true" suffix, as Apple has decided to drop
the trailing underscore from the typical F77 names, meaning that a
symbol such as `dgemm_` gets mangled to `dgemm$NEWLAPACK`, whereas a
CBLAS symbol such as `cblas_zdotc_sub` gets mangled to
`cblas_zdotc_sub$NEWLAPACK`.  This means that we need to selectively
erase the trailing underscore from some symbols when applying this
Accelerate suffix.

To do this, we add a new feature, enabled by default only on Apple
builds, called `SYMBOL_TRIMMING`, which allows a `suffix_hint` to
contain the ASCII "substitution character" `0x1a` as the first character
of the suffix hint to mean "remove a trailing underscore when applying
this suffix".

To make dealing with suffix hints easier for command-line users, these
suffix hints are available for use in `LBT_BACKING_LIBS` by listing
libraries separated by suffix hints with an exclamation point,  e.g.
`libname!suffix`.

Author: staticfloat

src/Make.inc

@@ -98,6 +98,11 @@ ifneq (,$(filter $(ARCH), x86_64 aarch64))
   COMPLEX_RETSTYLE_AUTODETECTION := 1
 endif
 
+# If we're on an apple platform, we need to support symbol name trimming
+ifeq ($(OS), Darwin)
+  LBT_CFLAGS += -DSYMBOL_TRIMMING
+  SYMBOL_TRIMMING := 1
+endif
 
 ifeq ($(VERBOSE),0)
 ENDCOLOR := "\033[0m"

src/autodetection.c

@@ -1,6 +1,30 @@
 #include "libblastrampoline_internal.h"
 #include <complex.h>
 
+/*
+ * Some vendors (such as Accelerate) decided to trim off the trailing underscore
+ * from the F77 symbol names when adding their ILP64 symbol names.  So the
+ * symbol name `dgemm_` turns into `dgemm$NEWLAPACK$ILP64`.  But of course,
+ * symbol names like `cblas_sdot` turn into `cblas_sdot$NEWLAPACK$ILP64`.
+ *
+ * So we need a way to selectively trim off the trailing underscore.  We do so
+ * by shoving an ASCII "substitute character" onto the start of `$NEWLAPACK`,
+ */
+void build_symbol_name(char * out, const char *symbol_name, const char *suffix) {
+    size_t symbol_len = strlen(symbol_name);
+#if defined(SYMBOL_TRIMMING)
+    if (suffix[0] == '\x1a') {
+        if (symbol_name[symbol_len-1] == '_') {
+            symbol_len -= 1;
+        }
+        suffix += 1;
+    }
+#endif
+
+    strncpy(out, symbol_name, MAX_SYMBOL_LEN);
+    strncpy(out + symbol_len, suffix, MAX_SYMBOL_LEN - symbol_len);
+}
+
 /*
  * Search for a symbol that ends in one of the given suffixes.  Returns NULL if not found.
  */
@@ -12,7 +36,7 @@ const char * symbol_suffix_search(void * handle, const char * symbol_name, const
             continue;
         }
 
-        sprintf(symbol_name_suffixed, "%s%s", symbol_name, suffixes[suffix_idx]);
+        build_symbol_name(symbol_name_suffixed, symbol_name, suffixes[suffix_idx]);
         if (lookup_symbol(handle, symbol_name_suffixed) != NULL) {
             return suffixes[suffix_idx];
         }
@@ -38,13 +62,25 @@ const char * autodetect_symbol_suffix(void * handle, const char * suffix_hint) {
         // Possibly-NULL suffix that we should search over
         suffix_hint,
 
-        // First, search for LP64-mangling suffixes, so that when we are loading MKL from a
+        // First, search for LP64-mangling suffixes, so that when we are loading libs from an
         // CLI environment, (where suffix hints are not easy) we want to give the most stable
         // configuration by default.
+#if defined(_OS_DARWIN_) && defined(SYMBOL_TRIMMING)
+        // Apple Accelerate has an updated LAPACK interface, default to that.
+        // Note that we are making use of our symbol trimming support here to eliminate
+        // the F77 trailing underscore by starting the string with `\x1a`.
+        "\x1a$NEWLAPACK",
+#endif
         "", "_", "__",
 
+        // Next, ILP64-mangling suffixes
+#if defined(_OS_DARWIN_) && defined(SYMBOL_TRIMMING)
+        // Once again, search for Accelerate's non-pure-suffixed names
+        "\x1a$NEWLAPACK$ILP64",
+#endif
+
         // Next, search for ILP64-mangling suffixes
-        "64", "64_", "_64__", "__64___",
+        "64", "_64", "64_", "_64_", "_64__", "__64___",
     };
 
     // If the suffix hint is NULL, just skip it when calling `lookup_symbol()`.
@@ -143,14 +179,14 @@ int32_t autodetect_interface(void * handle, const char * suffix) {
     char symbol_name[MAX_SYMBOL_LEN];
 
     // Attempt BLAS `isamax()` test
-    sprintf(symbol_name, "isamax_%s", suffix);
+    build_symbol_name(symbol_name, "isamax_", suffix);
     void * isamax = lookup_symbol(handle, symbol_name);
     if (isamax != NULL) {
         return autodetect_blas_interface(isamax);
     }
 
     // Attempt LAPACK `dpotrf()` test
-    sprintf(symbol_name, "dpotrf_%s", suffix);
+    build_symbol_name(symbol_name, "dpotrf_", suffix);
     void * dpotrf = lookup_symbol(handle, symbol_name);
     if (dpotrf != NULL) {
         return autodetect_lapack_interface(dpotrf);
@@ -164,7 +200,7 @@ int32_t autodetect_interface(void * handle, const char * suffix) {
 int32_t autodetect_complex_return_style(void * handle, const char * suffix) {
     char symbol_name[MAX_SYMBOL_LEN];
 
-    sprintf(symbol_name, "zdotc_%s", suffix);
+    build_symbol_name(symbol_name, "zdotc_", suffix);
     void * zdotc_addr = lookup_symbol(handle, symbol_name);
     if (zdotc_addr == NULL) {
         return LBT_COMPLEX_RETSTYLE_UNKNOWN;
@@ -209,8 +245,8 @@ int32_t autodetect_complex_return_style(void * handle, const char * suffix) {
 int32_t autodetect_f2c(void * handle, const char * suffix) {
     char symbol_name[MAX_SYMBOL_LEN];
 
-    // Attempt BLAS `sdot_()` test
-    sprintf(symbol_name, "sdot_%s", suffix);
+    // Attempt BLAS `sdot()` test
+    build_symbol_name(symbol_name, "sdot_", suffix);
     void * sdot_addr = lookup_symbol(handle, symbol_name);
     if (sdot_addr == NULL) {
         return LBT_F2C_UNKNOWN;
@@ -246,10 +282,10 @@ int32_t autodetect_f2c(void * handle, const char * suffix) {
 int32_t autodetect_cblas_divergence(void * handle, const char * suffix) {
     char symbol_name[MAX_SYMBOL_LEN];
 
-    sprintf(symbol_name, "zdotc_%s", suffix);
+    build_symbol_name(symbol_name, "zdotc_", suffix);
     if (lookup_symbol(handle, symbol_name) != NULL ) {
         // If we have both `zdotc_64` and `cblas_zdotc_sub64`, it's all good:
-        sprintf(symbol_name, "cblas_zdotc_sub%s", suffix);
+        build_symbol_name(symbol_name, "cblas_zdotc_sub", suffix);
         if (lookup_symbol(handle, symbol_name) != NULL ) {
             return LBT_CBLAS_CONFORMANT;
         }

src/config.c

@@ -21,6 +21,18 @@ LBT_DLLEXPORT const lbt_config_t * lbt_get_config() {
     lbt_config.build_flags |= LBT_BUILDFLAGS_F2C_CAPABLE;
 #endif
 
+#if defined(CBLAS_DIVERGENCE_AUTODETECTION)
+    lbt_config.build_flags |= LBT_BUILDFLAGS_CBLAS_DIVERGENCE;
+#endif
+
+#if defined(COMPLEX_RETSTYLE_AUTODETECTION)
+    lbt_config.build_flags |= LBT_BUILDFLAGS_COMPLEX_RETSTYLE;
+#endif
+
+#if defined(SYMBOL_TRIMMING)
+    lbt_config.build_flags |= LBT_BUILDFLAGS_SYMBOL_TRIMMING;
+#endif
+
     lbt_config.exported_symbols = (const char **)&exported_func_names[0];
     lbt_config.num_exported_symbols = NUM_EXPORTED_FUNCS;
 

src/libblastrampoline.c

@@ -80,7 +80,7 @@ int32_t set_forward_by_index(int32_t symbol_idx, const void * addr, int32_t inte
             // Report to the user that we're cblas-wrapping this one
             if (verbose) {
                 char exported_name[MAX_SYMBOL_LEN];
-                sprintf(exported_name, "%s%s", exported_func_names[symbol_idx], interface == LBT_INTERFACE_ILP64 ? "64_" : "");
+                build_symbol_name(exported_name, exported_func_names[symbol_idx], interface == LBT_INTERFACE_ILP64 ? "64_" : "");
                 printf(" - [%04d] complex(%s)\n", symbol_idx, exported_name);
             }
 
@@ -106,7 +106,7 @@ int32_t set_forward_by_index(int32_t symbol_idx, const void * addr, int32_t inte
 
             if (verbose) {
                 char exported_name[MAX_SYMBOL_LEN];
-                sprintf(exported_name, "%s%s", exported_func_names[symbol_idx], interface == LBT_INTERFACE_ILP64 ? "64_" : "");
+                build_symbol_name(exported_name, exported_func_names[symbol_idx], interface == LBT_INTERFACE_ILP64 ? "64_" : "");
                 printf(" - [%04d] f2c(%s)\n", symbol_idx, exported_name);
             }
 
@@ -182,7 +182,7 @@ LBT_DLLEXPORT int32_t lbt_set_forward(const char * symbol_name, const void * add
 // Load `libname`, clearing previous mappings if `clear` is set.
 LBT_DLLEXPORT int32_t lbt_forward(const char * libname, int32_t clear, int32_t verbose, const char * suffix_hint) {
     if (verbose) {
-        printf("Generating forwards to %s\n", libname);
+        printf("Generating forwards to %s (clear: %d, verbose: %d, suffix_hint: '%s')\n", libname, clear, verbose, suffix_hint);
     }
 
     // Load the library, throwing an error if we can't actually load it
@@ -353,7 +353,7 @@ LBT_DLLEXPORT int32_t lbt_forward(const char * libname, int32_t clear, int32_t v
         }
 
         // Look up this symbol in the given library, if it is a valid symbol, set it!
-        sprintf(symbol_name, "%s%s", exported_func_names[symbol_idx], lib_suffix);
+        build_symbol_name(symbol_name, exported_func_names[symbol_idx], lib_suffix);
         void *addr = lookup_symbol(handle, symbol_name);
         if (addr != NULL) {
             set_forward_by_index(symbol_idx,  addr, interface, complex_retstyle, f2c, verbose);
@@ -374,7 +374,7 @@ LBT_DLLEXPORT int32_t lbt_forward(const char * libname, int32_t clear, int32_t v
             // Report to the user that we're cblas-wrapping this one
             if (verbose) {
                 char exported_name[MAX_SYMBOL_LEN];
-                sprintf(exported_name, "%s%s", exported_func_names[symbol_idx], interface == LBT_INTERFACE_ILP64 ? "64_" : "");
+                build_symbol_name(exported_name, exported_func_names[symbol_idx], interface == LBT_INTERFACE_ILP64 ? "64_" : "");
                 printf(" - [%04d] cblas(%s)\n", symbol_idx, exported_name);
             }
 
@@ -443,7 +443,10 @@ __attribute__((constructor)) void init(void) {
         default_func = lookup_self_symbol("lbt_default_func_print_error_and_exit");
     }
 
-    // LBT_DEFAULT_LIBS is a semicolon-separated list of paths that should be loaded as BLAS libraries
+    // LBT_DEFAULT_LIBS is a semicolon-separated list of paths that should be loaded as BLAS libraries.
+    // Each library can have a `!suffix` tacked onto the end of it, providing a library-specific
+    // suffix_hint.  Example:
+    //    export LBT_DEFAULT_LIBS="libopenblas64.so!64_;/tmp/libfoo.so;/tmp/libbar.so!fastmath32"
     const char * default_libs = getenv("LBT_DEFAULT_LIBS");
 #if defined(LBT_FALLBACK_LIBS)
     if (default_libs == NULL) {
@@ -454,22 +457,41 @@ __attribute__((constructor)) void init(void) {
         const char * curr_lib_start = default_libs;
         int clear = 1;
         char curr_lib[PATH_MAX];
+        char suffix_buffer[MAX_SYMBOL_LEN];
         while (curr_lib_start[0] != '\0') {
             // Find the end of this current library name
             const char * end = curr_lib_start;
-            while (*end != ';' && *end != '\0')
+            const char * suffix_sep = NULL;
+            while (*end != ';' && *end != '\0') {
+                if (*end == '!' && suffix_sep == NULL) {
+                    suffix_sep = end;
+                }
                 end++;
+            }
+            const char * curr_lib_end = end;
+            if (suffix_sep != NULL) {
+                curr_lib_end = suffix_sep;
+            }
 
-            // Copy it into a temporary location
-            int len = end - curr_lib_start;
+            // Figure out if there's an embedded suffix_hint:
+            const char * suffix_hint = NULL;
+            if (suffix_sep != NULL) {
+                int len = end - (suffix_sep + 1);
+                memcpy(suffix_buffer, suffix_sep+1, len);
+                suffix_buffer[len] = '\0';
+                suffix_hint = &suffix_buffer[0];
+            }
+
+            int len = curr_lib_end - curr_lib_start;
             memcpy(curr_lib, curr_lib_start, len);
             curr_lib[len] = '\0';
             curr_lib_start = end;
+
             while (curr_lib_start[0] == ';')
                 curr_lib_start++;
 
             // Load functions from this library, clearing only the first time.
-            lbt_forward(curr_lib, clear, verbose, NULL);
+            lbt_forward(curr_lib, clear, verbose, suffix_hint);
             clear = 0;
         }
     }

src/libblastrampoline.h

@@ -111,8 +111,11 @@ typedef struct {
 } lbt_config_t;
 
 // Possible values for `build_flags` in `lbt_config_t`
-#define LBT_BUILDFLAGS_DEEPBINDLESS     0x01
-#define LBT_BUILDFLAGS_F2C_CAPABLE      0x02
+#define LBT_BUILDFLAGS_DEEPBINDLESS         0x01
+#define LBT_BUILDFLAGS_F2C_CAPABLE          0x02
+#define LBT_BUILDFLAGS_CBLAS_DIVERGENCE     0x04
+#define LBT_BUILDFLAGS_COMPLEX_RETSTYLE     0x08
+#define LBT_BUILDFLAGS_SYMBOL_TRIMMING      0x10
 
 /*
  * Load the given `libname`, lookup all registered symbols within our configured list of exported

src/libblastrampoline_internal.h

@@ -76,6 +76,7 @@ const char * lookup_self_path();
 void close_library(void * handle);
 
 // Functions in `autodetection.c`
+void build_symbol_name(char * out, const char *symbol_name, const char *suffix);
 const char * autodetect_symbol_suffix(void * handle, const char * suffix_hint);
 int32_t autodetect_blas_interface(void * isamax_addr);
 int32_t autodetect_lapack_interface(void * dpotrf_addr);

src/threading.c

@@ -56,7 +56,7 @@ LBT_DLLEXPORT int32_t lbt_get_num_threads() {
         lbt_library_info_t * lib = config->loaded_libs[lib_idx];
         for (int symbol_idx=0; getter_names[symbol_idx] != NULL; ++symbol_idx) {
             char symbol_name[MAX_SYMBOL_LEN];
-            sprintf(symbol_name, "%s%s", getter_names[symbol_idx], lib->suffix);
+            build_symbol_name(symbol_name, getter_names[symbol_idx], lib->suffix);
             int (*fptr)() = lookup_symbol(lib->handle, symbol_name);
             if (fptr != NULL) {
                 int new_threads = fptr();
@@ -78,7 +78,7 @@ LBT_DLLEXPORT void lbt_set_num_threads(int32_t nthreads) {
         lbt_library_info_t * lib = config->loaded_libs[lib_idx];
         for (int symbol_idx=0; setter_names[symbol_idx] != NULL; ++symbol_idx) {
             char symbol_name[MAX_SYMBOL_LEN];
-            sprintf(symbol_name, "%s%s", setter_names[symbol_idx], lib->suffix);
+            build_symbol_name(symbol_name, setter_names[symbol_idx], lib->suffix);
             void (*fptr)(int) = lookup_symbol(lib->handle, symbol_name);
             if (fptr != NULL) {
                 fptr(nthreads);

test/runtests.jl

@@ -170,6 +170,19 @@ if dlopen_e(veclib_blas_path) != C_NULL
     @testset "LBT -> vecLib/libLAPACK" begin
         run_all_tests(blastrampoline_link_name(), [lbt_dir], :LP64, string(veclib_blas_path, ";", veclib_lapack_path), tests=[dgemm, sgesv, sdot, zdotc])
     end
+
+    veclib_lapack_handle = dlopen(veclib_lapack_path)
+    if dlsym_e(veclib_lapack_handle, "dpotrf\$NEWLAPACK\$ILP64") != C_NULL
+        @testset "LBT -> vecLib/libBLAS (ILP64)" begin
+            veclib_blas_path_ilp64 = "$(veclib_blas_path)!\x1a\$NEWLAPACK\$ILP64"
+            run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, veclib_blas_path_ilp64; tests=[dgemm, sdot, zdotc])
+        end
+
+        @testset "LBT -> vecLib/libLAPACK (ILP64)" begin
+            veclib_lapack_path_ilp64 = "$(veclib_lapack_path)!\x1a\$NEWLAPACK\$ILP64"
+            run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, veclib_lapack_path_ilp64; tests=[dgemm, sgesv, sdot, zdotc])
+        end
+    end
 end
 
 

test/utils.jl

@@ -22,15 +22,31 @@ function append_libpath(paths::Vector{<:AbstractString})
     return join(vcat(paths..., get(ENV, LIBPATH_env, String[])), pathsep)
 end
 
-function capture_output(cmd::Cmd; verbose::Bool = false)
+function capture_output(cmd::Cmd; verbose::Bool = false, timeout = 10.0)
     out_pipe = Pipe()
     if verbose
         ld_env = filter(e -> startswith(e, "LBT_") || startswith(e, "LD_") || startswith(e, "DYLD_"), something(cmd.env, String[]))
         @info("Running $(basename(cmd.exec[1]))", ld_env)
     end
     p = run(pipeline(ignorestatus(cmd), stdout=out_pipe, stderr=out_pipe), wait=false)
     close(out_pipe.in)
-    output = @async read(out_pipe, String)
+    output = @async begin
+        lines = String[]
+        for line in readlines(out_pipe)
+            if verbose
+                println(line)
+            end
+            push!(lines, line)
+        end
+        return join(lines, "\n")
+    end
+    @async begin
+        sleep(timeout)
+        if process_running(p)
+            @warn("$(basename(cmd.exec[1])) timeout exceeded!")
+            kill(p)
+        end
+    end
     wait(p)
     return p, fetch(output)
 end