ggml : add repack testing support

This commit add support for testing the ggml-cpu repack feature which
enables the repackaging of quantized data into more optimal layout for
matrix multiplication for specific hardware architectures.

The motivation is to enable the testing of a cpu backend that uses
repacked data against a reference cpu backend that does not use repacked
data.

Building:
```console
$ cmake -B build \
    -DGGML_CPU_REF_BACKEND=ON
    -DGGML_BACKEND_DL=ON \
    -DGGML_CPU_ALL_VARIANTS=ON
```

List availble cpu architectures/variants:
```console
$ ./build/bin/test-backend-ops cpu-variants --list
CPU variants:
  CPU-alderlake   - 12th Gen Intel(R) Core(TM) i7-1260P
```
Run tests:
```console
./build-ref/bin/test-backend-ops cpu-variants \
    --variant CPU-alderlake \
    -o "MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1)"

Testing CPU variant 'CPU-alderlake' against cpu-ref backend...

repack: repack tensor a with q4_0_8x8
  MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK
repack: repack tensor a with q4_0_8x8
  MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK
  14491/14491 tests passed
```
All matrix multiplication tests can be run by use specifying
`-o "MUL_MAT"` but it may be harder to spot the ones that use repacking.

Author: danbev

ggml/CMakeLists.txt

@@ -252,9 +252,9 @@ set   (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING
 set   (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen")
 
 # extra artifacts
-option(GGML_BUILD_TESTS    "ggml: build tests"    ${GGML_STANDALONE})
-option(GGML_CPU_REF        "ggml: build reference CPU backend for testing" OFF)
-option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
+option(GGML_BUILD_TESTS     "ggml: build tests"    ${GGML_STANDALONE})
+option(GGML_CPU_REF_BACKEND "ggml: build reference CPU backend for testing" OFF)
+option(GGML_BUILD_EXAMPLES  "ggml: build examples" ${GGML_STANDALONE})
 
 #
 # dependencies
@@ -284,7 +284,9 @@ add_subdirectory(src)
 
 if (GGML_BUILD_TESTS)
     enable_testing()
-    add_subdirectory(tests)
+    if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests")
+        add_subdirectory(tests)
+    endif ()
 endif ()
 
 if (GGML_BUILD_EXAMPLES)

ggml/include/ggml-backend.h

@@ -243,6 +243,9 @@ extern "C" {
     // Load all known backends from dynamic libraries
     GGML_API void               ggml_backend_load_all(void);
     GGML_API void               ggml_backend_load_all_from_path(const char * dir_path);
+    // Load all variants for a backend and register them
+    GGML_API void               ggml_backend_load_all_variants(const char * name);
+    GGML_API void               ggml_backend_load_variant(const char * name, const char * variant);
 
     //
     // Backend scheduler

ggml/src/CMakeLists.txt

@@ -397,7 +397,21 @@ if (GGML_CPU_REF_BACKEND)
     set(GGML_CPU_HBM OFF)
     set(GGML_OPENMP OFF)
     set(GGML_CPU_KLEIDIAI OFF)
-    ggml_add_cpu_backend_variant_impl(ref)
+    set(GGML_CPU_REPACK OFF)
+    set(GGML_ACCELERATE OFF)
+
+    ggml_add_cpu_backend_variant(ref)
+
+    if (CMAKE_SYSTEM_PROCESSOR MATCHES "arm|aarch64|ARM|AARCH64")
+        target_compile_options(ggml-cpu-ref PRIVATE
+              -U__ARM_NEON
+              -U__ARM_FEATURE_FMA
+              -U__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+              -U__ARM_FEATURE_DOTPROD
+              -U__ARM_FEATURE_MATMUL_INT8
+              -U__ARM_FEATURE_SVE
+          )
+    endif()
     target_compile_definitions(ggml PRIVATE GGML_USE_CPU_REF)
 endif()
 

ggml/src/ggml-backend-reg.cpp

@@ -609,3 +609,72 @@ void ggml_backend_load_all_from_path(const char * dir_path) {
     ggml_backend_load_best("cpu-ref", silent, dir_path);
 #endif
 }
+
+void ggml_backend_load_all_variants(const char * name) {
+    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
+    const fs::path name_path = fs::u8path(name);
+    const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path("-").native();
+    const fs::path file_extension = backend_filename_extension();
+
+    std::vector<fs::path> search_paths;
+#ifdef GGML_BACKEND_DIR
+    search_paths.push_back(fs::u8path(GGML_BACKEND_DIR));
+#endif
+    // default search paths: executable directory, current directory
+    search_paths.push_back(get_executable_path());
+    search_paths.push_back(fs::current_path());
+
+    for (const auto & search_path : search_paths) {
+        if (!fs::exists(search_path)) {
+            GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
+            continue;
+        }
+        fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
+        for (const auto & entry : dir_it) {
+            if (entry.is_regular_file()) {
+                auto filename = entry.path().filename();
+                auto ext = entry.path().extension();
+                if (filename.native().find(file_prefix.native()) == 0 && ext == file_extension) {
+                    fs::path path = search_path / filename;
+                    ggml_backend_reg_t backend = get_reg().load_backend(path, false);
+                    if (backend == nullptr) {
+                        GGML_LOG_ERROR("%s: failed to load backend variant %s\n", __func__, path_str(entry.path()).c_str());
+                    }
+
+                }
+            }
+        }
+    }
+}
+
+void ggml_backend_load_variant(const char * name, const char * variant) {
+    const fs::path name_path = fs::u8path(name);
+    const fs::path variant_path = fs::u8path(variant);
+    const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path("-").native();
+    const fs::path target_filename = file_prefix.native() + variant_path.native() + backend_filename_extension().native();
+
+    std::vector<fs::path> search_paths;
+#ifdef GGML_BACKEND_DIR
+    search_paths.push_back(fs::u8path(GGML_BACKEND_DIR));
+#endif
+    // default search paths: executable directory, current directory
+    search_paths.push_back(get_executable_path());
+    search_paths.push_back(fs::current_path());
+
+    for (const auto & search_path : search_paths) {
+        if (!fs::exists(search_path)) {
+            GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
+            continue;
+        }
+
+        fs::path full_path = search_path / target_filename;
+        if (fs::exists(full_path) && fs::is_regular_file(full_path)) {
+            ggml_backend_reg_t backend = get_reg().load_backend(full_path, false);
+            if (backend == nullptr) {
+                GGML_LOG_ERROR("%s: failed to load backend variant %s\n", __func__, path_str(full_path).c_str());
+            } else {
+                return;
+            }
+        }
+    }
+}

ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -108,7 +108,7 @@ struct ggml_backend_cpu_context {
 };
 
 static const char * ggml_backend_cpu_get_name(ggml_backend_t backend) {
-    return "CPU";
+    return GGML_CPU_VARIANT_NAME;
 
     GGML_UNUSED(backend);
 }

ggml/src/ggml-cpu/repack.cpp

@@ -1869,8 +1869,43 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
     return nullptr;
 }
 
+static bool supports_tensor(const struct ggml_tensor * op) {
+    if (op->op == GGML_OP_MUL_MAT &&
+        op->src[0]->buffer &&
+        (ggml_n_dims(op->src[0]) == 2) && ggml_repack_get_optimal_repack_type(op->src[0])) {
+
+        if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {
+            return false;
+        }
+
+        if (op->src[1]->type == GGML_TYPE_F32) {
+            return true;
+        }
+
+    } else if (op->op == GGML_OP_MUL_MAT_ID && op->src[0]->buffer &&
+            (ggml_n_dims(op->src[0]) == 3) && ggml_repack_get_optimal_repack_type(op->src[0])) {
+
+        if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {
+            return false;
+        }
+
+        if (op->src[1]->type == GGML_TYPE_F32) {
+            return true;
+        }
+    }
+    return false;
+}
+
 static enum ggml_status ggml_backend_cpu_repack_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
-    tensor->extra = (void *) const_cast<ggml::cpu::tensor_traits *>(ggml_repack_get_optimal_repack_type(tensor));
+    if (tensor->op == GGML_OP_NONE) {
+        tensor->extra = (void *) const_cast<ggml::cpu::tensor_traits *>(ggml_repack_get_optimal_repack_type(tensor));
+        tensor->buffer = buffer;
+    }
+
+    if (supports_tensor(tensor)) {
+        tensor->src[0]->extra = (void *) const_cast<ggml::cpu::tensor_traits *>(ggml_repack_get_optimal_repack_type(tensor->src[0]));
+        tensor->src[0]->buffer = buffer;
+    }
 
     GGML_UNUSED(buffer);
     return GGML_STATUS_SUCCESS;

tests/CMakeLists.txt

@@ -202,15 +202,7 @@ if (NOT LLAMA_SANITIZE_ADDRESS)
 endif()
 llama_build_and_test(test-gguf.cpp)
 llama_build_and_test(test-backend-ops.cpp)
-
-if (GGML_CPU_REF_BACKEND)
-    if (WIN32)
-        set(GGML_CPU_REF_BACKEND_PATH "${CMAKE_BINARY_DIR}/bin/ggml-cpu-ref.dll")
-    else()
-        set(GGML_CPU_REF_BACKEND_PATH "${CMAKE_BINARY_DIR}/bin/libggml-cpu-ref.so")
-    endif()
-    target_compile_definitions(test-backend-ops PRIVATE GGML_CPU_REF_BACKEND_PATH="${GGML_CPU_REF_BACKEND_PATH}")
-endif()
+target_include_directories(test-backend-ops PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src)
 
 llama_build_and_test(test-model-load-cancel.cpp  LABEL "model")
 llama_build_and_test(test-autorelease.cpp        LABEL "model")