build OPTIMIZE use xxhash instead of jenkins one-at-a-time hash, if available (#2365)

* use xxhash instead of jenkins hash

In several benchmarks, xxhash scores far better than the jenkins one at
a time hash function.

See https://xxhash.com/ for a comparison.

* make xxhash dependency optional

Author: irfanHaslanded

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
             options: "-DENABLE_TESTS=ON",
             packager: "sudo apt-get",
             # no expect because stdout seems to be redirected
-            packages: "libcmocka-dev shunit2",
+            packages: "libcmocka-dev libxxhash-dev shunit2",
             snaps: "",
             build-cmd: "make"
           }
@@ -36,7 +36,7 @@ jobs:
             cc: "clang",
             options: "-DENABLE_TESTS=ON",
             packager: "sudo apt-get",
-            packages: "libcmocka-dev shunit2",
+            packages: "libcmocka-dev libxxhash-dev shunit2",
             snaps: "",
             build-cmd: "make"
           }
@@ -47,7 +47,7 @@ jobs:
             cc: "gcc",
             options: "",
             packager: "sudo apt-get",
-            packages: "libcmocka-dev valgrind shunit2",
+            packages: "libcmocka-dev libxxhash-dev valgrind shunit2",
             snaps: "",
             build-cmd: "make"
           }
@@ -59,7 +59,7 @@ jobs:
             options: "",
             packager: "sudo apt-get",
             # no valgrind because it does not support DWARF5 yet generated by clang 14
-            packages: "libcmocka-dev shunit2",
+            packages: "libcmocka-dev libxxhash-dev shunit2",
             snaps: "",
             build-cmd: "make"
           }
@@ -70,7 +70,7 @@ jobs:
             cc: "clang",
             options: "-DENABLE_TESTS=ON -DPATH_EXPECT=",
             packager: "brew",
-            packages: "cmocka shunit2 tcl-tk",
+            packages: "cmocka xxhash shunit2 tcl-tk",
             snaps: "",
             build-cmd: "make"
           }
@@ -81,7 +81,7 @@ jobs:
             cc: "clang",
             options: "-DCMAKE_C_FLAGS=-fsanitize=address,undefined -DENABLE_TESTS=ON -DENABLE_VALGRIND_TESTS=OFF",
             packager: "sudo apt-get",
-            packages: "libcmocka-dev",
+            packages: "libcmocka-dev libxxhash-dev",
             snaps: "",
             build-cmd: "make"
           }
@@ -92,7 +92,7 @@ jobs:
             cc: "gcc",
             options: "",
             packager: "sudo apt-get",
-            packages: "libcmocka-dev abi-dumper abi-compliance-checker",
+            packages: "libcmocka-dev libxxhash-dev abi-dumper abi-compliance-checker",
             snaps: "core universal-ctags",
             build-cmd: "make abi-check"
           }
@@ -103,7 +103,7 @@ jobs:
             cc: "gcc",
             options: "",
             packager: "sudo apt-get",
-            packages: "cmake debhelper libcmocka-dev python3-pip",
+            packages: "cmake debhelper libcmocka-dev libxxhash-dev python3-pip",
             snaps: "",
             build-cmd: ""
           }
@@ -210,7 +210,7 @@ jobs:
         uses: SimenB/github-actions-cpu-cores@v1
 
       - name: Install Windows dependencies
-        run: vcpkg install --triplet=${{ matrix.triplet }} pcre2 pthreads dirent dlfcn-win32 cmocka getopt
+        run: vcpkg install --triplet=${{ matrix.triplet }} pcre2 pthreads dirent dlfcn-win32 cmocka getopt xxhash
 
       - name: Configure
         shell: bash

.github/workflows/devel-push.yml

@@ -34,7 +34,7 @@ jobs:
             cc: "gcc",
             options: "-DENABLE_COVERAGE=ON",
             packager: "sudo apt-get",
-            packages: "libcmocka-dev lcov",
+            packages: "libcmocka-dev libxxhash-dev lcov",
             snaps: "",
             make-prepend: "",
             make-target: ""

CMakeLists.txt

@@ -242,6 +242,7 @@ else()
     option(ENABLE_TESTS "Build tests" OFF)
     option(ENABLE_VALGRIND_TESTS "Build tests with valgrind" OFF)
 endif()
+option(USE_XXHASH "Use xxhash if available" ON)
 option(ENABLE_PERF_TESTS "Build performance tests" OFF)
 option(ENABLE_COVERAGE "Build code coverage report from tests" OFF)
 option(ENABLE_FUZZ_TARGETS "Build target programs suitable for fuzzing with AFL" OFF)
@@ -411,6 +412,16 @@ find_package(PCRE2 10.21 REQUIRED)
 include_directories(${PCRE2_INCLUDE_DIRS})
 target_link_libraries(yang ${PCRE2_LIBRARIES})
 
+# XXHash include and library
+find_package(XXHash)
+if (XXHASH_FOUND AND USE_XXHASH)
+    add_definitions(-DUSE_XXHASH)
+    include_directories(${XXHASH_INCLUDE_DIR})
+    target_link_libraries(yang ${XXHASH_LIBRARY})
+else()
+    set (USE_XXHASH OFF)
+endif()
+
 # generated header list
 foreach(h IN LISTS gen_headers)
     list(APPEND g_headers ${PROJECT_BINARY_DIR}/libyang/${h})

CMakeModules/FindXXHash.cmake

@@ -0,0 +1,38 @@
+# Try to find XXHash
+# Once done this will define
+#
+# Read-Only variables:
+#  XXHASH_FOUND - system has XXHash
+#  XXHASH_INCLUDE_DIR - the XXHash include directory
+#  XXHASH_LIBRARY - Link these to use XXHash
+
+find_path(XXHASH_INCLUDE_DIR
+    NAMES
+        xxhash.h
+    PATHS
+        /usr/include
+        /usr/local/include
+        /opt/local/include
+        /sw/include
+        ${CMAKE_INCLUDE_PATH}
+        ${CMAKE_INSTALL_PREFIX}/include
+)
+
+find_library(XXHASH_LIBRARY
+    NAMES
+        xxhash
+        libxxhash
+    PATHS
+        PATHS
+        /usr/lib
+        /usr/lib64
+        /usr/local/lib
+        /usr/local/lib64
+        /opt/local/lib
+        /sw/lib
+        ${CMAKE_LIBRARY_PATH}
+        ${CMAKE_INSTALL_PREFIX}/lib
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(XXHash FOUND_VAR XXHASH_FOUND REQUIRED_VARS XXHASH_INCLUDE_DIR XXHASH_LIBRARY)

src/hash_table.c

@@ -26,6 +26,27 @@
 #include "log.h"
 #include "ly_common.h"
 
+#ifdef USE_XXHASH
+#include <xxhash.h>
+
+LIBYANG_API_DEF uint32_t
+lyht_hash_multi(uint32_t hash, const char *key_part, size_t len)
+{
+    if (key_part && len) {
+        return XXH3_64bits_withSeed(key_part, len, hash);
+    }
+
+    return XXH3_64bits_withSeed(NULL, 0, hash);
+}
+
+LIBYANG_API_DEF uint32_t
+lyht_hash(const char *key, size_t len)
+{
+    return XXH3_64bits(key, len);
+}
+
+#else
+
 LIBYANG_API_DEF uint32_t
 lyht_hash_multi(uint32_t hash, const char *key_part, size_t len)
 {
@@ -55,6 +76,8 @@ lyht_hash(const char *key, size_t len)
     return lyht_hash_multi(hash, NULL, len);
 }
 
+#endif
+
 static LY_ERR
 lyht_init_hlists_and_records(struct ly_ht *ht)
 {

src/hash_table.h

@@ -54,6 +54,8 @@ LIBYANG_API_DECL uint32_t lyht_hash_multi(uint32_t hash, const char *key_part, s
  *
  * Spooky hash is faster, but it works only for little endian architectures.
  *
+ * Uses XXH3_64bits internally if xxhash is available. See https://xxhash.com
+ *
  * @param[in] key Key to hash.
  * @param[in] len Length of @p key.
  * @return Hash of the key.

tests/CMakeLists.txt

@@ -30,6 +30,9 @@ function(ly_add_utest)
         endif()
     endif()
 
+    if (USE_XXHASH)
+        target_link_libraries(${TEST_NAME} ${XXHASH_LIBRARY})
+    endif()
     add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
     set_property(TEST ${TEST_NAME} APPEND PROPERTY ENVIRONMENT "MALLOC_CHECK_=3")
 

tests/perf/CMakeLists.txt

@@ -6,6 +6,10 @@ set(format_sources
 add_executable(ly_perf ${CMAKE_CURRENT_SOURCE_DIR}/perf.c $<TARGET_OBJECTS:yangobj>)
 set_target_properties(ly_perf PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests")
 target_link_libraries(ly_perf ${CMAKE_THREAD_LIBS_INIT} ${PCRE2_LIBRARIES} ${CMAKE_DL_LIBS})
+if (USE_XXHASH)
+    target_link_libraries(ly_perf ${XXHASH_LIBRARY})
+endif()
+
 if (NOT WIN32)
     target_link_libraries(ly_perf m)
 endif()

tests/style/CMakeLists.txt

@@ -9,4 +9,8 @@ endif()
 add_executable(cpp_compat cpp_compat.c $<TARGET_OBJECTS:yangobj>)
 target_include_directories(cpp_compat BEFORE PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(cpp_compat ${CMAKE_THREAD_LIBS_INIT} ${PCRE2_LIBRARIES} ${CMAKE_DL_LIBS} m)
+if (USE_XXHASH)
+    target_link_libraries(cpp_compat ${XXHASH_LIBRARY})
+endif()
+
 target_compile_options(cpp_compat PUBLIC "-Werror=c++-compat")