diff --git a/.github/workflows/build-bsd.yml b/.github/workflows/build-bsd.yml new file mode 100644 index 0000000000..add3f4ad58 --- /dev/null +++ b/.github/workflows/build-bsd.yml @@ -0,0 +1,107 @@ +name: BSD + +on: + push: + branches: + - master + - ci/* + pull_request: + branches: + - master + +env: + MELONDS_GIT_BRANCH: ${{ github.ref }} + MELONDS_GIT_HASH: ${{ github.sha }} + MELONDS_BUILD_PROVIDER: GitHub Actions + +jobs: + build: + strategy: + matrix: + os: [freebsd, netbsd, openbsd] + # The emulated aarch64 VMs are quite slow, so disable aarch64 + # runs for now. + arch: [x86_64] #, aarch64] + include: + - os: freebsd + os_name: FreeBSD + # CMake's autorcc for Qt tries to lock files, so we need to + # start lockd for NFS, otherwise the build will fail. + prepare_vm: service lockd onestart + install_deps: > + pkg install -y cmake kf6-extra-cmake-modules pkgconf ninja + qt6-base qt6-multimedia qt6-svg sdl2 libarchive zstd enet + faad2 + + - os: netbsd + os_name: NetBSD + # The NetBSD VM image does not come with X sets by default, + # so download and unpack the sets we need manually. + # See https://github.com/vmactions/netbsd-vm/issues/2 + # Also, for some reason pkg_add is not in $PATH, so we have + # to call it using its absolute location. + prepare_vm: | + /usr/sbin/pkg_add pkgin curl + mach=$(uname -m | sed "s/evbarm/evbarm-$(uname -p)/") + for set in xbase xcomp; do + curl -sL ${PKG_PATH%%/pkgsrc*}/NetBSD/NetBSD-$(uname -r + )/$mach/binary/sets/$set.tar.xz | tar xJpf - -C / + done + install_deps: > + pkgin -y install cmake extra-cmake-modules pkgconf + ninja-build qt6-qtbase qt6-qtmultimedia qt6-qtsvg SDL2 + libarchive zstd enet faad2 + + - os: openbsd + os_name: OpenBSD + install_deps: > + pkg_add -v cmake kf6-extra-cmake-modules pkgconf ninja + qt6-qtbase qt6-qtmultimedia qt6-qtsvg sdl2 libarchive zstd + enet faad + + + name: ${{ matrix.os_name }} / ${{ matrix.arch }} + runs-on: ubuntu-latest + defaults: + run: + shell: ${{ matrix.os }} {0} + + steps: + - uses: actions/checkout@v4 + name: Check out sources + + - name: Start ${{ matrix.os_name }} VM + uses: jenseng/dynamic-uses@v1 + with: + uses: vmactions/${{ matrix.os }}-vm@v1 + # As we are using multiple steps, we need to use NFS to sync the + # workspace between host and VM, as with the default, rsync, the + # workspace is only synced before and after the setup VM step. + with: > + { + "arch": ${{ matrix.arch }}, + "envs": "MELONDS_GIT_BRANCH MELONDS_GIT_HASH + MELONDS_BUILD_PROVIDER", + "prepare": ${{ toJSON(matrix.prepare_vm) }}, + "sync": "nfs", + "usesh": "true", + } + + - name: Install dependencies + run: ${{ matrix.install_deps }} + + - name: Configure + run: | + cd $GITHUB_WORKSPACE + cmake -B build -G Ninja -DMELONDS_EMBED_BUILD_INFO=ON + + - name: Build + run: | + cd $GITHUB_WORKSPACE + cmake --build build + + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: melonDS-${{ matrix.os }}-${{ matrix.arch }} + path: build/melonDS diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index cf9c7f45cd..7d5e94a4c5 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -10,7 +10,6 @@ on: - master env: - VCPKG_COMMIT: 2ad004460f5db4d3b66f62f5799ff66c265c4b5d MELONDS_GIT_BRANCH: ${{ github.ref }} MELONDS_GIT_HASH: ${{ github.sha }} MELONDS_BUILD_PROVIDER: GitHub Actions @@ -24,6 +23,10 @@ jobs: name: ${{ matrix.arch }} runs-on: macos-14 + + env: + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg_cache,readwrite + steps: - name: Check out sources uses: actions/checkout@v3 @@ -32,10 +35,16 @@ jobs: brew install autoconf automake autoconf-archive libtool python-setuptools - name: Set up CMake uses: lukka/get-cmake@latest + - name: Restore vcpkg cache + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }}/vcpkg_cache + key: vcpkg-${{ matrix.arch }}-osx-${{ hashFiles('vcpkg.json') }} + restore-keys: vcpkg-${{ matrix.arch }}-osx- - name: Set up vcpkg uses: lukka/run-vcpkg@v11 with: - vcpkgGitCommitId: ${{ env.VCPKG_COMMIT }} + vcpkgJsonGlob: vcpkg.json - name: Build uses: lukka/run-cmake@v10 with: @@ -53,6 +62,11 @@ jobs: name: macOS-${{ matrix.arch }} path: macOS-${{ matrix.arch }}.zip retention-days: 1 + - name: Save vcpkg cache + uses: actions/cache/save@v4 + with: + path: ${{ github.workspace }}/vcpkg_cache + key: vcpkg-${{ matrix.arch }}-osx-${{ hashFiles('vcpkg.json') }} universal-binary: name: Universal binary diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index f582c37996..c8597619ed 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -10,7 +10,6 @@ on: - master env: - VCPKG_COMMIT: 2ad004460f5db4d3b66f62f5799ff66c265c4b5d MELONDS_GIT_BRANCH: ${{ github.ref }} MELONDS_GIT_HASH: ${{ github.sha }} MELONDS_BUILD_PROVIDER: GitHub Actions @@ -18,10 +17,12 @@ env: jobs: build: - runs-on: windows-latest + runs-on: windows-2025 defaults: run: shell: msys2 {0} + env: + VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg_cache,readwrite steps: - name: Check out sources uses: actions/checkout@v3 @@ -31,10 +32,16 @@ jobs: msystem: ucrt64 update: true pacboy: gcc:p cmake:p ninja:p make:p + - name: Restore vcpkg cache + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }}/vcpkg_cache + key: vcpkg-x86_64-mingw-${{ hashFiles('vcpkg.json') }} + restore-keys: vcpkg-x86_64-mingw- - name: Set up vcpkg uses: lukka/run-vcpkg@v11 with: - vcpkgGitCommitId: ${{ env.VCPKG_COMMIT }} + vcpkgJsonGlob: vcpkg.json - name: Configure run: cmake --preset=release-mingw-x86_64 -DMELONDS_EMBED_BUILD_INFO=ON - name: Build @@ -43,3 +50,8 @@ jobs: with: name: melonDS-windows-x86_64 path: .\build\release-mingw-x86_64\melonDS.exe + - name: Save vcpkg cache + uses: actions/cache/save@v4 + with: + path: ${{ github.workspace }}/vcpkg_cache + key: vcpkg-x86_64-mingw-${{ hashFiles('vcpkg.json') }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 531ca19f15..c74f6b93a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ if (USE_VCPKG) endif() project(melonDS - VERSION 1.0 + VERSION 1.1 DESCRIPTION "DS emulator, sorta" HOMEPAGE_URL "https://melonds.kuribo64.net" LANGUAGES C CXX) @@ -63,7 +63,7 @@ detect_architecture("__arm__" ARM) detect_architecture("__aarch64__" ARM64) cmake_dependent_option(ENABLE_JIT "Enable JIT recompiler" ON - "ARCHITECTURE STREQUAL x86_64 OR ARCHITECTURE STREQUAL ARM64" OFF) + "ARCHITECTURE STREQUAL x86_64 OR ARCHITECTURE STREQUAL ARM64;NOT ARCHITECTURE STREQUAL x86_64 OR NOT APPLE" OFF) cmake_dependent_option(ENABLE_JIT_PROFILING "Enable JIT profiling with VTune" OFF "ENABLE_JIT" OFF) option(ENABLE_OGLRENDERER "Enable OpenGL renderer" ON) @@ -85,10 +85,15 @@ endif() if (WIN32) option(BUILD_STATIC "Statically link dependencies" OFF) -endif() - -if (BUILD_STATIC AND WIN32) - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + add_definitions(-DWIN32_LEAN_AND_MEAN -DNOMINMAX) + if (NOT MINGW) + add_definitions(-D__WIN32__ -D_CRT_SECURE_NO_WARNINGS -D_USE_MATH_DEFINES) + if (BUILD_STATIC) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + endif() + elseif (BUILD_STATIC AND MINGW) + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif() endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/README.md b/README.md index e08ca2b80f..db919b498b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

melonDS

- + @@ -10,6 +10,7 @@ +

DS emulator, sorta diff --git a/cmake/ConfigureVcpkg.cmake b/cmake/ConfigureVcpkg.cmake index 2f8e9ad305..b02587c5e1 100644 --- a/cmake/ConfigureVcpkg.cmake +++ b/cmake/ConfigureVcpkg.cmake @@ -9,17 +9,23 @@ if (VCPKG_ROOT STREQUAL "${_DEFAULT_VCPKG_ROOT}") endif() FetchContent_Declare(vcpkg GIT_REPOSITORY "https://github.com/Microsoft/vcpkg.git" - GIT_TAG 2ad004460f5db4d3b66f62f5799ff66c265c4b5d + GIT_TAG HEAD EXCLUDE_FROM_ALL SOURCE_DIR "${CMAKE_SOURCE_DIR}/vcpkg") FetchContent_MakeAvailable(vcpkg) endif() -set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_SOURCE_DIR}/cmake/overlay-triplets") -set(VCPKG_OVERLAY_PORTS "${CMAKE_SOURCE_DIR}/cmake/overlay-ports/faad2") - option(USE_RECOMMENDED_TRIPLETS "Use the recommended triplets that are used for official builds" ON) +# Required for Windows non-MinGW debug builds, optional for all other platforms +if (WIN32) + if (DEFINED ENV{MINGW_PREFIX}) + option(ENABLE_DEBUG_DEPS "Enable debug builds of vcpkg dependencies" OFF) + else() + option(ENABLE_DEBUG_DEPS "Enable debug builds of vcpkg dependencies" ON) + endif() +endif() + # Duplicated here because it needs to be set before project() option(USE_QT6 "Use Qt 6 instead of Qt 5" ON) @@ -62,9 +68,30 @@ if (USE_RECOMMENDED_TRIPLETS) set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15) endif() elseif(WIN32) - # TODO Windows arm64 if possible set(_CAN_TARGET_AS_HOST ON) - set(_WANTED_TRIPLET x64-mingw-static-release) + if (ENABLE_DEBUG_DEPS) + set(_RELEASE_SUFFIX "") + else() + set(_RELEASE_SUFFIX "-release") + endif() + + if (DEFINED ENV{MINGW_PREFIX}) + set(_WIN32_ENV mingw) + else() + set(_WIN32_ENV windows) + endif() + + if ("$ENV{PROCESSOR_ARCHITECTURE}" STREQUAL "AMD64") + set(_WIN32_ARCH x64) + elseif ("$ENV{PROCESSOR_ARCHITECTURE}" STREQUAL "ARM64") + set(_WIN32_ARCH arm64) + elseif ("$ENV{PROCESSOR_ARCHITECTURE}" STREQUAL "x86") + set(_WIN32_ARCH x86) + else() + message(FATAL_ERROR "Unknown processor architecture '$ENV{PROCESSOR_ARCHITECTURE}'. Please disable USE_RECOMMENDED_TRIPLETS and set your vcpkg settings manually.") + endif() + + set(_WANTED_TRIPLET ${_WIN32_ARCH}-${_WIN32_ENV}-static${_RELEASE_SUFFIX}) elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL Linux) # Can't really detect cross compiling here. set(_CAN_TARGET_AS_HOST ON) diff --git a/flake.lock b/flake.lock index 85be38faac..725c6d355e 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1739020877, - "narHash": "sha256-mIvECo/NNdJJ/bXjNqIh8yeoSjVLAuDuTUzAo7dzs8Y=", + "lastModified": 1761672384, + "narHash": "sha256-o9KF3DJL7g7iYMZq9SWgfS1BFlNbsm6xplRjVlOCkXI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a79cfe0ebd24952b580b1cf08cd906354996d547", + "rev": "08dacfca559e1d7da38f3cf05f1f45ee9bfd213c", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 8dedde8f0d..efa9be50c3 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,7 @@ libGL libslirp enet + faad2 ]) ++ optionals (!isDarwin) (with pkgs; [ kdePackages.extra-cmake-modules qt6.qtwayland @@ -79,6 +80,12 @@ packages = with pkgs; [ qt6.qttools ]; + + # https://discourse.nixos.org/t/how-to-create-a-qt-devshell/58529/5 + shellHook = '' + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$APPEND_LIBRARY_PATH" + eval $(echo "''${qtWrapperArgs[@]}"|perl -n -e '$_ =~ s/--prefix (\w+) : ([\w-.\/]+)/export ''${1}="''${2}:\''${''${1}}";/g;print') + ''; }; # Shell for building static melonDS release builds with vcpkg diff --git a/melonDLDI/melonDLDI.s b/melonDLDI/melonDLDI.s index 399ad032ee..eb98a67118 100755 --- a/melonDLDI/melonDLDI.s +++ b/melonDLDI/melonDLDI.s @@ -7,7 +7,7 @@ _start: .word 0xBF8DA5ED .string " Chishm" .byte 1 -.byte 9 @ size +.byte 10 @ size .byte 0 .byte 0 .string "melonDS DLDI driver" diff --git a/res/melon.rc.in b/res/melon.rc.in index c96d0f9e2c..ac2bb84b57 100644 --- a/res/melon.rc.in +++ b/res/melon.rc.in @@ -31,4 +31,6 @@ FILETYPE VFT_APP } } +#if (0${MINGW}) 1 24 "xp.manifest" +#endif diff --git a/src/ARCodeFile.cpp b/src/ARCodeFile.cpp index deb8bc403f..df6d26e4ce 100644 --- a/src/ARCodeFile.cpp +++ b/src/ARCodeFile.cpp @@ -16,9 +16,10 @@ with melonDS. If not, see http://www.gnu.org/licenses/. */ -#include -#include +#include +#include #include "ARCodeFile.h" +#include "ARDatabaseDAT.h" #include "Platform.h" namespace melonDS @@ -44,10 +45,21 @@ std::vector ARCodeFile::GetCodes() const noexcept std::vector codes; - for (const ARCodeCat& cat : Categories) + for (auto& item : RootCat.Children) { - for (const ARCode& code : cat.Codes) + if (std::holds_alternative(item)) { + auto& cat = std::get(item); + + for (auto& childitem : cat.Children) + { + auto& code = std::get(childitem); + codes.push_back(code); + } + } + else + { + auto& code = std::get(item); codes.push_back(code); } } @@ -60,13 +72,17 @@ bool ARCodeFile::Load() FileHandle* f = OpenFile(Filename, FileMode::ReadText); if (!f) return true; - Categories.clear(); + RootCat.Parent = nullptr; + RootCat.OnlyOneCodeEnabled = false; + RootCat.Children.clear(); bool isincat = false; - ARCodeCat curcat; + ARCodeCat* curcat = &RootCat; bool isincode = false; - ARCode curcode; + ARCode* curcode = nullptr; + + int lastentry = 0; char linebuf[1024]; while (!IsEndOfFile(f)) @@ -83,27 +99,54 @@ bool ARCodeFile::Load() if (start[0]=='#' || start[0]=='\r' || start[0]=='\n' || start[0]=='\0') continue; - if (!strncasecmp(start, "CAT", 3)) + if (!strncasecmp(start, "ROOT", 4)) + { + isincode = false; + isincat = true; + + curcat = &RootCat; + lastentry = 0; + } + else if (!strncasecmp(start, "CAT", 3)) { char catname[128]; - int ret = sscanf(start, "CAT %127[^\r\n]", catname); + int ret, retchk; + int onlyone; + if (start[3] == ' ' && (start[4] == '0' || start[4] == '1') && start[5] == ' ') + { + retchk = 2; + ret = sscanf(start, "CAT %d %127[^\r\n]", &onlyone, catname); + } + else + { + // backwards compatibility + onlyone = 0; + retchk = 1; + ret = sscanf(start, "CAT %127[^\r\n]", catname); + } catname[127] = '\0'; - if (ret < 1) + if (ret < retchk) { Log(LogLevel::Error, "AR: malformed CAT line: %s\n", start); CloseFile(f); return false; } - if (isincode) curcat.Codes.push_back(curcode); isincode = false; - - if (isincat) Categories.push_back(curcat); isincat = true; - curcat.Name = catname; - curcat.Codes.clear(); + ARCodeCat cat = { + .Parent = &RootCat, + .Name = catname, + .Description = "", + .OnlyOneCodeEnabled = onlyone!=0, + .Children = {} + }; + RootCat.Children.emplace_back(cat); + curcat = &std::get(RootCat.Children.back()); + + lastentry = 1; } else if (!strncasecmp(start, "CODE", 4)) { @@ -126,12 +169,39 @@ bool ARCodeFile::Load() return false; } - if (isincode) curcat.Codes.push_back(curcode); isincode = true; - curcode.Name = codename; - curcode.Enabled = enable!=0; - curcode.Code.clear(); + ARCode code = { + .Parent = curcat, + .Name = codename, + .Description = "", + .Enabled = enable!=0, + .Code = {} + }; + curcat->Children.emplace_back(code); + curcode = &std::get(curcat->Children.back()); + + lastentry = 2; + } + else if (!strncasecmp(start, "DESC", 4)) + { + char desc[256]; + int ret = sscanf(start, "DESC %255[^\r\n]", desc); + desc[255] = '\0'; + + if (ret < 1) + continue; + + if (lastentry == 2) + curcode->Description = desc; + else if (lastentry == 1) + curcat->Description = desc; + else + { + Log(LogLevel::Error, "AR: encountered DESC line not part of anything\n"); + CloseFile(f); + return false; + } } else { @@ -152,13 +222,12 @@ bool ARCodeFile::Load() return false; } - curcode.Code.push_back(c0); - curcode.Code.push_back(c1); + curcode->Code.push_back(c0); + curcode->Code.push_back(c1); } } - if (isincode) curcat.Codes.push_back(curcode); - if (isincat) Categories.push_back(curcat); + FinalizeList(); CloseFile(f); return true; @@ -169,22 +238,51 @@ bool ARCodeFile::Save() FileHandle* f = Platform::OpenFile(Filename, FileMode::WriteText); if (!f) return false; - for (ARCodeCatList::iterator it = Categories.begin(); it != Categories.end(); it++) + bool isincat = true; + + for (auto& item : RootCat.Children) { - ARCodeCat& cat = *it; + if (std::holds_alternative(item)) + { + auto& cat = std::get(item); - if (it != Categories.begin()) FileWriteFormatted(f, "\n"); - FileWriteFormatted(f, "CAT %s\n\n", cat.Name.c_str()); + FileWriteFormatted(f, "CAT %d %s\n", cat.OnlyOneCodeEnabled, cat.Name.c_str()); + if (!cat.Description.empty()) + FileWriteFormatted(f, "DESC %s\n", cat.Description.c_str()); + FileWriteFormatted(f, "\n"); + + isincat = true; + + for (auto& childitem : cat.Children) + { + auto& code = std::get(childitem); - for (ARCodeList::iterator jt = cat.Codes.begin(); jt != cat.Codes.end(); jt++) + FileWriteFormatted(f, "CODE %d %s\n", code.Enabled, code.Name.c_str()); + if (!code.Description.empty()) + FileWriteFormatted(f, "DESC %s\n", code.Description.c_str()); + + for (size_t i = 0; i < code.Code.size(); i+=2) + FileWriteFormatted(f, "%08X %08X\n", code.Code[i], code.Code[i + 1]); + + FileWriteFormatted(f, "\n"); + } + } + else { - ARCode& code = *jt; + auto& code = std::get(item); + + if (isincat) + { + isincat = false; + FileWriteFormatted(f, "ROOT\n\n"); + } + FileWriteFormatted(f, "CODE %d %s\n", code.Enabled, code.Name.c_str()); + if (!code.Description.empty()) + FileWriteFormatted(f, "DESC %s\n", code.Description.c_str()); for (size_t i = 0; i < code.Code.size(); i+=2) - { FileWriteFormatted(f, "%08X %08X\n", code.Code[i], code.Code[i + 1]); - } FileWriteFormatted(f, "\n"); } @@ -194,4 +292,109 @@ bool ARCodeFile::Save() return true; } +void ARCodeFile::Import(ARDatabaseEntry& dbentry, ARCodeEnableMap& enablemap, bool clear) +{ + bool hasenablemap = !enablemap.empty(); + + if (clear) + RootCat.Children.clear(); + + for (auto& item : dbentry.RootCat.Children) + { + if (std::holds_alternative(item)) + { + auto& cat = std::get(item); + + bool shouldimport = false; + if (hasenablemap) + { + for (auto& childitem: cat.Children) + { + auto& code = std::get(childitem); + if (enablemap[&code]) + { + shouldimport = true; + break; + } + } + } + else + shouldimport = true; + + if (!shouldimport) + continue; + + ARCodeCat newcat = { + .Parent = &RootCat, + .Name = cat.Name, + .Description = cat.Description, + .OnlyOneCodeEnabled = cat.OnlyOneCodeEnabled, + .Children = {} + }; + RootCat.Children.emplace_back(newcat); + ARCodeCat* parentptr = &std::get(RootCat.Children.back()); + + for (auto& childitem : cat.Children) + { + auto& code = std::get(childitem); + if (hasenablemap && (!enablemap[&code])) + continue; + + ARCode newcode = { + .Parent = parentptr, + .Name = code.Name, + .Description = code.Description, + .Enabled = code.Enabled, + .Code = code.Code + }; + parentptr->Children.emplace_back(newcode); + } + } + else + { + auto& code = std::get(item); + if (hasenablemap && (!enablemap[&code])) + continue; + + ARCode newcode = { + .Parent = &RootCat, + .Name = code.Name, + .Description = code.Description, + .Enabled = code.Enabled, + .Code = code.Code + }; + RootCat.Children.emplace_back(newcode); + } + } + + FinalizeList(); +} + +void ARCodeFile::FinalizeList() +{ + for (auto& item : RootCat.Children) + { + if (!std::holds_alternative(item)) + continue; + + auto& cat = std::get(item); + if (!cat.OnlyOneCodeEnabled) + continue; + + // for categories that only allow one code to be enabled: + // make sure we don't have multiple ones enabled + + bool foundone = false; + for (auto& childitem : cat.Children) + { + auto& code = std::get(childitem); + if (!code.Enabled) continue; + if (foundone) + code.Enabled = false; + else + foundone = true; + } + } +} + } \ No newline at end of file diff --git a/src/ARCodeFile.h b/src/ARCodeFile.h index 823871da70..200259e8d5 100644 --- a/src/ARCodeFile.h +++ b/src/ARCodeFile.h @@ -22,26 +22,38 @@ #include #include #include +#include +#include #include "types.h" namespace melonDS { + +struct ARCodeCat; + struct ARCode { + ARCodeCat* Parent; std::string Name; + std::string Description; bool Enabled; std::vector Code; }; -typedef std::list ARCodeList; +typedef std::variant ARCodeItem; +typedef std::list ARCodeItemList; struct ARCodeCat { + ARCodeCat* Parent; std::string Name; - ARCodeList Codes; + std::string Description; + bool OnlyOneCodeEnabled; + ARCodeItemList Children; }; -typedef std::list ARCodeCatList; +struct ARDatabaseEntry; +typedef std::unordered_map ARCodeEnableMap; class ARCodeFile @@ -57,10 +69,14 @@ class ARCodeFile bool Load(); bool Save(); - ARCodeCatList Categories {}; + void Import(ARDatabaseEntry& dbentry, ARCodeEnableMap& enablemap, bool clear); + + ARCodeCat RootCat {}; private: std::string Filename; + + void FinalizeList(); }; } diff --git a/src/ARDatabaseDAT.cpp b/src/ARDatabaseDAT.cpp new file mode 100644 index 0000000000..7246480ceb --- /dev/null +++ b/src/ARDatabaseDAT.cpp @@ -0,0 +1,374 @@ +/* + Copyright 2016-2025 melonDS team + + This file is part of melonDS. + + melonDS is free software: you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + melonDS is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with melonDS. If not, see http://www.gnu.org/licenses/. +*/ + +#include +#include "ARDatabaseDAT.h" +#include "Platform.h" + +/* FILE FORMAT for usrcheat.dat + + header: + 00: ID string "R4 CheatCode" (12 bytes) + 0C: version? must be 0x100 + 10: database description + 4C: ??? (gets set by r4cce.exe, but zero in usrcheat.dat) + 50: database active flag + 100: entry list + + entry: + 00: game code + 04: checksum of ROM header (logical NOT of CRC32 over first 512 bytes) + 08: offset to cheat data (absolute) + 0C: zero + + the entry list is terminated with an all-zero entry + + cheat data: header then items + + header: + * game name (null-term string) + * padding (to align entry below to 4-byte boundary) + * flags (word): bit0-27 = total number of items, bit28-31 = game active (0=inactive, F=active) + * master codes (8 words) - use unclear + + item: + * flags (word): bit0-23 = data length, bit24 = flag, bit28 = type + * item name (null-term string) + * item description (null-term string) + * padding (to align data below to 4-byte boundary) + * item data + + meaning of item type/flag: + * type=0: cheat, bit0-23 = total data length in words, bit24 = cheat active + * type=1: category, bit0-23 = number of cheats inside, bit24 = only one cheat may be active in this category + + empty description still takes up one extra byte (for the null terminator) + + for a category: item data is a list of cheat items + + for a cheat: item data is as follows: + * number of code words (word) + * code +*/ + +namespace melonDS +{ +using namespace Platform; + +// TODO: more user-friendly error reporting + +std::string ReadNTString(Platform::FileHandle* f); +void AlignFilePos(Platform::FileHandle* f); + + +ARDatabaseDAT::ARDatabaseDAT(const std::string& filename) +{ + Filename = filename; + + if (!LoadEntries()) + Error = true; +} + +bool ARDatabaseDAT::FindGameCode(u32 gamecode) +{ + auto it = EntryList.find(gamecode); + if (it == EntryList.end()) + return false; + if ((*it).second.empty()) + return false; + return true; +} + +ARDatabaseEntryList ARDatabaseDAT::GetEntriesByGameCode(u32 gamecode) +{ + ARDatabaseEntryList ret; + auto it = EntryList.find(gamecode); + if (it == EntryList.end()) + return ret; + + for (auto& info : (*it).second) + { + ARDatabaseEntry entry; + LoadCheatCodes(info, entry); + ret.push_back(entry); + } + + return ret; +} + +bool ARDatabaseDAT::LoadEntries() +{ + FileHandle* f = OpenFile(Filename, FileMode::Read); + if (!f) return false; + + u64 filelen = FileLength(f); + if (filelen > 0xFFFFFFFFULL) + { + CloseFile(f); + return false; + } + + char header[16]; + FileRead(header, 16, 1, f); + //if (strncmp(header, "R4 CheatCode", 12) != 0) + if (memcmp(header, "R4 CheatCode\x00\x01\x00\x00", 16) != 0) + { + CloseFile(f); + return false; + } + + char name[0x3D] = {0}; + FileRead(name, 0x3C, 1, f); + DBName = name; + + FileSeek(f, 0x100, FileSeekOrigin::Start); + while (!IsEndOfFile(f)) + { + u32 entrydata[4]; + FileRead(entrydata, 16, 1, f); + + // a zero entry marks the end of the entry list + if (entrydata[0] == 0) + break; + + if ((entrydata[2] < 0x100) || (entrydata[2] >= filelen)) + { + Log(LogLevel::Error, "AR: malformed database file (invalid offset %08X)\n", entrydata[2]); + CloseFile(f); + return false; + } + + EntryInfo entry; + entry.GameCode = entrydata[0]; + entry.Checksum = entrydata[1]; + entry.Offset = entrydata[2]; + + EntryList[entry.GameCode].push_back(entry); + } + + CloseFile(f); + return true; +} + +bool ARDatabaseDAT::LoadCheatCodes(EntryInfo& info, ARDatabaseEntry& entry) +{ + FileHandle* f = OpenFile(Filename, FileMode::Read); + if (!f) return false; + + u64 filelen = FileLength(f); + if (filelen > 0xFFFFFFFFULL) + { + CloseFile(f); + return false; + } + + if ((info.Offset < 0x100) || (info.Offset >= filelen)) + { + CloseFile(f); + return false; + } + + entry.GameCode = info.GameCode; + entry.Checksum = info.Checksum; + + FileSeek(f, info.Offset, FileSeekOrigin::Start); + + entry.Name = ReadNTString(f); + AlignFilePos(f); + + u32 flags[9] = {0}; + FileRead(flags, 4*9, 1, f); + + entry.RootCat.Parent = nullptr; + entry.RootCat.OnlyOneCodeEnabled = false; + entry.RootCat.Children.clear(); + + ARCodeCat* curcat = &entry.RootCat; + int catlen = 0; + + u32 numentries = flags[0] & 0xFFFFFF; + for (u32 i = 0; i < numentries; i++) + { + if (IsEndOfFile((f))) + break; + + u32 itemflags = 0; + FileRead(&itemflags, 4, 1, f); + + u32 totallen = itemflags & 0xFFFFFF; + + std::string itemname = ReadNTString(f); + std::string itemdesc = ReadNTString(f); + AlignFilePos(f); + + if (itemflags & (1<<28)) + { + // this item is a category + + if ((totallen >= 0x10000) || (totallen == 0)) + { + Log(LogLevel::Error, "AR: unreasonable category length %08X\n", + totallen); + Log(LogLevel::Error, "game=%s, offset=%08X, cat=%s\n", + entry.Name.c_str(), info.Offset, itemname.c_str()); + + CloseFile(f); + return false; + } + + ARCodeCat cat = { + .Parent = &entry.RootCat, + .Name = itemname, + .Description = itemdesc, + .OnlyOneCodeEnabled = !!(itemflags & (1<<24)), + .Children = {} + }; + entry.RootCat.Children.emplace_back(cat); + curcat = &std::get(entry.RootCat.Children.back()); + + catlen = totallen; + } + else + { + // this item is a code + + u32 codelen = 0; + FileRead(&codelen, 4, 1, f); + + u32 chklen = itemname.length() + 1 + itemdesc.length() + 1; + chklen = ((chklen + 3) >> 2) + 1 + codelen; + if (chklen != totallen) + { + Log(LogLevel::Error, "AR: malformed code entry, codelen=%08X, totallen=%08X (expected %08X)\n", + codelen, totallen, chklen); + Log(LogLevel::Error, "game=%s, offset=%08X, cheat=%s\n", + entry.Name.c_str(), info.Offset, itemname.c_str()); + + CloseFile(f); + return false; + } + + if ((codelen >= 0x100000) || (codelen & 1)) + { + Log(LogLevel::Error, "AR: unreasonable code length %08X\n", + codelen); + Log(LogLevel::Error, "game=%s, offset=%08X, cheat=%s\n", + entry.Name.c_str(), info.Offset, itemname.c_str()); + + CloseFile(f); + return false; + } + + if (catlen == 0) + { + curcat = &entry.RootCat; + } + + ARCode code; + code.Name = itemname; + code.Description = itemdesc; + code.Enabled = !!(itemflags & (1<<24)); + + u32* rawcode = new u32[codelen]; + FileRead(rawcode, codelen*4, 1, f); + + for (u32 j = 0; j < codelen; j+=2) + { + code.Code.push_back(rawcode[j]); + code.Code.push_back(rawcode[j+1]); + } + + delete[] rawcode; + + curcat->Children.emplace_back(code); + + if (catlen >= 0) + catlen--; + } + } + + for (auto& item : entry.RootCat.Children) + { + if (!std::holds_alternative(item)) + continue; + + auto& cat = std::get(item); + if (!cat.OnlyOneCodeEnabled) + continue; + + // for categories that only allow one code to be enabled: + // make sure we don't have multiple ones enabled + + bool foundone = false; + for (auto& childitem : cat.Children) + { + auto& code = std::get(childitem); + if (!code.Enabled) continue; + if (foundone) + code.Enabled = false; + else + foundone = true; + } + } + + CloseFile(f); + return true; +} + + +std::string ReadNTString(Platform::FileHandle* f) +{ + char tmp[256]; + std::string ret; + int readlen = 0; + u64 startpos = FilePosition(f); + + // TODO might break with UTF8 and shit + + while (!IsEndOfFile(f)) + { + // read 256 bytes of data, see where the string actually ends + memset(tmp, 0, 256); + u64 nread = FileRead(tmp, 1, 256, f); + + bool done = false; + for (int i = 0; i < nread; i++) + { + readlen++; + if (!tmp[i]) { done = true; break; } + ret += tmp[i]; + } + + if (done) + break; + } + + // correct the file position to point right after the end of this string + FileSeek(f, startpos + readlen, FileSeekOrigin::Start); + + return ret; +} + +void AlignFilePos(Platform::FileHandle* f) +{ + u64 pos = FilePosition(f); + if (pos & 3) + FileSeek(f, (pos + 3) & (~3), FileSeekOrigin::Start); +} + +} diff --git a/src/ARDatabaseDAT.h b/src/ARDatabaseDAT.h new file mode 100644 index 0000000000..acf0fb862f --- /dev/null +++ b/src/ARDatabaseDAT.h @@ -0,0 +1,74 @@ +/* + Copyright 2016-2025 melonDS team + + This file is part of melonDS. + + melonDS is free software: you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + melonDS is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with melonDS. If not, see http://www.gnu.org/licenses/. +*/ + +#ifndef ARDATABASEDAT_H +#define ARDATABASEDAT_H + +#include +#include +#include +#include +#include "types.h" +#include "ARCodeFile.h" + +namespace melonDS +{ + +struct ARDatabaseEntry +{ + u32 GameCode; + u32 Checksum; + std::string Name; + ARCodeCat RootCat; +}; + +typedef std::vector ARDatabaseEntryList; + + +class ARDatabaseDAT +{ +public: + ARDatabaseDAT(const std::string& filename); + ~ARDatabaseDAT() noexcept = default; + + bool Error = false; + + std::string GetDBName() const { return DBName; } + bool FindGameCode(u32 gamecode); + ARDatabaseEntryList GetEntriesByGameCode(u32 gamecode); + +private: + std::string Filename; + std::string DBName; + + struct EntryInfo + { + u32 GameCode; + u32 Checksum; + u32 Offset; + }; + + // list of entries per gamecode + std::unordered_map> EntryList; + + bool LoadEntries(); + bool LoadCheatCodes(EntryInfo& info, ARDatabaseEntry& entry); +}; + +} +#endif // ARDATABASEDAT_H diff --git a/src/AREngine.cpp b/src/AREngine.cpp index 8a5b088117..2a5233cbc3 100644 --- a/src/AREngine.cpp +++ b/src/AREngine.cpp @@ -285,8 +285,39 @@ void AREngine::RunCheat(const ARCode& arcode) offset = b; break; - case 0xD4: // datareg += b - datareg += b; + case 0xD4: // data op + switch (a & 0xFF) + { + case 0x00: datareg += b; break; + case 0x01: datareg |= b; break; + case 0x02: datareg &= b; break; + case 0x03: datareg ^= b; break; + + case 0x04: + b &= 0xFF; + if (b > 31) datareg = 0; + else datareg <<= b; + break; + case 0x05: + b &= 0xFF; + if (b > 31) datareg = 0; + else datareg >>= b; + break; + case 0x06: + datareg = ROR(datareg, b & 0x1F); + break; + case 0x07: + b &= 0xFF; + if (b > 31) datareg = ((s32)datareg) >> 31; + else datareg = ((s32)datareg) >> b; + break; + + case 0x08: datareg *= b; break; + + default: + Log(LogLevel::Warn, "!! bad AR D4 opcode %08X %08X\n", a, b); + break; + } break; case 0xD5: // datareg = b diff --git a/src/ARMInterpreter_Branch.cpp b/src/ARMInterpreter_Branch.cpp index df24f20964..0c8b6676b0 100644 --- a/src/ARMInterpreter_Branch.cpp +++ b/src/ARMInterpreter_Branch.cpp @@ -106,11 +106,22 @@ void T_BL_LONG_2(ARM* cpu) { s32 offset = (cpu->CurInstr & 0x7FF) << 1; u32 pc = cpu->R[14] + offset; - cpu->R[14] = (cpu->R[15] - 2) | 1; - if ((cpu->Num==1) || (cpu->CurInstr & (1<<12))) + if ((cpu->Num==1) || (cpu->CurInstr & (1<<12))) // BL + { pc |= 1; + } + else // BLX + { + if (cpu->CurInstr & 1) // lsb of immediate is set, implying halfword offset; this raises undefined. + return T_UNK(cpu); + // instruction always switches to arm mode + // interworking bit should be cleared. + pc &= ~1; + } + + cpu->R[14] = (cpu->R[15] - 2) | 1; cpu->JumpTo(pc); } diff --git a/src/ARMJIT.cpp b/src/ARMJIT.cpp index 39abdd59bb..a9489cac0d 100644 --- a/src/ARMJIT.cpp +++ b/src/ARMJIT.cpp @@ -1161,6 +1161,8 @@ void ARMJIT::JitEnableWrite() noexcept #if defined(__APPLE__) && defined(__aarch64__) if (__builtin_available(macOS 11.0, *)) pthread_jit_write_protect_np(false); + #elif defined(__NetBSD__) + mprotect(JITCompiler.CodeMemBase, ARMJIT_Global::CodeMemorySliceSize, PROT_READ | PROT_WRITE); #endif } @@ -1169,6 +1171,8 @@ void ARMJIT::JitEnableExecute() noexcept #if defined(__APPLE__) && defined(__aarch64__) if (__builtin_available(macOS 11.0, *)) pthread_jit_write_protect_np(true); + #elif defined(__NetBSD__) + mprotect(JITCompiler.CodeMemBase, ARMJIT_Global::CodeMemorySliceSize, PROT_READ | PROT_EXEC); #endif } diff --git a/src/ARMJIT_Global.cpp b/src/ARMJIT_Global.cpp index c9432cbf39..e4aa42f9b3 100644 --- a/src/ARMJIT_Global.cpp +++ b/src/ARMJIT_Global.cpp @@ -25,7 +25,7 @@ std::mutex globalMutex; #define APPLE_AARCH64 #endif -#ifndef APPLE_AARCH64 +#if !defined(APPLE_AARCH64) && !defined(__NetBSD__) && !defined(__OpenBSD__) static constexpr size_t NumCodeMemSlices = 4; static constexpr size_t CodeMemoryAlignedSize = NumCodeMemSlices * CodeMemorySliceSize; @@ -46,7 +46,7 @@ void* AllocateCodeMem() { std::lock_guard guard(globalMutex); -#ifndef APPLE_AARCH64 +#if !defined(APPLE_AARCH64) && !defined(__NetBSD__) && !defined(__OpenBSD__) if (AvailableCodeMemSlices) { int slice = __builtin_ctz(AvailableCodeMemSlices); @@ -61,6 +61,8 @@ void* AllocateCodeMem() return VirtualAlloc(nullptr, CodeMemorySliceSize, MEM_RESERVE|MEM_COMMIT, PAGE_EXECUTE_READWRITE); #elif defined(APPLE_AARCH64) return mmap(NULL, CodeMemorySliceSize, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT,-1, 0); +#elif defined(__NetBSD__) + return mmap(nullptr, CodeMemorySliceSize, PROT_MPROTECT(PROT_READ | PROT_WRITE | PROT_EXEC), MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); #else //printf("mmaping...\n"); return mmap(nullptr, CodeMemorySliceSize, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -71,7 +73,7 @@ void FreeCodeMem(void* codeMem) { std::lock_guard guard(globalMutex); -#ifndef APPLE_AARCH64 +#if !defined(APPLE_AARCH64) && !defined(__NetBSD__) && !defined(__OpenBSD__) for (int i = 0; i < NumCodeMemSlices; i++) { if (codeMem == &GetAlignedCodeMemoryStart()[CodeMemorySliceSize * i]) @@ -100,7 +102,7 @@ void Init() #ifdef _WIN32 DWORD dummy; VirtualProtect(GetAlignedCodeMemoryStart(), CodeMemoryAlignedSize, PAGE_EXECUTE_READWRITE, &dummy); - #elif defined(APPLE_AARCH64) + #elif defined(APPLE_AARCH64) || defined(__NetBSD__) || defined(__OpenBSD__) // Apple aarch64 always uses dynamic allocation #else mprotect(GetAlignedCodeMemoryStart(), CodeMemoryAlignedSize, PROT_EXEC | PROT_READ | PROT_WRITE); diff --git a/src/ARMJIT_Memory.cpp b/src/ARMJIT_Memory.cpp index cc971df014..9c509f2e1e 100644 --- a/src/ARMJIT_Memory.cpp +++ b/src/ARMJIT_Memory.cpp @@ -83,6 +83,8 @@ #define CONTEXT_PC uc_mcontext.mc_rip #elif defined(__NetBSD__) #define CONTEXT_PC uc_mcontext.__gregs[_REG_RIP] + #elif defined(__OpenBSD__) + #define CONTEXT_PC sc_rip #endif #elif defined(__aarch64__) #if defined(_WIN32) @@ -95,6 +97,8 @@ #define CONTEXT_PC uc_mcontext.mc_gpregs.gp_elr #elif defined(__NetBSD__) #define CONTEXT_PC uc_mcontext.__gregs[_REG_PC] + #elif defined(__OpenBSD__) + #define CONTEXT_PC sc_elr #endif #endif @@ -746,7 +750,7 @@ bool ARMJIT_Memory::IsFastMemSupported() #ifdef WATERBOX return false; #endif -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__OpenBSD__) return false; #else static bool initialised = false; @@ -837,7 +841,11 @@ bool ARMJIT_Memory::FaultHandler(FaultDescription& faultDesc, melonDS::NDS& nds) rewriteToSlowPath = !nds.JIT.Memory.MapAtAddress(faultDesc.EmulatedFaultAddr); if (rewriteToSlowPath) + { + nds.JIT.JitEnableWrite(); faultDesc.FaultPC = nds.JIT.JITCompiler.RewriteMemAccess(faultDesc.FaultPC); + nds.JIT.JitEnableExecute(); + } return true; } diff --git a/src/ARMJIT_x64/ARMJIT_Compiler.cpp b/src/ARMJIT_x64/ARMJIT_Compiler.cpp index b52344307b..976b151d49 100644 --- a/src/ARMJIT_x64/ARMJIT_Compiler.cpp +++ b/src/ARMJIT_x64/ARMJIT_Compiler.cpp @@ -228,6 +228,8 @@ Compiler::Compiler(melonDS::NDS& nds) : XEmitter(), NDS(nds) ARMJIT_Global::Init(); CodeMemBase = static_cast(ARMJIT_Global::AllocateCodeMem()); + nds.JIT.JitEnableWrite(); + CodeMemSize = ARMJIT_Global::CodeMemorySliceSize; ResetStart = CodeMemBase; diff --git a/src/ARMJIT_x64/ARMJIT_Linkage.S b/src/ARMJIT_x64/ARMJIT_Linkage.S index 838e0515ce..96968d6a10 100644 --- a/src/ARMJIT_x64/ARMJIT_Linkage.S +++ b/src/ARMJIT_x64/ARMJIT_Linkage.S @@ -20,6 +20,11 @@ #include "ARMJIT_Offsets.h" +// Fixes a stupid bug on clang-cl builds (defines _WIN64 but not _MSC_VER (this one is correct) or WIN64 (this one is MinGW weirdness)) +#if !defined(WIN64) && defined(_WIN64) +#define WIN64 _WIN64 +#endif + .text #define RCPU rbp @@ -105,7 +110,7 @@ ARM_Ret: ret -#if !defined(__APPLE__) && !defined(WIN64) +#if !defined(__APPLE__) && !defined(WIN64) && !defined(_MSC_VER) .section .note.GNU-stack,"",@progbits #endif diff --git a/src/Args.h b/src/Args.h index cc1fc86b4a..7a54175672 100644 --- a/src/Args.h +++ b/src/Args.h @@ -106,6 +106,7 @@ struct NDSArgs AudioBitDepth BitDepth = AudioBitDepth::Auto; AudioInterpolation Interpolation = AudioInterpolation::None; + double OutputSampleRate = 48000.f; /// How the GDB stub should be handled. /// Defaults to disabled. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15daa2e1fc..e85dd42958 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ include(FixInterfaceIncludes) add_library(core STATIC ARCodeFile.cpp + ARDatabaseDAT.cpp AREngine.cpp ARM.cpp ARM_InstrTable.h @@ -72,7 +73,9 @@ add_library(core STATIC sha1/sha1.c tiny-AES-c/aes.c - xxhash/xxhash.c) + xxhash/xxhash.c + + blip-buf/blip_buf.c) if (ENABLE_GDBSTUB) message(NOTICE "Enabling GDB stub") @@ -115,6 +118,10 @@ if (ENABLE_JIT) target_link_libraries(core PRIVATE onecore) endif() + if (CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") + target_link_options(core PUBLIC -Wl,-z,wxneeded) + endif() + if (ARCHITECTURE STREQUAL x86_64) target_sources(core PRIVATE dolphin/x64ABI.cpp @@ -178,6 +185,10 @@ if (NOT MSVC) # (which is fundamentally non-portable, so this is fine) endif() +# Fix xxhash compile errors when using -Og +# See https://github.com/Cyan4973/xxHash/blob/136cc1f8fe4d5ea62a7c16c8424d4fa5158f6d68/README.md?plain=1#L159-L163 +target_compile_definitions(core PRIVATE $<$:XXH_NO_INLINE_HINTS=1>) + find_library(m MATH_LIBRARY) if (MATH_LIBRARY) @@ -195,7 +206,11 @@ endif() if (WIN32) target_link_libraries(core PRIVATE ole32 comctl32 wsock32 ws2_32) - target_compile_definitions(core PUBLIC WIN32_LEAN_AND_MEAN NOMINMAX) + + if (NOT MINGW) + find_path(DIRENT_INCLUDE_DIRS "dirent.h") + target_include_directories(core PUBLIC ${DIRENT_INCLUDE_DIRS}) + endif() elseif(NOT APPLE AND NOT HAIKU) check_library_exists(rt shm_open "" NEED_LIBRT) if (NEED_LIBRT) diff --git a/src/DSi.cpp b/src/DSi.cpp index 1a32932090..dafbbe927b 100644 --- a/src/DSi.cpp +++ b/src/DSi.cpp @@ -421,6 +421,8 @@ void DSi::SetupDirectBoot() u32 cartid = NDSCartSlot.GetCart()->ID(); DSi_TSC* tsc = (DSi_TSC*)SPI.GetTSC(); + // TODO properly setup SCFG_EXT + // TODO: add controls for forcing DS or DSi mode? if (!(header.UnitCode & 0x02)) dsmode = true; @@ -448,6 +450,9 @@ void DSi::SetupDirectBoot() tsc->SetMode(0x00); Set_SCFG_Clock9(0x0000); + + SCFG_EXT[0] &= ~0xC000; + ApplyNewRAMSize(0); } else { @@ -599,8 +604,6 @@ void DSi::SetupDirectBoot() ARM9Write8(0x02FFFDFB, 0x01); } - // TODO: for DS-mode ROMs, switch RAM size here - u32 arm9start = 0; // load the ARM9 secure area @@ -1134,12 +1137,6 @@ void DSi::MapNWRAM_B(u32 num, u8 val) u8 oldval = (MBK[0][mbkn] >> mbks) & 0xFF; if (oldval == val) return; - // TODO REMOVE ME - if ((val&3)==2 || (val&3)==3) - { - //debug(0); - } - JIT.Memory.RemapNWRAM(1); MBK[0][mbkn] &= ~(0xFF << mbks); @@ -1317,6 +1314,19 @@ void DSi::ApplyNewRAMSize(u32 size) Log(LogLevel::Debug, "RAM: 16MB\n"); break; } + + // mirror the RAM size setting to the ARM7 register + SCFG_EXT[1] &= ~0xC000; + SCFG_EXT[1] |= (size << 14); +} + +void DSi::CheckDSiLoaderHack() +{ + if (!(SCFG_EXT[0] & SCFG_DSiLoaderHack)) + return; + + SCFG_EXT[0] &= ~SCFG_DSiLoaderHack; + ApplyNewRAMSize((SCFG_EXT[0] >> 14) & 0x3); } @@ -2591,30 +2601,51 @@ void DSi::ARM9IOWrite32(u32 addr, u32 val) SCFG_EXT[0] &= ~0x8007F19F; SCFG_EXT[0] |= (val & 0x8007F19F); - SCFG_EXT[1] &= ~0x0000F080; - SCFG_EXT[1] |= (val & 0x0000F080); + SCFG_EXT[1] &= ~0x00003080; + SCFG_EXT[1] |= (val & 0x00003080); Log(LogLevel::Debug, "SCFG_EXT = %08X / %08X (val9 %08X)\n", SCFG_EXT[0], SCFG_EXT[1], val); - /*switch ((SCFG_EXT[0] >> 14) & 0x3) + + if (newram != oldram) { - case 0: - case 1: - NDS::MainRAMMask = 0x3FFFFF; - printf("RAM: 4MB\n"); - //baziderp=true; - break; - case 2: - case 3: // TODO: debug console w/ 32MB? - NDS::MainRAMMask = 0xFFFFFF; - printf("RAM: 16MB\n"); - break; - }*/ - // HAX!! - // a change to the RAM size setting is supposed to apply immediately (it does so on hardware) - // however, doing so will cause DS-mode app startup to break, because the change happens while the ARM7 - // is still busy clearing/relocating shit - //if (newram != oldram) - // NDS::ScheduleEvent(NDS::Event_DSi_RAMSizeChange, false, 512*512*512, ApplyNewRAMSize, newram); - Log(LogLevel::Debug, "from %08X, ARM7 %08X, %08X\n", NDS::GetPC(0), NDS::GetPC(1), ARM7.R[1]); + bool isDSiLoader = (ARM9.R[15] == 0x023FEED0) && (IPCSync9 == 0x0505); + if (isDSiLoader) + { + /* + -- DSI LOADER HACK -- + + When loading a title, the DSi loader does the following: (after other steps) + * ARM7 sends IPCSYNC=5 and waits for ARM9 + * ARM9 sends IPCSYNC=5 + * ARM7 clears/moves a bunch of memory regions, then sends IPCSYNC=0 + * ARM9 clears DTCM (clear/move lists are empty, so that's it) + * ARM9 changes RAM size if required (when loading a DS title) + * ARM9 waits for ARM7, then sends IPCSYNC=0 + * the title is booted + + When loading a DS game, the loader crashes, because the ARM9 tries to change the RAM size + while the ARM7 is still clearing/moving RAM, causing it to overwrite the ARM9 code. + + The ARM9 has caches off. The ARM7 has priority over main RAM. + On hardware, the loader works due to main RAM contention: the ARM7-side memory clearing/moving + slows down the ARM9 a lot, because the two are actively competing for main RAM access. + Thus, the ARM9 is only able to change the RAM size after the ARM7 has finished its work. + If EXMEMCNT is changed to give priority to the ARM9, the loader will crash. + + Main RAM contention would be incredibly difficult to emulate without a cycle-accurate emulator + (which melonDS is not). + A possible workaround might be to count main RAM accesses over a time slice and calculate a + penalty to apply to either ARM9 or ARM7. This would be gross and have potential to cause new + timing issues unless it is calibrated precisely. + + In the meantime, this hack will take care of the DSi loader's shoddy programming, hopefully + with no side effects. + */ + + SCFG_EXT[0] |= SCFG_DSiLoaderHack; + } + else + ApplyNewRAMSize(newram); + } UpdateVRAMTimings(); } @@ -3025,6 +3056,12 @@ void DSi::ARM7IOWrite16(u32 addr, u16 val) assert(ConsoleType == 1); switch (addr) { + case 0x04000180: + // DSi loader hack hook + if ((val & 0x0F00) == 0x0000) + CheckDSiLoaderHack(); + return NDS::ARM7IOWrite16(addr, val); + case 0x04000218: NDS::IE2 = (val & 0x7FF7); NDS::UpdateIRQ(1); return; case 0x0400021C: NDS::IF2 &= ~(val & 0x7FF7); NDS::UpdateIRQ(1); return; @@ -3139,6 +3176,12 @@ void DSi::ARM7IOWrite32(u32 addr, u32 val) assert(ConsoleType == 1); switch (addr) { + case 0x04000180: + // DSi loader hack hook + if ((val & 0x0F00) == 0x0000) + CheckDSiLoaderHack(); + return NDS::ARM7IOWrite32(addr, val); + case 0x04000218: NDS::IE2 = (val & 0x7FF7); NDS::UpdateIRQ(1); return; case 0x0400021C: NDS::IF2 &= ~(val & 0x7FF7); NDS::UpdateIRQ(1); return; diff --git a/src/DSi.h b/src/DSi.h index 250d5d3c6f..416453cd4a 100644 --- a/src/DSi.h +++ b/src/DSi.h @@ -42,6 +42,11 @@ namespace DSi_NAND class NANDImage; } +enum +{ + SCFG_DSiLoaderHack = (1<<30), +}; + class DSi final : public NDS { protected: @@ -188,6 +193,7 @@ class DSi final : public NDS void Set_SCFG_MC(u32 val); void DecryptModcryptArea(u32 offset, u32 size, const u8* iv); void ApplyNewRAMSize(u32 size); + void CheckDSiLoaderHack(); }; } diff --git a/src/DSi_Camera.cpp b/src/DSi_Camera.cpp index 55c9c14427..e66bea3eac 100644 --- a/src/DSi_Camera.cpp +++ b/src/DSi_Camera.cpp @@ -319,7 +319,7 @@ u16 DSi_CamModule::Read16(u32 addr) { switch (addr) { - case 0x04004200: return ModuleCnt; + case 0x04004200: return ModuleCnt | (1<<7); case 0x04004202: return Cnt | (Transferring ? (1<<15) : 0); } @@ -366,21 +366,23 @@ void DSi_CamModule::Write16(u32 addr, u16 val) { case 0x04004200: { + if (IsTransferring()) return; + u16 oldcnt = ModuleCnt; ModuleCnt = val; - if ((ModuleCnt & (1<<1)) && !(oldcnt & (1<<1))) - { - // reset shit to zero - // CHECKME + // bit 1 doesn't seem to actually do anything? + // bit 5 controls camera reset - Cnt = 0; - Transferring = false; + if ((!(ModuleCnt & (1<<5))) && (oldcnt & (1<<5))) + { + Camera0->Reset(); + Camera1->Reset(); } - - if ((ModuleCnt & (1<<5)) && !(oldcnt & (1<<5))) + else if ((ModuleCnt & (1<<5)) && (!(oldcnt & (1<<5)))) { - // TODO: reset I2C?? + Camera0->ReleaseReset(); + Camera1->ReleaseReset(); } } return; @@ -483,6 +485,8 @@ void DSi_Camera::Reset() { Platform::Camera_Stop(Num, DSi.UserData); + ResetHeld = true; + DataPos = 0; RegAddr = 0; RegData = 0; @@ -505,6 +509,11 @@ void DSi_Camera::Reset() memset(FrameBuffer, 0, (640*480/2)*sizeof(u32)); } +void DSi_Camera::ReleaseReset() +{ + ResetHeld = false; +} + void DSi_Camera::Stop() { Platform::Camera_Stop(Num, DSi.UserData); @@ -625,6 +634,8 @@ void DSi_Camera::Acquire() u8 DSi_Camera::Read(bool last) { + if (ResetHeld) return 0xFF; + u8 ret; if (DataPos & 0x1) @@ -646,6 +657,8 @@ u8 DSi_Camera::Read(bool last) void DSi_Camera::Write(u8 val, bool last) { + if (ResetHeld) return; + if (DataPos < 2) { if (DataPos == 0) diff --git a/src/DSi_Camera.h b/src/DSi_Camera.h index 4e93e0a8ec..01c51218ff 100644 --- a/src/DSi_Camera.h +++ b/src/DSi_Camera.h @@ -37,6 +37,7 @@ class DSi_Camera : public DSi_I2CDevice void DoSavestate(Savestate* file) override; void Reset() override; + void ReleaseReset(); void Stop(); bool IsActivated() const; @@ -55,6 +56,8 @@ class DSi_Camera : public DSi_I2CDevice u32 Num; private: + bool ResetHeld; + u32 DataPos; u32 RegAddr; u16 RegData; diff --git a/src/DSi_SD.cpp b/src/DSi_SD.cpp index 08f4172725..aab460b776 100644 --- a/src/DSi_SD.cpp +++ b/src/DSi_SD.cpp @@ -287,8 +287,10 @@ void DSi_SDHost::SetCardIRQ() u16 oldflags = CardIRQStatus & ~CardIRQMask; DSi_SDDevice* dev = Ports[PortSelect & 0x1].get(); - if (dev->IRQ) CardIRQStatus |= (1<<0); - else CardIRQStatus &= ~(1<<0); + if (dev && dev->IRQ) + CardIRQStatus |= (1<<0); + else + CardIRQStatus &= ~(1<<0); u16 newflags = CardIRQStatus & ~CardIRQMask; diff --git a/src/FATStorage.cpp b/src/FATStorage.cpp index 7a870d55a3..c93e90b99a 100644 --- a/src/FATStorage.cpp +++ b/src/FATStorage.cpp @@ -192,16 +192,14 @@ u64 FATStorage::GetSectorCount() const ff_disk_read_cb FATStorage::FF_ReadStorage() const noexcept { return [this](BYTE* buf, LBA_t sector, UINT num) { - u32 res = ReadSectorsInternal(File, FileSize, sector, num, buf); - return res / 0x200; + return ReadSectorsInternal(File, FileSize, sector, num, buf); }; } ff_disk_write_cb FATStorage::FF_WriteStorage() const noexcept { return [this](const BYTE* buf, LBA_t sector, UINT num) { - u32 res = WriteSectorsInternal(File, FileSize, sector, num, buf); - return res / 0x200; + return WriteSectorsInternal(File, FileSize, sector, num, buf); }; } @@ -1164,4 +1162,4 @@ bool FATStorage::Save() return true; } -} +} \ No newline at end of file diff --git a/src/FIFO.h b/src/FIFO.h index 159752f80c..62ca883eb5 100644 --- a/src/FIFO.h +++ b/src/FIFO.h @@ -225,7 +225,7 @@ class RingBuffer u32 part1 = Size - WritePos; memcpy(&Buffer[WritePos], data, part1); if (len > part1) - memcpy(Buffer, &((u8*)data)[part1], len - part1); + memcpy(Buffer, &((const u8*)data)[part1], len - part1); WritePos = len - part1; } else diff --git a/src/GPU3D_Compute.cpp b/src/GPU3D_Compute.cpp index 3e89f02a55..eef2bf0f23 100644 --- a/src/GPU3D_Compute.cpp +++ b/src/GPU3D_Compute.cpp @@ -19,6 +19,9 @@ #include "GPU3D_Compute.h" #include +#include + +#include "Utils.h" #include "OpenGLSupport.h" @@ -50,6 +53,14 @@ bool ComputeRenderer::CompileShader(GLuint& shader, const std::string& source, c shaderSource += std::to_string(ScreenHeight); shaderSource += "\n#define MaxWorkTiles "; shaderSource += std::to_string(MaxWorkTiles); + shaderSource += "\n#define TileSize "; + shaderSource += std::to_string(TileSize); + shaderSource += "\nconst int CoarseTileCountY = "; + shaderSource += std::to_string(CoarseTileCountY) + ";"; + shaderSource += "\n#define CoarseTileArea "; + shaderSource += std::to_string(CoarseTileArea); + shaderSource += "\n#define ClearCoarseBinMaskLocalSize "; + shaderSource += std::to_string(ClearCoarseBinMaskLocalSize); shaderSource += ComputeRendererShaders::Common; shaderSource += source; @@ -297,6 +308,8 @@ void ComputeRenderer::Reset(GPU& gpu) void ComputeRenderer::SetRenderSettings(int scale, bool highResolutionCoordinates) { + u8 TileScale; + CurGLCompositor.SetScaleFactor(scale); if (ScaleFactor != -1) @@ -310,6 +323,22 @@ void ComputeRenderer::SetRenderSettings(int scale, bool highResolutionCoordinate ScreenWidth = 256 * ScaleFactor; ScreenHeight = 192 * ScaleFactor; + //Starting at 4.5x we want to double TileSize every time scale doubles + TileScale = 2 * ScaleFactor / 9; + TileScale = GetMSBit(TileScale); + TileScale <<= 1; + TileScale += TileScale == 0; + + std::printf("Scale: %d\n", ScaleFactor); + std::printf("TileScale: %d\n", TileScale); + + TileSize = std::min(8 * TileScale, 32); + CoarseTileCountY = TileSize < 32 ? 4 : 6; + ClearCoarseBinMaskLocalSize = TileSize < 32 ? 64 : 48; + CoarseTileArea = CoarseTileCountX * CoarseTileCountY; + CoarseTileW = CoarseTileCountX * TileSize; + CoarseTileH = CoarseTileCountY * TileSize; + TilesPerLine = ScreenWidth/TileSize; TileLines = ScreenHeight/TileSize; @@ -918,7 +947,7 @@ void ComputeRenderer::RenderFrame(GPU& gpu) glBindBufferBase(GL_UNIFORM_BUFFER, 0, MetaUniformMemory); glUseProgram(ShaderClearCoarseBinMask); - glDispatchCompute(TilesPerLine*TileLines/32, 1, 1); + glDispatchCompute(TilesPerLine*TileLines/ClearCoarseBinMaskLocalSize, 1, 1); bool wbuffer = false; if (numYSpans > 0) @@ -932,23 +961,23 @@ void ComputeRenderer::RenderFrame(GPU& gpu) glBindImageTexture(0, YSpanIndicesTexture, 0, GL_FALSE, 0, GL_READ_ONLY, GL_RGBA16UI); glUseProgram(ShaderInterpXSpans[wbuffer]); glDispatchCompute((numSetupIndices + 31) / 32, 1, 1); - glMemoryBarrier(GL_SHADER_STORAGE_BUFFER); + glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT | GL_COMMAND_BARRIER_BIT); // bin polygons glUseProgram(ShaderBinCombined); glDispatchCompute(((gpu.GPU3D.RenderNumPolygons + 31) / 32), ScreenWidth/CoarseTileW, ScreenHeight/CoarseTileH); - glMemoryBarrier(GL_SHADER_STORAGE_BUFFER); + glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT | GL_COMMAND_BARRIER_BIT); // calculate list offsets glUseProgram(ShaderCalculateWorkListOffset); glDispatchCompute((numVariants + 31) / 32, 1, 1); - glMemoryBarrier(GL_SHADER_STORAGE_BUFFER); + glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT | GL_COMMAND_BARRIER_BIT); // sort shader work glUseProgram(ShaderSortWork); glBindBuffer(GL_DISPATCH_INDIRECT_BUFFER, BinResultMemory); glDispatchComputeIndirect(offsetof(BinResultHeader, SortWorkWorkCount)); - glMemoryBarrier(GL_SHADER_STORAGE_BUFFER); + glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT | GL_COMMAND_BARRIER_BIT); glActiveTexture(GL_TEXTURE0); @@ -1134,4 +1163,4 @@ void ComputeRenderer::Stop(const GPU &gpu) CurGLCompositor.Stop(gpu); } -} \ No newline at end of file +} diff --git a/src/GPU3D_Compute.h b/src/GPU3D_Compute.h index 0429f3aef7..06d60311f1 100644 --- a/src/GPU3D_Compute.h +++ b/src/GPU3D_Compute.h @@ -163,11 +163,13 @@ class ComputeRenderer : public Renderer3D float TextureLayer; }; - static constexpr int TileSize = 8; + int TileSize; static constexpr int CoarseTileCountX = 8; - static constexpr int CoarseTileCountY = 4; - static constexpr int CoarseTileW = CoarseTileCountX * TileSize; - static constexpr int CoarseTileH = CoarseTileCountY * TileSize; + int CoarseTileCountY; + int CoarseTileArea; + int CoarseTileW; + int CoarseTileH; + int ClearCoarseBinMaskLocalSize; static constexpr int BinStride = 2048/32; static constexpr int CoarseBinStride = BinStride/32; diff --git a/src/GPU3D_Compute_shaders.h b/src/GPU3D_Compute_shaders.h index 8b8f6c5a00..dab28a6ea4 100644 --- a/src/GPU3D_Compute_shaders.h +++ b/src/GPU3D_Compute_shaders.h @@ -339,9 +339,7 @@ const uint ResultAttrStart = ResultDepthStart+ScreenWidth*ScreenHeight*2; const char* Common = R"( -#define TileSize 8 const int CoarseTileCountX = 8; -const int CoarseTileCountY = 4; const int CoarseTileW = (CoarseTileCountX * TileSize); const int CoarseTileH = (CoarseTileCountY * TileSize); @@ -848,7 +846,7 @@ void main() const std::string ClearCoarseBinMask = BinningBuffer + R"( -layout (local_size_x = 32) in; +layout (local_size_x = ClearCoarseBinMaskLocalSize) in; void main() { @@ -864,7 +862,7 @@ const std::string BinCombined = XSpanSetupBuffer + WorkDescBuffer + R"( -layout (local_size_x = 32) in; +layout (local_size_x = CoarseTileArea) in; bool BinPolygon(Polygon polygon, ivec2 topLeft, ivec2 botRight) { diff --git a/src/GPU3D_Soft.cpp b/src/GPU3D_Soft.cpp index 93be00537e..e2d7f617fc 100644 --- a/src/GPU3D_Soft.cpp +++ b/src/GPU3D_Soft.cpp @@ -585,7 +585,7 @@ void SoftRenderer::SetupPolygonLeftEdge(SoftRenderer::RendererPolygon* rp, s32 y rp->XL = rp->SlopeL.Setup(polygon->Vertices[rp->CurVL]->FinalPosition[0], polygon->Vertices[rp->NextVL]->FinalPosition[0], polygon->Vertices[rp->CurVL]->FinalPosition[1], polygon->Vertices[rp->NextVL]->FinalPosition[1], - polygon->FinalW[rp->CurVL], polygon->FinalW[rp->NextVL], y); + polygon->FinalW[rp->CurVL], polygon->FinalW[rp->NextVL], y, polygon->WBuffer); } void SoftRenderer::SetupPolygonRightEdge(SoftRenderer::RendererPolygon* rp, s32 y) const @@ -612,7 +612,7 @@ void SoftRenderer::SetupPolygonRightEdge(SoftRenderer::RendererPolygon* rp, s32 rp->XR = rp->SlopeR.Setup(polygon->Vertices[rp->CurVR]->FinalPosition[0], polygon->Vertices[rp->NextVR]->FinalPosition[0], polygon->Vertices[rp->CurVR]->FinalPosition[1], polygon->Vertices[rp->NextVR]->FinalPosition[1], - polygon->FinalW[rp->CurVR], polygon->FinalW[rp->NextVR], y); + polygon->FinalW[rp->CurVR], polygon->FinalW[rp->NextVR], y, polygon->WBuffer); } void SoftRenderer::SetupPolygon(SoftRenderer::RendererPolygon* rp, Polygon* polygon) const @@ -658,8 +658,8 @@ void SoftRenderer::SetupPolygon(SoftRenderer::RendererPolygon* rp, Polygon* poly rp->CurVL = vtop; rp->NextVL = vtop; rp->CurVR = vbot; rp->NextVR = vbot; - rp->XL = rp->SlopeL.SetupDummy(polygon->Vertices[rp->CurVL]->FinalPosition[0]); - rp->XR = rp->SlopeR.SetupDummy(polygon->Vertices[rp->CurVR]->FinalPosition[0]); + rp->XL = rp->SlopeL.SetupDummy(polygon->Vertices[rp->CurVL]->FinalPosition[0], polygon->WBuffer); + rp->XR = rp->SlopeR.SetupDummy(polygon->Vertices[rp->CurVR]->FinalPosition[0], polygon->WBuffer); } else { @@ -718,8 +718,8 @@ void SoftRenderer::RenderShadowMaskScanline(const GPU3D& gpu3d, RendererPolygon* s32 wl = rp->SlopeL.Interp.Interpolate(polygon->FinalW[rp->CurVL], polygon->FinalW[rp->NextVL]); s32 wr = rp->SlopeR.Interp.Interpolate(polygon->FinalW[rp->CurVR], polygon->FinalW[rp->NextVR]); - s32 zl = rp->SlopeL.Interp.InterpolateZ(polygon->FinalZ[rp->CurVL], polygon->FinalZ[rp->NextVL], polygon->WBuffer); - s32 zr = rp->SlopeR.Interp.InterpolateZ(polygon->FinalZ[rp->CurVR], polygon->FinalZ[rp->NextVR], polygon->WBuffer); + s32 zl = rp->SlopeL.Interp.InterpolateZ(polygon->FinalZ[rp->CurVL], polygon->FinalZ[rp->NextVL]); + s32 zr = rp->SlopeR.Interp.InterpolateZ(polygon->FinalZ[rp->CurVR], polygon->FinalZ[rp->NextVR]); // right vertical edges are pushed 1px to the left as long as either: // the left edge slope is not 0, or the span is not 0 pixels wide, and it is not at the leftmost pixel of the screen @@ -804,7 +804,7 @@ void SoftRenderer::RenderShadowMaskScanline(const GPU3D& gpu3d, RendererPolygon* int edge; s32 x = xstart; - Interpolator<0> interpX(xstart, xend+1, wl, wr); + Interpolator<0> interpX(xstart, xend+1, wl, wr, polygon->WBuffer); if (x < 0) x = 0; s32 xlimit; @@ -826,7 +826,7 @@ void SoftRenderer::RenderShadowMaskScanline(const GPU3D& gpu3d, RendererPolygon* interpX.SetX(x); - s32 z = interpX.InterpolateZ(zl, zr, polygon->WBuffer); + s32 z = interpX.InterpolateZ(zl, zr); u32 dstattr = AttrBuffer[pixeladdr]; if (!fnDepthTest(DepthBuffer[pixeladdr], z, dstattr)) @@ -852,7 +852,7 @@ void SoftRenderer::RenderShadowMaskScanline(const GPU3D& gpu3d, RendererPolygon* interpX.SetX(x); - s32 z = interpX.InterpolateZ(zl, zr, polygon->WBuffer); + s32 z = interpX.InterpolateZ(zl, zr); u32 dstattr = AttrBuffer[pixeladdr]; if (!fnDepthTest(DepthBuffer[pixeladdr], z, dstattr)) @@ -878,7 +878,7 @@ void SoftRenderer::RenderShadowMaskScanline(const GPU3D& gpu3d, RendererPolygon* interpX.SetX(x); - s32 z = interpX.InterpolateZ(zl, zr, polygon->WBuffer); + s32 z = interpX.InterpolateZ(zl, zr); u32 dstattr = AttrBuffer[pixeladdr]; if (!fnDepthTest(DepthBuffer[pixeladdr], z, dstattr)) @@ -943,8 +943,8 @@ void SoftRenderer::RenderPolygonScanline(const GPU& gpu, RendererPolygon* rp, s3 s32 wl = rp->SlopeL.Interp.Interpolate(polygon->FinalW[rp->CurVL], polygon->FinalW[rp->NextVL]); s32 wr = rp->SlopeR.Interp.Interpolate(polygon->FinalW[rp->CurVR], polygon->FinalW[rp->NextVR]); - s32 zl = rp->SlopeL.Interp.InterpolateZ(polygon->FinalZ[rp->CurVL], polygon->FinalZ[rp->NextVL], polygon->WBuffer); - s32 zr = rp->SlopeR.Interp.InterpolateZ(polygon->FinalZ[rp->CurVR], polygon->FinalZ[rp->NextVR], polygon->WBuffer); + s32 zl = rp->SlopeL.Interp.InterpolateZ(polygon->FinalZ[rp->CurVL], polygon->FinalZ[rp->NextVL]); + s32 zr = rp->SlopeR.Interp.InterpolateZ(polygon->FinalZ[rp->CurVR], polygon->FinalZ[rp->NextVR]); // right vertical edges are pushed 1px to the left as long as either: // the left edge slope is not 0, or the span is not 0 pixels wide, and it is not at the leftmost pixel of the screen @@ -1054,7 +1054,7 @@ void SoftRenderer::RenderPolygonScanline(const GPU& gpu, RendererPolygon* rp, s3 int edge; s32 x = xstart; - Interpolator<0> interpX(xstart, xend+1, wl, wr); + Interpolator<0> interpX(xstart, xend+1, wl, wr, polygon->WBuffer); if (x < 0) x = 0; s32 xlimit; @@ -1093,7 +1093,7 @@ void SoftRenderer::RenderPolygonScanline(const GPU& gpu, RendererPolygon* rp, s3 interpX.SetX(x); - s32 z = interpX.InterpolateZ(zl, zr, polygon->WBuffer); + s32 z = interpX.InterpolateZ(zl, zr); // if depth test against the topmost pixel fails, test // against the pixel underneath @@ -1189,7 +1189,7 @@ void SoftRenderer::RenderPolygonScanline(const GPU& gpu, RendererPolygon* rp, s3 interpX.SetX(x); - s32 z = interpX.InterpolateZ(zl, zr, polygon->WBuffer); + s32 z = interpX.InterpolateZ(zl, zr); // if depth test against the topmost pixel fails, test // against the pixel underneath @@ -1281,7 +1281,7 @@ void SoftRenderer::RenderPolygonScanline(const GPU& gpu, RendererPolygon* rp, s3 interpX.SetX(x); - s32 z = interpX.InterpolateZ(zl, zr, polygon->WBuffer); + s32 z = interpX.InterpolateZ(zl, zr); // if depth test against the topmost pixel fails, test // against the pixel underneath diff --git a/src/GPU3D_Soft.h b/src/GPU3D_Soft.h index 886df84832..7520f2c3d6 100644 --- a/src/GPU3D_Soft.h +++ b/src/GPU3D_Soft.h @@ -69,16 +69,17 @@ class SoftRenderer : public Renderer3D { public: constexpr Interpolator() {} - constexpr Interpolator(s32 x0, s32 x1, s32 w0, s32 w1) + constexpr Interpolator(s32 x0, s32 x1, s32 w0, s32 w1, bool wbuffer) { - Setup(x0, x1, w0, w1); + Setup(x0, x1, w0, w1, wbuffer); } - constexpr void Setup(s32 x0, s32 x1, s32 w0, s32 w1) + constexpr void Setup(s32 x0, s32 x1, s32 w0, s32 w1, bool wbuffer) { this->x0 = x0; this->x1 = x1; this->xdiff = x1 - x0; + this->wbuffer = wbuffer; // calculate reciprocal for Z interpolation // TODO eventually: use a faster reciprocal function? @@ -123,7 +124,7 @@ class SoftRenderer : public Renderer3D { x -= x0; this->x = x; - if (xdiff != 0 && !linear) + if ((xdiff != 0) && ((!linear) || wbuffer)) { u32 num = (x * w0n) << shift; u32 den = (x * w0d) + ((xdiff-x) * w1d); @@ -157,7 +158,7 @@ class SoftRenderer : public Renderer3D } } - constexpr s32 InterpolateZ(s32 z0, s32 z1, bool wbuffer) const + constexpr s32 InterpolateZ(s32 z0, s32 z1) const { if (xdiff == 0 || z0 == z1) return z0; @@ -212,6 +213,7 @@ class SoftRenderer : public Renderer3D int shift; bool linear; + bool wbuffer; s32 xrecip_z; s32 w0n, w0d, w1d; @@ -226,7 +228,7 @@ class SoftRenderer : public Renderer3D public: constexpr Slope() {} - constexpr s32 SetupDummy(s32 x0) + constexpr s32 SetupDummy(s32 x0, bool wbuffer) { dx = 0; @@ -237,7 +239,7 @@ class SoftRenderer : public Renderer3D Increment = 0; XMajor = false; - Interp.Setup(0, 0, 0, 0); + Interp.Setup(0, 0, 0, 0, wbuffer); Interp.SetX(0); xcov_incr = 0; @@ -245,7 +247,7 @@ class SoftRenderer : public Renderer3D return x0; } - constexpr s32 Setup(s32 x0, s32 x1, s32 y0, s32 y1, s32 w0, s32 w1, s32 y) + constexpr s32 Setup(s32 x0, s32 x1, s32 y0, s32 y1, s32 w0, s32 w1, s32 y, bool wbuffer) { this->x0 = x0; this->y = y; @@ -311,7 +313,7 @@ class SoftRenderer : public Renderer3D s32 x = XVal(); int interpoffset = (Increment >= 0x40000) && (side ^ Negative); - Interp.Setup(y0-interpoffset, y1-interpoffset, w0, w1); + Interp.Setup(y0-interpoffset, y1-interpoffset, w0, w1, wbuffer); Interp.SetX(y); // used for calculating AA coverage diff --git a/src/Mic.cpp b/src/Mic.cpp index 31aea02209..a777fad4ed 100644 --- a/src/Mic.cpp +++ b/src/Mic.cpp @@ -204,13 +204,13 @@ void Mic::Advance(u32 cycles) } // NDS mic input has no explicit start/stop control - // so this counter will turn it off if the TSC AUX input doesn't get sampled for one video frame + // so this counter will turn it off if the TSC AUX input doesn't get sampled for two video frames // other sources also get similar counters, in case we are dealing with stupid code that keeps // turning the mic on and off (hi DSi-mode libnds) for (int i = 0; i < 3; i++) { StopCount[i] += cycles; - if (StopCount[i] >= 560190) + if (StopCount[i] >= 560190*2) DoStop((MicSource)i); } } diff --git a/src/NDS.cpp b/src/NDS.cpp index d32ff08bbf..982f385ff4 100644 --- a/src/NDS.cpp +++ b/src/NDS.cpp @@ -95,7 +95,7 @@ NDS::NDS(NDSArgs&& args, int type, void* userdata) noexcept : ARM7BIOSNative(CRC32(ARM7BIOS.data(), ARM7BIOS.size()) == ARM7BIOSCRC32), ARM9BIOSNative(CRC32(ARM9BIOS.data(), ARM9BIOS.size()) == ARM9BIOSCRC32), JIT(*this, args.JIT), - SPU(*this, args.BitDepth, args.Interpolation), + SPU(*this, args.BitDepth, args.Interpolation, args.OutputSampleRate), Mic(*this), GPU(*this, std::move(args.Renderer3D)), SPI(*this, std::move(args.Firmware)), @@ -221,7 +221,7 @@ void NDS::SetJITArgs(std::optional args) noexcept } else if (args.has_value() != EnableJIT) { // Else if we want to turn the JIT off, and it wasn't already off... - JIT.ResetBlockCache(); + JIT.Reset(); } EnableJIT = args.has_value(); @@ -895,7 +895,7 @@ void NDS::RunSystemSleep(u64 timestamp) for (int i = 0; i < Event_MAX; i++) { if (!mask) break; - if (i == Event_SPU || i == Event_RTC) + if (i == Event_RTC) { if (mask & 0x1) { @@ -905,14 +905,8 @@ void NDS::RunSystemSleep(u64 timestamp) { SchedListMask &= ~(1< args) noexcept {} #endif -private: +protected: void InitTimings(); u32 SchedListMask; u64 SysTimestamp; diff --git a/src/Platform.h b/src/Platform.h index 92aee471a0..c83a007d30 100644 --- a/src/Platform.h +++ b/src/Platform.h @@ -213,6 +213,9 @@ bool IsEndOfFile(FileHandle* file); /// @see fgets bool FileReadLine(char* str, int count, FileHandle* file); +/// @see ftell +u64 FilePosition(FileHandle* file); + /// @see fseek bool FileSeek(FileHandle* file, s64 offset, FileSeekOrigin origin); diff --git a/src/SPU.cpp b/src/SPU.cpp index d510452a0f..3c4aca77ad 100644 --- a/src/SPU.cpp +++ b/src/SPU.cpp @@ -26,6 +26,10 @@ #include "DSi_I2S.h" #include "SPU.h" +#include "blip-buf/blip_buf.h" + +#define INTERNAL_SAMPLE_RATE 16756991.f + namespace melonDS { using Platform::Log; @@ -177,7 +181,7 @@ const std::array InterpSNESGauss = { 0x513, 0x514, 0x514, 0x515, 0x516, 0x516, 0x517, 0x517, 0x517, 0x518, 0x518, 0x518, 0x518, 0x518, 0x519, 0x519 }; -SPU::SPU(melonDS::NDS& nds, AudioBitDepth bitdepth, AudioInterpolation interpolation) : +SPU::SPU(melonDS::NDS& nds, AudioBitDepth bitdepth, AudioInterpolation interpolation, double outputSampleRate) : NDS(nds), Channels { SPUChannel(0, nds, interpolation), @@ -201,34 +205,38 @@ SPU::SPU(melonDS::NDS& nds, AudioBitDepth bitdepth, AudioInterpolation interpola SPUCaptureUnit(0, nds), SPUCaptureUnit(1, nds), }, - Degrade10Bit(bitdepth == AudioBitDepth::_10Bit || (nds.ConsoleType == 1 && bitdepth == AudioBitDepth::Auto)) + AudioLock(Platform::Mutex_Create()), + Degrade10Bit(bitdepth == AudioBitDepth::_10Bit || (nds.ConsoleType == 1 && bitdepth == AudioBitDepth::Auto)), + OutputSampleRate(outputSampleRate), + OutputBuffer(nullptr) { - BlipL = blip_new(4096); - BlipR = blip_new(4096); - blip_set_rates(BlipL, 16756991, 44100); - blip_set_rates(BlipR, 16756991, 44100); - NDS.RegisterEventFuncs(Event_SPU, this, {MakeEventThunk(SPU, Mix)}); ApplyBias = true; + Degrade10Bit = false; + + BlipLeft = blip_new(8192); + BlipRight = blip_new(8192); + + OutputBufferReadPos = 0; + OutputBufferWritePos = 0; SetSampleRate(AudioSampleRate::_32KHz); } SPU::~SPU() { - blip_delete(BlipL); - blip_delete(BlipR); + Platform::Mutex_Free(AudioLock); + AudioLock = nullptr; + blip_delete(BlipLeft); + blip_delete(BlipRight); + NDS.UnregisterEventFuncs(Event_SPU); } void SPU::Reset() { - blip_clear(BlipL); - blip_clear(BlipR); - BlipAccumulate = 0; - LeftLatch = 0; - RightLatch = 0; + InitOutput(); Cnt = 0; MasterVolume = 0; @@ -246,8 +254,16 @@ void SPU::Reset() void SPU::Stop() { - blip_clear(BlipL); - blip_clear(BlipR); + Platform::Mutex_Lock(AudioLock); + memset(OutputBuffer, 0, 2*OutputBufferSize*2); + + blip_clear(BlipLeft); + blip_clear(BlipRight); + BlipTimer = 0; + + OutputBufferReadPos = 0; + OutputBufferWritePos = 0; + Platform::Mutex_Unlock(AudioLock); } void SPU::DoSavestate(Savestate* file) @@ -258,8 +274,6 @@ void SPU::DoSavestate(Savestate* file) file->Var8(&MasterVolume); file->Var16(&Bias); - file->Var8(&OutputSamplePos); - file->Var8(&OutputSampleInc); file->VarArray(OutputLastSamples, sizeof(OutputLastSamples)); file->Var32(&MixInterval); @@ -285,15 +299,12 @@ void SPU::SetSampleRate(AudioSampleRate rate) if (rate == AudioSampleRate::_47KHz) { MixInterval = 704; - OutputSampleInc = 16; } else { MixInterval = 1024; - OutputSampleInc = 11; } - OutputSamplePos = 0; memset(OutputLastSamples, 0, sizeof(OutputLastSamples)); } @@ -989,36 +1000,180 @@ void SPU::Mix(u32 spucycles) output[1] &= 0xFFC0; } - BlipAccumulate += spucycles; - // prevent blip from gaining too many samples - if (BlipAccumulate > 512 * 8192) BlipAccumulate = 512 * 8192; + BlipTimer += spucycles; + if (BlipTimer >= 8191 * 512) + BlipTimer = 8191 * 512; + + if (output[0] != OutputLastSamples[0]) + blip_add_delta(BlipLeft, BlipTimer, (int) output[0] - OutputLastSamples[0]); + if (output[1] != OutputLastSamples[1]) + blip_add_delta(BlipRight, BlipTimer, (int) output[1] - OutputLastSamples[1]); - if (LeftLatch != output[0]) + OutputLastSamples[0] = output[0]; + OutputLastSamples[1] = output[1]; + + NDS.ScheduleEvent(Event_SPU, true, MixInterval, 0, MixInterval >> 1); +} + +void SPU::EndFrame() +{ + blip_end_frame(BlipLeft, BlipTimer); + blip_end_frame(BlipRight, BlipTimer); + BlipTimer = 0; + + int avail = blip_samples_avail(BlipLeft); + s16 temp[avail * 2]; + blip_read_samples(BlipLeft, temp, avail, true); + blip_read_samples(BlipRight, temp + 1, avail, true); + + Platform::Mutex_Lock(AudioLock); + for (int i = 0; i < avail * 2; i += 2) { - s32 diff = output[0] - LeftLatch; - LeftLatch = output[0]; - blip_add_delta(BlipL, BlipAccumulate, diff); + OutputBuffer[OutputBufferWritePos++] = temp[i]; + OutputBuffer[OutputBufferWritePos++] = temp[i+1]; + + OutputBufferWritePos &= ((2*OutputBufferSize)-1); + + if (OutputBufferWritePos == OutputBufferReadPos) + { + // advance the read position too, to avoid losing the entire FIFO + OutputBufferReadPos += 2; + OutputBufferReadPos &= ((2*OutputBufferSize)-1); + } } + Platform::Mutex_Unlock(AudioLock); +} + +void SPU::TrimOutput() +{ + Platform::Mutex_Lock(AudioLock); + const int halflimit = (OutputBufferSize / 2); - if (RightLatch != output[1]) + int readpos = OutputBufferWritePos - (halflimit*2); + if (readpos < 0) readpos += (OutputBufferSize*2); + + OutputBufferReadPos = readpos; + Platform::Mutex_Unlock(AudioLock); +} + +void SPU::DrainOutput() +{ + Platform::Mutex_Lock(AudioLock); + OutputBufferReadPos = 0; + OutputBufferWritePos = 0; + Platform::Mutex_Unlock(AudioLock); +} + +void SPU::InitOutput() +{ + Platform::Mutex_Lock(AudioLock); + + blip_set_rates(BlipLeft, INTERNAL_SAMPLE_RATE * OutputSkew, OutputSampleRate); + blip_set_rates(BlipRight, INTERNAL_SAMPLE_RATE * OutputSkew, OutputSampleRate); + + u32 needSamples = (u32) ceil(INTERNAL_SAMPLE_RATE / 60 / INTERNAL_SAMPLE_RATE * OutputSampleRate); + u32 newBufferSize = 512; + while (newBufferSize < needSamples) + newBufferSize <<= 1; + newBufferSize <<= 1; + + if (newBufferSize != OutputBufferSize) { - s32 diff = output[1] - RightLatch; - RightLatch = output[1]; - blip_add_delta(BlipR, BlipAccumulate, diff); + if (OutputBuffer != nullptr) + free(OutputBuffer); + OutputBuffer = (s16*) malloc(2 * newBufferSize * 2); + OutputBufferSize = newBufferSize; } - NDS.ScheduleEvent(Event_SPU, true, MixInterval, 0, MixInterval >> 1); + memset(OutputBuffer, 0, 2*OutputBufferSize*2); + OutputBufferReadPos = 0; + OutputBufferWritePos = 0; + Platform::Mutex_Unlock(AudioLock); +} + +int SPU::GetOutputSize() const +{ + Platform::Mutex_Lock(AudioLock); + + int ret; + if (OutputBufferWritePos >= OutputBufferReadPos) + ret = OutputBufferWritePos - OutputBufferReadPos; + else + ret = (OutputBufferSize*2) - OutputBufferReadPos + OutputBufferWritePos; + + ret >>= 1; + + Platform::Mutex_Unlock(AudioLock); + return ret; +} + +void SPU::Sync(bool wait) +{ + // this function is currently not used anywhere + // depending on the usage context the thread safety measures could be made + // a lot faster + + // sync to audio output in case the core is running too fast + // * wait=true: wait until enough audio data has been played + // * wait=false: merely skip some audio data to avoid a FIFO overflow + + const int halflimit = (OutputBufferSize / 2); + + if (wait) + { + // TODO: less CPU-intensive wait? + while (GetOutputSize() > halflimit); + } + else if (GetOutputSize() > halflimit) + { + Platform::Mutex_Lock(AudioLock); + + int readpos = OutputBufferWritePos - (halflimit*2); + if (readpos < 0) readpos += (OutputBufferSize*2); + + OutputBufferReadPos = readpos; + + Platform::Mutex_Unlock(AudioLock); + } +} + +int SPU::ReadOutput(s16* data, int samples) +{ + Platform::Mutex_Lock(AudioLock); + if (OutputBufferReadPos == OutputBufferWritePos) + { + Platform::Mutex_Unlock(AudioLock); + return 0; + } + + for (int i = 0; i < samples; i++) + { + *data++ = OutputBuffer[OutputBufferReadPos++]; + *data++ = OutputBuffer[OutputBufferReadPos++]; + OutputBufferReadPos &= ((2*OutputBufferSize)-1); + + if (OutputBufferWritePos == OutputBufferReadPos) + { + Platform::Mutex_Unlock(AudioLock); + return i+1; + } + } + + Platform::Mutex_Unlock(AudioLock); + return samples; +} + +void SPU::SetOutputSampleRate(double rate) +{ + OutputSampleRate = rate; + InitOutput(); } -u32 SPU::ReadOutput(s16* data, u32 maxSamples) +void SPU::SetOutputSkew(double skew) { - blip_end_frame(BlipL, BlipAccumulate); - blip_end_frame(BlipR, BlipAccumulate); - BlipAccumulate = 0; - u32 samples = blip_samples_avail(BlipL); - blip_read_samples(BlipL, data + 0, maxSamples, true); - blip_read_samples(BlipR, data + 1, maxSamples, true); - return std::min(samples, maxSamples); + blip_set_rates(BlipLeft, INTERNAL_SAMPLE_RATE * skew, OutputSampleRate); + blip_set_rates(BlipRight, INTERNAL_SAMPLE_RATE * skew, OutputSampleRate); + OutputSkew = skew; } diff --git a/src/SPU.h b/src/SPU.h index 0e6b1d472c..ef7f895bca 100644 --- a/src/SPU.h +++ b/src/SPU.h @@ -22,10 +22,11 @@ #include "Savestate.h" #include "Platform.h" -#include "blip_buf/blip_buf.h" +struct blip_t; namespace melonDS { + class NDS; class SPU; @@ -224,7 +225,7 @@ class SPUCaptureUnit class SPU { public: - explicit SPU(melonDS::NDS& nds, AudioBitDepth bitdepth, AudioInterpolation interpolation); + explicit SPU(melonDS::NDS& nds, AudioBitDepth bitdepth, AudioInterpolation interpolation, double outputSampleRate); ~SPU(); void Reset(); void DoSavestate(Savestate* file); @@ -244,8 +245,16 @@ class SPU void SetApplyBias(bool enable); void Mix(u32 spucycles); + void EndFrame(); - u32 ReadOutput(s16* data, u32 maxSamples); + void TrimOutput(); + void DrainOutput(); + void InitOutput(); + int GetOutputSize() const; + void Sync(bool wait); + int ReadOutput(s16* data, int samples); + void SetOutputSampleRate(double rate); + void SetOutputSkew(double skew); u8 Read8(u32 addr); u16 Read16(u32 addr); @@ -255,21 +264,23 @@ class SPU void Write32(u32 addr, u32 val); private: + u32 OutputBufferSize = 0; + double OutputSampleRate; + double OutputSkew = 1.0; melonDS::NDS& NDS; - // sample pos/inc are 4-bit fractional - // 32KHz sample rate is 11/16 of 47KHz - // so they don't need to be very precise - u8 OutputSamplePos; - u8 OutputSampleInc; + blip_t* BlipLeft; + blip_t* BlipRight; + int BlipTimer = 0; + + s16* OutputBuffer; + u32 OutputBufferWritePos = 0; + u32 OutputBufferReadPos = 0; s16 OutputLastSamples[2]; u32 MixInterval; - blip_t* BlipL; - blip_t* BlipR; - u32 BlipAccumulate = 0; - s32 LeftLatch = 0, RightLatch = 0; + Platform::Mutex* AudioLock; u16 Cnt = 0; u8 MasterVolume = 0; diff --git a/src/Utils.h b/src/Utils.h index 41e14c86d8..b497cbc9b0 100644 --- a/src/Utils.h +++ b/src/Utils.h @@ -38,6 +38,20 @@ std::pair, u32> PadToPowerOf2(const u8* data, u32 len) noe std::unique_ptr CopyToUnique(const u8* data, u32 len) noexcept; +template +T GetMSBit(T val) +{ + val |= (val >> 1); + val |= (val >> 2); + val |= (val >> 4); + + if constexpr(sizeof(val) > 1) val |= (val >> 8); + if constexpr(sizeof(val) > 2) val |= (val >> 16); + if constexpr(sizeof(val) > 4) val |= (val >> 32); + + return val - (val >> 1); +} + } #endif // MELONDS_UTILS_H diff --git a/src/blip-buf/blip_buf.c b/src/blip-buf/blip_buf.c new file mode 100644 index 0000000000..929b8fc669 --- /dev/null +++ b/src/blip-buf/blip_buf.c @@ -0,0 +1,344 @@ +/* blip_buf $vers. http://www.slack.net/~ant/ */ + +#include "blip_buf.h" + +#include +#include +#include +#include + +/* Library Copyright (C) 2003-2009 Shay Green. This library is free software; +you can redistribute it and/or modify it under the terms of the GNU Lesser +General Public License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. This +library is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +details. You should have received a copy of the GNU Lesser General Public +License along with this module; if not, write to the Free Software Foundation, +Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ + +#if defined (BLARGG_TEST) && BLARGG_TEST + #include "blargg_test.h" +#endif + +/* Equivalent to ULONG_MAX >= 0xFFFFFFFF00000000. +Avoids constants that don't fit in 32 bits. */ +#if ULONG_MAX/0xFFFFFFFF > 0xFFFFFFFF + typedef unsigned long fixed_t; + enum { pre_shift = 32 }; + +#elif defined(ULLONG_MAX) + typedef unsigned long long fixed_t; + enum { pre_shift = 32 }; + +#else + typedef unsigned fixed_t; + enum { pre_shift = 0 }; + +#endif + +enum { time_bits = pre_shift + 20 }; + +static fixed_t const time_unit = (fixed_t) 1 << time_bits; + +enum { bass_shift = 9 }; /* affects high-pass filter breakpoint frequency */ +enum { end_frame_extra = 2 }; /* allows deltas slightly after frame length */ + +enum { half_width = 8 }; +enum { buf_extra = half_width*2 + end_frame_extra }; +enum { phase_bits = 5 }; +enum { phase_count = 1 << phase_bits }; +enum { delta_bits = 15 }; +enum { delta_unit = 1 << delta_bits }; +enum { frac_bits = time_bits - pre_shift }; + +/* We could eliminate avail and encode whole samples in offset, but that would +limit the total buffered samples to blip_max_frame. That could only be +increased by decreasing time_bits, which would reduce resample ratio accuracy. +*/ + +/** Sample buffer that resamples to output rate and accumulates samples +until they're read out */ +struct blip_t +{ + fixed_t factor; + fixed_t offset; + int avail; + int size; + int integrator; +}; + +typedef int buf_t; + +/* probably not totally portable */ +#define SAMPLES( buf ) ((buf_t*) ((buf) + 1)) + +/* Arithmetic (sign-preserving) right shift */ +#define ARITH_SHIFT( n, shift ) \ + ((n) >> (shift)) + +enum { max_sample = +32767 }; +enum { min_sample = -32768 }; + +#define CLAMP( n ) \ + {\ + if ( (short) n != n )\ + n = ARITH_SHIFT( n, 16 ) ^ max_sample;\ + } + +static void check_assumptions( void ) +{ + int n; + + #if INT_MAX < 0x7FFFFFFF || UINT_MAX < 0xFFFFFFFF + #error "int must be at least 32 bits" + #endif + + assert( (-3 >> 1) == -2 ); /* right shift must preserve sign */ + + n = max_sample * 2; + CLAMP( n ); + assert( n == max_sample ); + + n = min_sample * 2; + CLAMP( n ); + assert( n == min_sample ); + + assert( blip_max_ratio <= time_unit ); + assert( blip_max_frame <= (fixed_t) -1 >> time_bits ); +} + +blip_t* blip_new( int size ) +{ + blip_t* m; + assert( size >= 0 ); + + m = (blip_t*) malloc( sizeof *m + (size + buf_extra) * sizeof (buf_t) ); + if ( m ) + { + m->factor = time_unit / blip_max_ratio; + m->size = size; + blip_clear( m ); + check_assumptions(); + } + return m; +} + +void blip_delete( blip_t* m ) +{ + if ( m != NULL ) + { + /* Clear fields in case user tries to use after freeing */ + memset( m, 0, sizeof *m ); + free( m ); + } +} + +void blip_set_rates( blip_t* m, double clock_rate, double sample_rate ) +{ + double factor = time_unit * sample_rate / clock_rate; + m->factor = (fixed_t) factor; + + /* Fails if clock_rate exceeds maximum, relative to sample_rate */ + assert( 0 <= factor - m->factor && factor - m->factor < 1 ); + + /* Avoid requiring math.h. Equivalent to + m->factor = (int) ceil( factor ) */ + if ( m->factor < factor ) + m->factor++; + + /* At this point, factor is most likely rounded up, but could still + have been rounded down in the floating-point calculation. */ +} + +void blip_clear( blip_t* m ) +{ + /* We could set offset to 0, factor/2, or factor-1. 0 is suitable if + factor is rounded up. factor-1 is suitable if factor is rounded down. + Since we don't know rounding direction, factor/2 accommodates either, + with the slight loss of showing an error in half the time. Since for + a 64-bit factor this is years, the halving isn't a problem. */ + + m->offset = m->factor / 2; + m->avail = 0; + m->integrator = 0; + memset( SAMPLES( m ), 0, (m->size + buf_extra) * sizeof (buf_t) ); +} + +int blip_clocks_needed( const blip_t* m, int samples ) +{ + fixed_t needed; + + /* Fails if buffer can't hold that many more samples */ + assert( samples >= 0 && m->avail + samples <= m->size ); + + needed = (fixed_t) samples * time_unit; + if ( needed < m->offset ) + return 0; + + return (needed - m->offset + m->factor - 1) / m->factor; +} + +void blip_end_frame( blip_t* m, unsigned t ) +{ + fixed_t off = t * m->factor + m->offset; + m->avail += off >> time_bits; + m->offset = off & (time_unit - 1); + + /* Fails if buffer size was exceeded */ + assert( m->avail <= m->size ); +} + +int blip_samples_avail( const blip_t* m ) +{ + return m->avail; +} + +static void remove_samples( blip_t* m, int count ) +{ + buf_t* buf = SAMPLES( m ); + int remain = m->avail + buf_extra - count; + m->avail -= count; + + memmove( &buf [0], &buf [count], remain * sizeof buf [0] ); + memset( &buf [remain], 0, count * sizeof buf [0] ); +} + +int blip_read_samples( blip_t* m, short out [], int count, int stereo ) +{ + assert( count >= 0 ); + + if ( count > m->avail ) + count = m->avail; + + if ( count ) + { + int const step = stereo ? 2 : 1; + buf_t const* in = SAMPLES( m ); + buf_t const* end = in + count; + int sum = m->integrator; + do + { + /* Eliminate fraction */ + int s = ARITH_SHIFT( sum, delta_bits ); + + sum += *in++; + + CLAMP( s ); + + *out = s; + out += step; + + /* High-pass filter */ + sum -= s << (delta_bits - bass_shift); + } + while ( in != end ); + m->integrator = sum; + + remove_samples( m, count ); + } + + return count; +} + +/* Things that didn't help performance on x86: + __attribute__((aligned(128))) + #define short int + restrict +*/ + +/* Sinc_Generator( 0.9, 0.55, 4.5 ) */ +static short const bl_step [phase_count + 1] [half_width] = +{ +{ 43, -115, 350, -488, 1136, -914, 5861,21022}, +{ 44, -118, 348, -473, 1076, -799, 5274,21001}, +{ 45, -121, 344, -454, 1011, -677, 4706,20936}, +{ 46, -122, 336, -431, 942, -549, 4156,20829}, +{ 47, -123, 327, -404, 868, -418, 3629,20679}, +{ 47, -122, 316, -375, 792, -285, 3124,20488}, +{ 47, -120, 303, -344, 714, -151, 2644,20256}, +{ 46, -117, 289, -310, 634, -17, 2188,19985}, +{ 46, -114, 273, -275, 553, 117, 1758,19675}, +{ 44, -108, 255, -237, 471, 247, 1356,19327}, +{ 43, -103, 237, -199, 390, 373, 981,18944}, +{ 42, -98, 218, -160, 310, 495, 633,18527}, +{ 40, -91, 198, -121, 231, 611, 314,18078}, +{ 38, -84, 178, -81, 153, 722, 22,17599}, +{ 36, -76, 157, -43, 80, 824, -241,17092}, +{ 34, -68, 135, -3, 8, 919, -476,16558}, +{ 32, -61, 115, 34, -60, 1006, -683,16001}, +{ 29, -52, 94, 70, -123, 1083, -862,15422}, +{ 27, -44, 73, 106, -184, 1152,-1015,14824}, +{ 25, -36, 53, 139, -239, 1211,-1142,14210}, +{ 22, -27, 34, 170, -290, 1261,-1244,13582}, +{ 20, -20, 16, 199, -335, 1301,-1322,12942}, +{ 18, -12, -3, 226, -375, 1331,-1376,12293}, +{ 15, -4, -19, 250, -410, 1351,-1408,11638}, +{ 13, 3, -35, 272, -439, 1361,-1419,10979}, +{ 11, 9, -49, 292, -464, 1362,-1410,10319}, +{ 9, 16, -63, 309, -483, 1354,-1383, 9660}, +{ 7, 22, -75, 322, -496, 1337,-1339, 9005}, +{ 6, 26, -85, 333, -504, 1312,-1280, 8355}, +{ 4, 31, -94, 341, -507, 1278,-1205, 7713}, +{ 3, 35, -102, 347, -506, 1238,-1119, 7082}, +{ 1, 40, -110, 350, -499, 1190,-1021, 6464}, +{ 0, 43, -115, 350, -488, 1136, -914, 5861} +}; + +/* Shifting by pre_shift allows calculation using unsigned int rather than +possibly-wider fixed_t. On 32-bit platforms, this is likely more efficient. +And by having pre_shift 32, a 32-bit platform can easily do the shift by +simply ignoring the low half. */ + +void blip_add_delta( blip_t* m, unsigned time, int delta ) +{ + unsigned fixed = (unsigned) ((time * m->factor + m->offset) >> pre_shift); + buf_t* out = SAMPLES( m ) + m->avail + (fixed >> frac_bits); + + int const phase_shift = frac_bits - phase_bits; + int phase = fixed >> phase_shift & (phase_count - 1); + short const* in = bl_step [phase]; + short const* rev = bl_step [phase_count - phase]; + + int interp = fixed >> (phase_shift - delta_bits) & (delta_unit - 1); + int delta2 = (delta * interp) >> delta_bits; + delta -= delta2; + + /* Fails if buffer size was exceeded */ + assert( out <= &SAMPLES( m ) [m->size + end_frame_extra] ); + + out [0] += in[0]*delta + in[half_width+0]*delta2; + out [1] += in[1]*delta + in[half_width+1]*delta2; + out [2] += in[2]*delta + in[half_width+2]*delta2; + out [3] += in[3]*delta + in[half_width+3]*delta2; + out [4] += in[4]*delta + in[half_width+4]*delta2; + out [5] += in[5]*delta + in[half_width+5]*delta2; + out [6] += in[6]*delta + in[half_width+6]*delta2; + out [7] += in[7]*delta + in[half_width+7]*delta2; + + in = rev; + out [ 8] += in[7]*delta + in[7-half_width]*delta2; + out [ 9] += in[6]*delta + in[6-half_width]*delta2; + out [10] += in[5]*delta + in[5-half_width]*delta2; + out [11] += in[4]*delta + in[4-half_width]*delta2; + out [12] += in[3]*delta + in[3-half_width]*delta2; + out [13] += in[2]*delta + in[2-half_width]*delta2; + out [14] += in[1]*delta + in[1-half_width]*delta2; + out [15] += in[0]*delta + in[0-half_width]*delta2; +} + +void blip_add_delta_fast( blip_t* m, unsigned time, int delta ) +{ + unsigned fixed = (unsigned) ((time * m->factor + m->offset) >> pre_shift); + buf_t* out = SAMPLES( m ) + m->avail + (fixed >> frac_bits); + + int interp = fixed >> (frac_bits - delta_bits) & (delta_unit - 1); + int delta2 = delta * interp; + + /* Fails if buffer size was exceeded */ + assert( out <= &SAMPLES( m ) [m->size + end_frame_extra] ); + + out [7] += delta * delta_unit - delta2; + out [8] += delta2; +} diff --git a/src/blip-buf/blip_buf.h b/src/blip-buf/blip_buf.h new file mode 100644 index 0000000000..0853595efe --- /dev/null +++ b/src/blip-buf/blip_buf.h @@ -0,0 +1,72 @@ +/** \file +Sample buffer that resamples from input clock rate to output sample rate */ + +/* blip_buf $vers */ +#ifndef BLIP_BUF_H +#define BLIP_BUF_H + +#ifdef __cplusplus + extern "C" { +#endif + +/** First parameter of most functions is blip_t*, or const blip_t* if nothing +is changed. */ +typedef struct blip_t blip_t; + +/** Creates new buffer that can hold at most sample_count samples. Sets rates +so that there are blip_max_ratio clocks per sample. Returns pointer to new +buffer, or NULL if insufficient memory. */ +blip_t* blip_new( int sample_count ); + +/** Sets approximate input clock rate and output sample rate. For every +clock_rate input clocks, approximately sample_rate samples are generated. */ +void blip_set_rates( blip_t*, double clock_rate, double sample_rate ); + +enum { /** Maximum clock_rate/sample_rate ratio. For a given sample_rate, +clock_rate must not be greater than sample_rate*blip_max_ratio. */ +blip_max_ratio = 1 << 20 }; + +/** Clears entire buffer. Afterwards, blip_samples_avail() == 0. */ +void blip_clear( blip_t* ); + +/** Adds positive/negative delta into buffer at specified clock time. */ +void blip_add_delta( blip_t*, unsigned int clock_time, int delta ); + +/** Same as blip_add_delta(), but uses faster, lower-quality synthesis. */ +void blip_add_delta_fast( blip_t*, unsigned int clock_time, int delta ); + +/** Length of time frame, in clocks, needed to make sample_count additional +samples available. */ +int blip_clocks_needed( const blip_t*, int sample_count ); + +enum { /** Maximum number of samples that can be generated from one time frame. */ +blip_max_frame = 4000 }; + +/** Makes input clocks before clock_duration available for reading as output +samples. Also begins new time frame at clock_duration, so that clock time 0 in +the new time frame specifies the same clock as clock_duration in the old time +frame specified. Deltas can have been added slightly past clock_duration (up to +however many clocks there are in two output samples). */ +void blip_end_frame( blip_t*, unsigned int clock_duration ); + +/** Number of buffered samples available for reading. */ +int blip_samples_avail( const blip_t* ); + +/** Reads and removes at most 'count' samples and writes them to 'out'. If +'stereo' is true, writes output to every other element of 'out', allowing easy +interleaving of two buffers into a stereo sample stream. Outputs 16-bit signed +samples. Returns number of samples actually read. */ +int blip_read_samples( blip_t*, short out [], int count, int stereo ); + +/** Frees buffer. No effect if NULL is passed. */ +void blip_delete( blip_t* ); + + +/* Deprecated */ +typedef blip_t blip_buffer_t; + +#ifdef __cplusplus + } +#endif + +#endif diff --git a/src/blip-buf/license.txt b/src/blip-buf/license.txt new file mode 100644 index 0000000000..5faba9d48c --- /dev/null +++ b/src/blip-buf/license.txt @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/src/blip-buf/readme.txt b/src/blip-buf/readme.txt new file mode 100644 index 0000000000..6a4aa1c3e8 --- /dev/null +++ b/src/blip-buf/readme.txt @@ -0,0 +1,62 @@ +blip_buf $vers: Band-Limited Audio Buffer +----------------------------------------- +Blip_buf is a small waveform synthesis library meant for use in classic +video game sound chip emulation. It greatly simplifies sound chip +emulation code by handling all the details of resampling. The emulator +merely sets the input clock rate and output sample rate, adds waveforms +by specifying the clock times where their amplitude changes, then reads +the resulting output samples. For a more full-features synthesis +library, get Blip_Buffer. + +* Simple C interface and usage. +* Several code examples, including simple sound chip emulator. +* Uses fast, high-quality band-limited resampling algorithm (BLEP). +* Output is low-pass and high-pass filtered and clamped to 16-bit range. +* Supports mono, stereo, and multi-channel synthesis. + +Author : Shay Green +Website : http://www.slack.net/~ant/ +License : GNU Lesser General Public License (LGPL) +Language: C + + +Getting Started +--------------- +Build the demos by typing "make" at the command-line. If that doesn't +work, manually build a program from demo.c, blip_buf.c, and +wave_writer.c. Run "demo", which should record a square wave sweep to +"out.wav". The others demo advanced topics, and their source code can be +used as a starting point for experimentation (after making any changes, +just type "make" to do any necessary recompilation). + +To build the SDL demo, type "make demo_sdl" at the command-line, or +manually build a program from demo_sdl.c, blip_buf.c, and the SDL +multimedia library. + +See blip_buf.h for reference and blip_buf.txt for documentation. + + +Files +----- +readme.txt Essential information +blip_buf.txt Library documentation +license.txt GNU Lesser General Public License + +makefile Builds demos (type "make" at command-line) + +demo.c Generates square wave sweep +demo_stereo.c Generates stereo sound using two blip buffers +demo_fixed.c Works in fixed-point time rather than clocks +demo_sdl.c Plays sound live using SDL multimedia library +demo_chip.c Emulates sound hardware and plays back log.txt +demo_log.txt Log of sound hardware writes for demo_chip.c +wave_writer.h Simple wave sound file writer used by demos +wave_writer.c + +blip_buf.h Library header and reference (Doxygen-enabled) +blip_buf.c source code + +tests/ Unit tests (build with "make test") + +-- +Shay Green diff --git a/src/debug/GdbCmds.cpp b/src/debug/GdbCmds.cpp index c2b1f9c48d..1730efcf11 100644 --- a/src/debug/GdbCmds.cpp +++ b/src/debug/GdbCmds.cpp @@ -886,14 +886,14 @@ ExecResult GdbStub::Handle_v_MustReplyEmpty(GdbStub* stub, const u8* cmd, ssize_ ExecResult GdbStub::Handle_v_Cont(GdbStub* stub, const u8* cmd, ssize_t len) { - if (len < 1) + if (len < 2) { printf("insufficient length"); stub->RespStr("E01"); return ExecResult::Ok; } - switch (cmd[0]) + switch (cmd[1]) { case 'c': stub->RespStr("OK"); @@ -905,7 +905,7 @@ ExecResult GdbStub::Handle_v_Cont(GdbStub* stub, const u8* cmd, ssize_t len) stub->RespStr("OK"); return ExecResult::MustBreak; default: - printf("invalid continue %c %s\n", cmd[0], cmd); + printf("invalid continue %c %s\n", cmd[1], cmd); stub->RespStr("E01"); return ExecResult::Ok; } diff --git a/src/debug/GdbProto.cpp b/src/debug/GdbProto.cpp index 33f11dd90f..ece2aa5db0 100644 --- a/src/debug/GdbProto.cpp +++ b/src/debug/GdbProto.cpp @@ -8,13 +8,13 @@ #include #include #include -#include #include #include #include #include #ifndef _WIN32 +#include #include #include #include diff --git a/src/debug/GdbStub.cpp b/src/debug/GdbStub.cpp index 3bc54b1d68..b1dcea4daa 100644 --- a/src/debug/GdbStub.cpp +++ b/src/debug/GdbStub.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -20,6 +19,10 @@ #include #include #include +#include +// actually do the define for unix platforms rather than windows so that +// it's clearer what the code does +#define closesocket(x) close(x) #endif @@ -139,11 +142,7 @@ bool GdbStub::Init(int port) err: if (SockFd != 0) { -#ifdef _WIN32 closesocket(SockFd); -#else - close(SockFd); -#endif SockFd = 0; } @@ -153,13 +152,13 @@ bool GdbStub::Init(int port) void GdbStub::Close() { Disconnect(); - if (SockFd > 0) close(SockFd); + if (SockFd > 0) closesocket(SockFd); SockFd = 0; } void GdbStub::Disconnect() { - if (ConnFd > 0) close(ConnFd); + if (ConnFd > 0) closesocket(ConnFd); ConnFd = 0; } @@ -318,7 +317,7 @@ StubState GdbStub::Poll(bool wait) if (WaitAckBlocking(&a, 1000) < 0) { Log(LogLevel::Error, "[GDB] inital handshake: didn't receive inital ack!\n"); - close(ConnFd); + closesocket(ConnFd); ConnFd = 0; return StubState::Disconnect; } @@ -404,7 +403,7 @@ StubState GdbStub::Poll(bool wait) case_gdbp_eof: case ReadResult::Eof: Log(LogLevel::Info, "[GDB] EOF!\n"); - close(ConnFd); + closesocket(ConnFd); ConnFd = 0; return StubState::Disconnect; case ReadResult::CksumErr: diff --git a/src/dolphin/x64CPUDetect.cpp b/src/dolphin/x64CPUDetect.cpp index 2339048646..3b4b3b528c 100644 --- a/src/dolphin/x64CPUDetect.cpp +++ b/src/dolphin/x64CPUDetect.cpp @@ -8,7 +8,9 @@ #include "CPUDetect.h" #include "../types.h" -#ifndef _MSVC_VER +using namespace melonDS; + +#ifndef _MSC_VER #ifdef __FreeBSD__ #include @@ -17,8 +19,6 @@ #include #endif -using namespace melonDS; - static inline void __cpuidex(int info[4], int function_id, int subfunction_id) { #ifdef __FreeBSD__ @@ -40,7 +40,7 @@ static inline void __cpuid(int info[4], int function_id) #endif // ifndef _WIN32 -#ifdef _MSVC_VER +#ifdef _MSC_VER static u64 xgetbv(u32 index) { diff --git a/src/frontend/qt_sdl/CMakeLists.txt b/src/frontend/qt_sdl/CMakeLists.txt index 1afa856f1f..db2d413ea7 100644 --- a/src/frontend/qt_sdl/CMakeLists.txt +++ b/src/frontend/qt_sdl/CMakeLists.txt @@ -11,6 +11,7 @@ set(SOURCES_QT_SDL EmuInstanceAudio.cpp EmuInstanceInput.cpp EmuThread.cpp + CheatImportDialog.cpp CheatsDialog.cpp Config.cpp DateTimeDialog.cpp @@ -109,6 +110,13 @@ if (WIN32) if (MINGW AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") set_property(TARGET melonDS PROPERTY AUTORCC_OPTIONS "--no-zstd") endif() + + if (BUILD_STATIC) + find_package(SDL2 CONFIG COMPONENTS SDL2main) + if (TARGET SDL2::SDL2main) + target_link_libraries(melonDS PRIVATE SDL2::SDL2main) + endif() + endif() elseif (APPLE) if (NOT USE_QT6) find_library(COCOA_LIB Cocoa) @@ -205,10 +213,19 @@ if (WIN32) string(TOLOWER "${ARCHITECTURE}" WIN32_ARCHITECTURE) endif() - configure_file("${CMAKE_SOURCE_DIR}/res/xp.manifest.in" "${CMAKE_BINARY_DIR}/res/xp.manifest") target_link_libraries(melonDS PRIVATE ws2_32 iphlpapi) - set_target_properties(melonDS PROPERTIES LINK_FLAGS_DEBUG "-mconsole") + set_target_properties(melonDS PROPERTIES WIN32_EXECUTABLE $,OFF,ON>) + + if (MINGW) + configure_file("${CMAKE_SOURCE_DIR}/res/xp.manifest.in" "${CMAKE_BINARY_DIR}/res/xp.manifest") + else() + # Work around faad2 unconditionally linking libm even on Windows + if (USE_VCPKG) + add_library(m STATIC) + target_link_libraries(melonDS PUBLIC m) + endif() + endif() endif() if (APPLE) diff --git a/src/frontend/qt_sdl/CheatImportDialog.cpp b/src/frontend/qt_sdl/CheatImportDialog.cpp new file mode 100644 index 0000000000..4ff71b855b --- /dev/null +++ b/src/frontend/qt_sdl/CheatImportDialog.cpp @@ -0,0 +1,366 @@ +/* + Copyright 2016-2025 melonDS team + + This file is part of melonDS. + + melonDS is free software: you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + melonDS is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with melonDS. If not, see http://www.gnu.org/licenses/. +*/ + +#include +#include +#include +#include + +#include "types.h" +#include "Platform.h" +#include "Config.h" +#include "EmuInstance.h" +#include "ARDatabaseDAT.h" + +#include "CheatImportDialog.h" +#include "ui_CheatImportDialog.h" + +using namespace melonDS; +using Platform::Log; +using Platform::LogLevel; + + +CheatImportDialog::CheatImportDialog(QWidget *parent, ARDatabaseDAT* db, u32 gamecode, u32 checksum) +: QDialog(parent), ui(new Ui::CheatImportDialog), database(db), gameCode(gamecode), gameChecksum(checksum) +{ + ui->setupUi(this); + setAttribute(Qt::WA_DeleteOnClose); + + dbEntriesByGameCode = database->GetEntriesByGameCode(gameCode); + assert(!dbEntriesByGameCode.empty()); + + hasChecksumMatches = false; + for (auto& entry : dbEntriesByGameCode) + { + if (entry.Checksum == gameChecksum) + { + hasChecksumMatches = true; + break; + } + } + + if (hasChecksumMatches) + { + ui->chkShowAllMatches->setEnabled(true); + ui->chkShowAllMatches->setCheckState(Qt::Unchecked); + } + else + { + ui->chkShowAllMatches->setEnabled(false); + ui->chkShowAllMatches->setCheckState(Qt::Checked); + } + + ui->chkRemoveOld->setCheckState(Qt::Checked); + + auto dbname = database->GetDBName(); + ui->lblDatabaseName->setText(QString::fromStdString(dbname)); + + ui->gbCodeInfo->hide(); + ui->vlRightPanel->addStretch(); + + QImage blank(64, 64, QImage::Format_Alpha8); + blank.fill(0); + blankIcon = QIcon(QPixmap::fromImage(blank.copy())); + + matchIcon = style()->standardIcon(QStyle::SP_DialogApplyButton); + + updatingImportChk = true; + auto treemodel = new QStandardItemModel(ui->tvCheatList); + ui->tvCheatList->setModel(treemodel); + connect(treemodel, &QStandardItemModel::itemChanged, this, &CheatImportDialog::onCheatEntryModified); + connect(ui->tvCheatList->selectionModel(), &QItemSelectionModel::selectionChanged, this, &CheatImportDialog::onCheatSelectionChanged); + + populateEntryList(); + updatingImportChk = false; +} + +CheatImportDialog::~CheatImportDialog() +{ + delete ui; +} + +ARDatabaseEntry& CheatImportDialog::getImportCheats() +{ + QVariant data = ui->cbEntryList->currentData(); + auto entry = data.value(); + return *entry; +} + +ARCodeEnableMap& CheatImportDialog::getImportEnableMap() +{ + return importEnableMap; +} + +bool CheatImportDialog::getRemoveOldCodes() +{ + return (ui->chkRemoveOld->checkState() == Qt::Checked); +} + +void CheatImportDialog::on_chkShowAllMatches_clicked(bool checked) +{ + populateEntryList(); +} + +void CheatImportDialog::on_cbEntryList_currentIndexChanged(int idx) +{ + populateEntryInfo(); +} + +void CheatImportDialog::onCheatSelectionChanged(const QItemSelection &sel, const QItemSelection &desel) +{ + populateCheatInfo(); +} + +void CheatImportDialog::onCheatEntryModified(QStandardItem* item) +{ + int itemtype = item->data(Qt::UserRole).toInt(); + if (itemtype == 2) + { + // sync up the enable map + auto code = item->data(Qt::UserRole+1).value(); + importEnableMap[code] = (item->checkState() == Qt::Checked); + } + + if (updatingImportChk) return; + updatingImportChk = true; + + if (item->hasChildren()) + { + // if this item is a category, we need to check/uncheck all the children + + Qt::CheckState chk = item->checkState(); + if (chk == Qt::PartiallyChecked) + { + updatingImportChk = false; + return; + } + + int rows = item->rowCount(); + for (int i = 0; i < rows; i++) + item->child(i)->setCheckState(chk); + } + else + { + // if this item is a code, we need to update the check on the parent + + QStandardItem* parent = item->parent(); + QStandardItem* root = ((QStandardItemModel*)ui->tvCheatList->model())->invisibleRootItem(); + if ((!parent) || (parent == root)) + { + updatingImportChk = false; + return; + } + + bool allyes = true; + bool allno = true; + int rows = parent->rowCount(); + for (int i = 0; i < rows; i++) + { + Qt::CheckState chk = parent->child(i)->checkState(); + if (chk != Qt::Checked) allyes = false; + if (chk != Qt::Unchecked) allno = false; + } + + if (allyes) parent->setCheckState(Qt::Checked); + else if (allno) parent->setCheckState(Qt::Unchecked); + else parent->setCheckState(Qt::PartiallyChecked); + } + + updatingImportChk = false; +} + +void CheatImportDialog::populateEntryList() +{ + ui->cbEntryList->clear(); + + bool showall = ui->chkShowAllMatches->isChecked(); + int i = 0, sel = -1; + for (auto& entry : dbEntriesByGameCode) + { + bool crcmatch = (entry.Checksum == gameChecksum); + if ((!showall) && (!crcmatch)) + continue; + + QIcon& icon = crcmatch ? matchIcon : blankIcon; + QString name = QString::fromStdString(entry.Name); + ui->cbEntryList->addItem(icon, name, QVariant::fromValue(&entry)); + + if (crcmatch && (sel == -1)) + sel = i; + i++; + } + + if (sel != -1) + ui->cbEntryList->setCurrentIndex(sel); + + populateEntryInfo(); +} + +void CheatImportDialog::populateEntryInfo() +{ + int idx = ui->cbEntryList->currentIndex(); + if ((idx < 0) || (idx >= ui->cbEntryList->count())) + { + // nothing selected + + ui->gbEntryInfo->hide(); + return; + } + + ui->gbEntryInfo->show(); + + QVariant data = ui->cbEntryList->currentData(); + auto entry = data.value(); + + ui->lblEntryName->setText(QString::fromStdString(entry->Name)); + + char gamecode[5] = {0}; + memcpy(gamecode, &entry->GameCode, 4); + ui->lblEntryGameCode->setText(gamecode); + + auto chksum = QString::asprintf("0x%08X (ROM: 0x%08X)", entry->Checksum, gameChecksum); + ui->lblEntryChecksum->setText(chksum); + + populateCheatList(); +} + +void CheatImportDialog::populateCheatListCat(QStandardItem* parentitem, ARCodeCat& parentcat) +{ + for (auto& item : parentcat.Children) + { + if (std::holds_alternative(item)) + { + auto& cat = std::get(item); + + QString catname = QString::fromStdString(cat.Name); + + auto catitem = new QStandardItem(catname); + parentitem->appendRow(catitem); + + catitem->setData(1, Qt::UserRole); + catitem->setData(QVariant::fromValue(&cat), Qt::UserRole+1); + catitem->setCheckable(true); + catitem->setCheckState(Qt::Checked); + + populateCheatListCat(catitem, cat); + } + else + { + auto& code = std::get(item); + + QString codename = QString::fromStdString(code.Name); + + auto codeitem = new QStandardItem(codename); + parentitem->appendRow(codeitem); + + codeitem->setData(2, Qt::UserRole); + codeitem->setData(QVariant::fromValue(&code), Qt::UserRole+1); + codeitem->setCheckable(true); + codeitem->setCheckState(Qt::Checked); + + importEnableMap[&code] = true; + } + } +} + +void CheatImportDialog::populateCheatList() +{ + updatingImportChk = true; + + auto treemodel = (QStandardItemModel*)ui->tvCheatList->model(); + treemodel->clear(); + importEnableMap.clear(); + + int idx = ui->cbEntryList->currentIndex(); + if ((idx < 0) || (idx >= ui->cbEntryList->count())) + { + updatingImportChk = false; + populateCheatInfo(); + return; + } + + QStandardItem* rootitem = treemodel->invisibleRootItem(); + + QVariant data = ui->cbEntryList->currentData(); + auto entry = data.value(); + populateCheatListCat(rootitem, entry->RootCat); + + updatingImportChk = false; + populateCheatInfo(); +} + +void CheatImportDialog::populateCheatInfo() +{ + auto selmodel = ui->tvCheatList->selectionModel(); + if (!selmodel->hasSelection()) + { + ui->gbCodeInfo->hide(); + return; + } + + auto index = selmodel->selectedIndexes()[0]; + int itemtype = index.data(Qt::UserRole).toInt(); + if (itemtype == 1) + { + // category + + ui->gbCodeInfo->show(); + ui->gbCodeInfo->setTitle("Selected category"); + + auto cat = index.data(Qt::UserRole+1).value(); + QString catname = QString::fromStdString(cat->Name); + QString catdesc = QString::fromStdString(cat->Description); + QString codeenable = cat->OnlyOneCodeEnabled ? "Only one" : "Multiple"; + + ui->lblCodeName->setText(catname); + ui->lblCodeDesc->setText(catdesc); + ui->lblCodeStatusLabel->setText("Code enable:"); + ui->lblCodeStatus->setText(codeenable); + + ui->lblCodeCodeLabel->hide(); + ui->txtCodeCode->hide(); + } + else if (itemtype == 2) + { + // cheat + + ui->gbCodeInfo->show(); + ui->gbCodeInfo->setTitle("Selected code"); + + auto code = index.data(Qt::UserRole+1).value(); + QString codename = QString::fromStdString(code->Name); + QString codedesc = QString::fromStdString(code->Description); + QString codeenable = code->Enabled ? "Enabled" : "Disabled"; + + ui->lblCodeName->setText(codename); + ui->lblCodeDesc->setText(codedesc); + ui->lblCodeStatusLabel->setText("Status:"); + ui->lblCodeStatus->setText(codeenable); + + QString codecode = ""; + for (int i = 0; i < code->Code.size(); i+=2) + { + if (i > 0) codecode += "\n"; + codecode += QString::asprintf("%08X %08X", code->Code[i], code->Code[i+1]); + } + + ui->lblCodeCodeLabel->show(); + ui->txtCodeCode->show(); + ui->txtCodeCode->setPlainText(codecode); + } +} diff --git a/src/frontend/qt_sdl/CheatImportDialog.h b/src/frontend/qt_sdl/CheatImportDialog.h new file mode 100644 index 0000000000..79698754e7 --- /dev/null +++ b/src/frontend/qt_sdl/CheatImportDialog.h @@ -0,0 +1,73 @@ +/* + Copyright 2016-2025 melonDS team + + This file is part of melonDS. + + melonDS is free software: you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + melonDS is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with melonDS. If not, see http://www.gnu.org/licenses/. +*/ + +#ifndef CHEATIMPORTDIALOG_H +#define CHEATIMPORTDIALOG_H + +#include +#include +#include +#include + +#include "ARCodeFile.h" +#include "ARDatabaseDAT.h" + +namespace Ui { class CheatImportDialog; } +class CheatImportDialog; + +class CheatImportDialog : public QDialog +{ + Q_OBJECT + +public: + explicit CheatImportDialog(QWidget* parent, melonDS::ARDatabaseDAT* db, melonDS::u32 gamecode, melonDS::u32 checksum); + ~CheatImportDialog(); + + melonDS::ARDatabaseEntry& getImportCheats(); + melonDS::ARCodeEnableMap& getImportEnableMap(); + bool getRemoveOldCodes(); + +private slots: + void on_chkShowAllMatches_clicked(bool checked); + void on_cbEntryList_currentIndexChanged(int idx); + void onCheatSelectionChanged(const QItemSelection& sel, const QItemSelection& desel); + void onCheatEntryModified(QStandardItem* item); + +private: + Ui::CheatImportDialog* ui; + QIcon blankIcon, matchIcon; + + melonDS::ARDatabaseDAT* database; + melonDS::u32 gameCode; + melonDS::u32 gameChecksum; + + melonDS::ARDatabaseEntryList dbEntriesByGameCode; + bool hasChecksumMatches; + + bool updatingImportChk; + + melonDS::ARCodeEnableMap importEnableMap; + + void populateEntryList(); + void populateEntryInfo(); + void populateCheatListCat(QStandardItem* parentitem, melonDS::ARCodeCat& parentcat); + void populateCheatList(); + void populateCheatInfo(); +}; + +#endif // CHEATIMPORTDIALOG_H diff --git a/src/frontend/qt_sdl/CheatImportDialog.ui b/src/frontend/qt_sdl/CheatImportDialog.ui new file mode 100644 index 0000000000..23093d757f --- /dev/null +++ b/src/frontend/qt_sdl/CheatImportDialog.ui @@ -0,0 +1,385 @@ + + + CheatImportDialog + + + + 0 + 0 + 712 + 555 + + + + Import cheat codes - melonDS + + + + + + Qt::Orientation::Horizontal + + + 2 + + + false + + + + + 4 + + + + + Show all game code matches + + + + + + + + + + Clear cheat file before import + + + + + + + Check the cheat codes to import in the list below. + + + + + + + + 0 + 0 + + + + QAbstractItemView::EditTrigger::NoEditTriggers + + + true + + + + + + + + + 4 + + + + + Database + + + + + + + 0 + 0 + + + + super duper cheat database by epic haxor team + + + Qt::TextFormat::PlainText + + + true + + + + + + + + + + + 0 + 0 + + + + Selected game + + + + + + Name: + + + + + + + + 0 + 0 + + + + Super Mario Odyssey DS Edition + + + Qt::TextFormat::PlainText + + + true + + + + + + + Game code: + + + + + + + FART + + + Qt::TextFormat::PlainText + + + + + + + Checksum: + + + + + + + 0xDEADBEEF + + + Qt::TextFormat::PlainText + + + + + + + + + + + 0 + 0 + + + + Selected code + + + + + + + 0 + 0 + + + + Name: + + + Qt::AlignmentFlag::AlignLeading|Qt::AlignmentFlag::AlignLeft|Qt::AlignmentFlag::AlignTop + + + + + + + + 0 + 0 + + + + super duper code + + + Qt::TextFormat::PlainText + + + true + + + + + + + + 0 + 0 + + + + Description: + + + Qt::AlignmentFlag::AlignLeading|Qt::AlignmentFlag::AlignLeft|Qt::AlignmentFlag::AlignTop + + + + + + + + 0 + 0 + + + + this code is absolutely epic + + + Qt::TextFormat::PlainText + + + true + + + + + + + + 0 + 0 + + + + Status: + + + + + + + + 0 + 0 + + + + enabled + + + Qt::TextFormat::PlainText + + + + + + + + 0 + 0 + + + + Code: + + + Qt::AlignmentFlag::AlignLeading|Qt::AlignmentFlag::AlignLeft|Qt::AlignmentFlag::AlignTop + + + + + + + + 0 + 0 + + + + + Noto Sans Mono + + + + QAbstractScrollArea::SizeAdjustPolicy::AdjustToContents + + + true + + + + + + + + + + + + + + Qt::Orientation::Horizontal + + + QDialogButtonBox::StandardButton::Cancel|QDialogButtonBox::StandardButton::Ok + + + + + + + + + buttonBox + accepted() + CheatImportDialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + CheatImportDialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + diff --git a/src/frontend/qt_sdl/CheatsDialog.cpp b/src/frontend/qt_sdl/CheatsDialog.cpp index 75279a2afd..ec906e7a73 100644 --- a/src/frontend/qt_sdl/CheatsDialog.cpp +++ b/src/frontend/qt_sdl/CheatsDialog.cpp @@ -16,18 +16,19 @@ with melonDS. If not, see http://www.gnu.org/licenses/. */ -#include #include #include #include +#include #include "types.h" #include "Platform.h" -#include "Config.h" #include "EmuInstance.h" +#include "ARDatabaseDAT.h" #include "CheatsDialog.h" #include "ui_CheatsDialog.h" +#include "CheatImportDialog.h" using namespace melonDS; using Platform::Log; @@ -41,71 +42,43 @@ CheatsDialog::CheatsDialog(QWidget* parent) : QDialog(parent), ui(new Ui::Cheats ui->setupUi(this); setAttribute(Qt::WA_DeleteOnClose); + importDB = nullptr; + importDlg = nullptr; + updatingEnableChk = false; + + ui->splitter->setStretchFactor(0, 2); + ui->splitter->setStretchFactor(1, 3); + emuInstance = ((MainWindow*)parent)->getEmuInstance(); + auto rom = emuInstance->getNDS()->NDSCartSlot.GetCart(); + gameCode = rom->GetHeader().GameCodeAsU32(); + gameChecksum = ~CRC32(rom->GetROM(), 0x200, 0); + codeFile = emuInstance->getCheatFile(); - QStandardItemModel* model = new QStandardItemModel(); + auto* model = new CheatListModel(ui->tvCodeList); ui->tvCodeList->setModel(model); connect(model, &QStandardItemModel::itemChanged, this, &CheatsDialog::onCheatEntryModified); connect(ui->tvCodeList->selectionModel(), &QItemSelectionModel::selectionChanged, this, &CheatsDialog::onCheatSelectionChanged); - { - QStandardItem* root = model->invisibleRootItem(); - - for (ARCodeCatList::iterator i = codeFile->Categories.begin(); i != codeFile->Categories.end(); i++) - { - ARCodeCat& cat = *i; - - QStandardItem* catitem = new QStandardItem(QString::fromStdString(cat.Name)); - catitem->setEditable(true); - catitem->setData(QVariant::fromValue(i)); - root->appendRow(catitem); + populateCheatList(); - for (ARCodeList::iterator j = cat.Codes.begin(); j != cat.Codes.end(); j++) - { - ARCode& code = *j; - - QStandardItem* codeitem = new QStandardItem(QString::fromStdString(code.Name)); - codeitem->setEditable(true); - codeitem->setCheckable(true); - codeitem->setCheckState(code.Enabled ? Qt::Checked : Qt::Unchecked); - codeitem->setData(QVariant::fromValue(j)); - catitem->appendRow(codeitem); - } - } - } - - ui->txtCode->setPlaceholderText(""); codeChecker = new ARCodeChecker(ui->txtCode->document()); - - ui->btnNewARCode->setEnabled(false); - ui->btnDeleteCode->setEnabled(false); - ui->txtCode->setEnabled(false); } CheatsDialog::~CheatsDialog() { - QAbstractItemModel* model = ui->tvCodeList->model(); - ui->tvCodeList->setModel(nullptr); - delete model; - delete codeChecker; delete ui; } -void CheatsDialog::on_CheatsDialog_accepted() +void CheatsDialog::done(int r) { codeFile->Save(); - closeDlg(); -} - -void CheatsDialog::on_CheatsDialog_rejected() -{ - codeFile->Load(); - + QDialog::done(r); closeDlg(); } @@ -113,70 +86,72 @@ void CheatsDialog::on_btnNewCat_clicked() { QStandardItem* root = ((QStandardItemModel*)ui->tvCodeList->model())->invisibleRootItem(); + ARCodeCat& parentcat = codeFile->RootCat; + ARCodeCat cat; - cat.Codes.clear(); + cat.Parent = &parentcat; cat.Name = "(new category)"; + cat.OnlyOneCodeEnabled = false; - codeFile->Categories.push_back(cat); - ARCodeCatList::iterator id = codeFile->Categories.end(); id--; + parentcat.Children.emplace_back(cat); + auto catptr = &std::get(parentcat.Children.back()); - QStandardItem* catitem = new QStandardItem(QString::fromStdString(cat.Name)); - catitem->setEditable(true); - catitem->setData(QVariant::fromValue(id)); + auto catitem = new QStandardItem(QString::fromStdString(cat.Name)); + catitem->setDropEnabled(true); + catitem->setData(1, Qt::UserRole); + catitem->setData(QVariant::fromValue(catptr), Qt::UserRole+1); root->appendRow(catitem); ui->tvCodeList->selectionModel()->select(catitem->index(), QItemSelectionModel::ClearAndSelect); - ui->tvCodeList->edit(catitem->index()); + ui->tvCodeList->scrollTo(catitem->index()); + + ui->btnEditCode->click(); } void CheatsDialog::on_btnNewARCode_clicked() { - QModelIndexList indices = ui->tvCodeList->selectionModel()->selectedIndexes(); - if (indices.isEmpty()) - { - // ???? - return; - } - - QStandardItemModel* model = (QStandardItemModel*)ui->tvCodeList->model(); - QStandardItem* item = model->itemFromIndex(indices.first()); - QStandardItem* parentitem; + auto model = (QStandardItemModel*)ui->tvCodeList->model(); + auto selmodel = ui->tvCodeList->selectionModel(); + QStandardItem* parentitem = nullptr; + ARCodeCat* cat; - QVariant data = item->data(); - if (data.canConvert()) - { - parentitem = item; - } - else if (data.canConvert()) - { - parentitem = item->parent(); - } - else + if (selmodel->hasSelection()) { - Log(LogLevel::Warn, "what?? :(\n"); - return; + QModelIndexList indices = selmodel->selectedIndexes(); + QStandardItem *item = model->itemFromIndex(indices.first()); + + int itemtype = item->data(Qt::UserRole).toInt(); + if (itemtype == 1) + parentitem = item; + else if (itemtype == 2) + parentitem = item->parent(); + else + assert(false); } - ARCodeCatList::iterator it_cat = parentitem->data().value(); - ARCodeCat& cat = *it_cat; + if (!parentitem) parentitem = model->invisibleRootItem(); + cat = parentitem->data(Qt::UserRole+1).value(); ARCode code; + code.Parent = cat; code.Name = "(new AR code)"; - code.Enabled = true; - code.Code.clear(); + code.Enabled = false; - cat.Codes.push_back(code); - ARCodeList::iterator id = cat.Codes.end(); id--; + cat->Children.emplace_back(code); + auto codeptr = &std::get(cat->Children.back()); - QStandardItem* codeitem = new QStandardItem(QString::fromStdString(code.Name)); - codeitem->setEditable(true); + auto codeitem = new QStandardItem(QString::fromStdString(code.Name)); codeitem->setCheckable(true); codeitem->setCheckState(code.Enabled ? Qt::Checked : Qt::Unchecked); - codeitem->setData(QVariant::fromValue(id)); + codeitem->setDropEnabled(false); + codeitem->setData(2, Qt::UserRole); + codeitem->setData(QVariant::fromValue(codeptr), Qt::UserRole+1); parentitem->appendRow(codeitem); ui->tvCodeList->selectionModel()->select(codeitem->index(), QItemSelectionModel::ClearAndSelect); - ui->tvCodeList->edit(codeitem->index()); + ui->tvCodeList->scrollTo(codeitem->index()); + + ui->btnEditCode->click(); } void CheatsDialog::on_btnDeleteCode_clicked() @@ -195,127 +170,381 @@ void CheatsDialog::on_btnDeleteCode_clicked() QMessageBox::No); if (res != QMessageBox::Yes) return; - QStandardItemModel* model = (QStandardItemModel*)ui->tvCodeList->model(); + auto model = (QStandardItemModel*)ui->tvCodeList->model(); QStandardItem* item = model->itemFromIndex(indices.first()); - QVariant data = item->data(); - if (data.canConvert()) + QStandardItem* itemparent = item->parent(); + if (!itemparent) itemparent = model->invisibleRootItem(); + + int itemtype = item->data(Qt::UserRole).toInt(); + if (itemtype == 1) { - ARCodeCatList::iterator it_cat = data.value(); + auto cat = item->data(Qt::UserRole+1).value(); - (*it_cat).Codes.clear(); - codeFile->Categories.erase(it_cat); + auto parentcat = cat->Parent; + for (auto it = parentcat->Children.begin(); it != parentcat->Children.end(); it++) + { + auto& childitem = *it; + if (!std::holds_alternative(childitem)) + continue; + if (&std::get(childitem) != cat) + continue; - model->invisibleRootItem()->removeRow(item->row()); + parentcat->Children.erase(it); + break; + } } - else if (data.canConvert()) + else if (itemtype == 2) { - ARCodeList::iterator it_code = data.value(); - ARCodeCatList::iterator it_cat = item->parent()->data().value(); + auto code = item->data(Qt::UserRole+1).value(); - (*it_cat).Codes.erase(it_code); + auto parentcat = code->Parent; + for (auto it = parentcat->Children.begin(); it != parentcat->Children.end(); it++) + { + auto& childitem = *it; + if (!std::holds_alternative(childitem)) + continue; + if (&std::get(childitem) != code) + continue; - item->parent()->removeRow(item->row()); + parentcat->Children.erase(it); + break; + } } + else + assert(false); + + itemparent->removeRow(item->row()); } -void CheatsDialog::onCheatSelectionChanged(const QItemSelection& sel, const QItemSelection& desel) +void CheatsDialog::on_btnImportCheats_clicked() { - QModelIndexList indices = sel.indexes(); - if (indices.isEmpty()) + QString file = QFileDialog::getOpenFileName(this, + "Select cheat database...", + emuDirectory, + "R4 cheat database (*.dat);;Any file (*.*)"); + + if (file.isEmpty()) return; + + importDB = new ARDatabaseDAT(file.toStdString()); + if (importDB->Error) { - ui->btnNewARCode->setEnabled(false); - ui->btnDeleteCode->setEnabled(false); - ui->txtCode->setEnabled(false); - ui->txtCode->setPlaceholderText(""); - ui->txtCode->clear(); + QMessageBox::critical(this, "melonDS", + "Failed to open this cheat database file."); + delete importDB; + return; } - else + + if (!importDB->FindGameCode(gameCode)) { - QStandardItem* item = ((QStandardItemModel*)ui->tvCodeList->model())->itemFromIndex(indices.first()); + QMessageBox::critical(this, "melonDS", + "No cheat codes were found in this database for the current game."); + delete importDB; + return; + } - QVariant data = item->data(); - if (data.canConvert()) - { - ui->btnDeleteCode->setEnabled(true); - ui->txtCode->setEnabled(false); - ui->txtCode->setPlaceholderText(""); - ui->txtCode->clear(); - } - else if (data.canConvert()) - { - ARCode& code = *(data.value()); + importDlg = new CheatImportDialog(this, importDB, gameCode, gameChecksum); + importDlg->open(); + connect(importDlg, &CheatImportDialog::finished, this, &CheatsDialog::onImportCheatsFinished); - ui->btnDeleteCode->setEnabled(true); - ui->txtCode->setEnabled(true); - ui->txtCode->setPlaceholderText("(enter AR code here)"); + importDlg->show(); +} - QString codestr = ""; - for (size_t i = 0; i < code.Code.size(); i += 2) - { - u32 c0 = code.Code[i+0]; - u32 c1 = code.Code[i+1]; - //codestr += QString("%1 %2\n").arg(c0, 8, 16, '0').arg(c1, 8, 16, '0').toUpper(); - codestr += QString::asprintf("%08X %08X\n", c0, c1); - } - ui->txtCode->setPlainText(codestr); - } +void CheatsDialog::onImportCheatsFinished(int res) +{ + if (!importDlg) return; + + if (res == QDialog::Accepted) + { + auto& cheats = importDlg->getImportCheats(); + auto& enablemap = importDlg->getImportEnableMap(); + auto removeold = importDlg->getRemoveOldCodes(); - ui->btnNewARCode->setEnabled(true); + codeFile->Import(cheats, enablemap, removeold); + + populateCheatList(); } + + delete importDB; + importDB = nullptr; + importDlg = nullptr; +} + +void CheatsDialog::onCheatSelectionChanged(const QItemSelection& sel, const QItemSelection& desel) +{ + populateCheatInfo(); } void CheatsDialog::onCheatEntryModified(QStandardItem* item) { - QVariant data = item->data(); - if (data.canConvert()) + if (updatingEnableChk) return; + + auto selmodel = ui->tvCodeList->selectionModel(); + int itemtype = item->data(Qt::UserRole).toInt(); + if (itemtype == 2) { - ARCodeCat& cat = *(data.value()); + auto code = item->data(Qt::UserRole+1).value(); - if (item->text().isEmpty()) + code->Enabled = (item->checkState() == Qt::Checked); + + if (code->Enabled && code->Parent->OnlyOneCodeEnabled) { - QString oldname = QString::fromStdString(cat.Name); - item->setText(oldname.isEmpty() ? "(blank category name?)" : oldname); + // if needed, disable other codes + + updatingEnableChk = true; + + auto parent = item->parent(); + for (int i = 0; i < parent->rowCount(); i++) + { + auto child = parent->child(i); + if (child == item) continue; + + int childtype = child->data(Qt::UserRole).toInt(); + if (childtype == 2) + { + auto childcode = child->data(Qt::UserRole+1).value(); + childcode->Enabled = false; + + child->setCheckState(Qt::Unchecked); + + if (selmodel->hasSelection()) + { + auto index = selmodel->selectedIndexes()[0]; + if (index == child->index()) + { + ui->chkItemOption->setChecked(childcode->Enabled); + } + } + } + } + + updatingEnableChk = false; } - else + + // if this item is selected, reflect the checkbox change + if (selmodel->hasSelection()) { - cat.Name = item->text().toStdString(); + auto index = selmodel->selectedIndexes()[0]; + if (index == item->index()) + { + ui->chkItemOption->setChecked(code->Enabled); + } } } - else if (data.canConvert()) +} + +void CheatsDialog::on_btnEditCode_clicked() +{ + ui->btnNewCat->setEnabled(false); + ui->btnNewARCode->setEnabled(false); + ui->btnImportCheats->setEnabled(false); + ui->tvCodeList->setEnabled(false); + + ui->btnEditCode->hide(); + ui->btnDeleteCode->setEnabled(false); + ui->btnSaveCode->show(); + ui->btnCancelEdit->show(); + + ui->txtItemName->setReadOnly(false); + ui->txtItemDesc->setReadOnly(false); + ui->chkItemOption->setEnabled(true); + ui->txtCode->setReadOnly(false); +} + +void CheatsDialog::on_btnSaveCode_clicked() +{ + auto selmodel = ui->tvCodeList->selectionModel(); + assert(selmodel->hasSelection()); + auto index = selmodel->selectedIndexes()[0]; + + if (ui->txtItemName->text().trimmed().isEmpty()) + { + QMessageBox::critical(this, "melonDS", "Error: no name entered."); + return; + } + + int itemtype = index.data(Qt::UserRole).toInt(); + if (itemtype == 1) { - ARCode& code = *(data.value()); + auto cat = index.data(Qt::UserRole+1).value(); - if (item->text().isEmpty()) - { - QString oldname = QString::fromStdString(code.Name); - item->setText(oldname.isEmpty() ? "(blank code name?)" : oldname); - } - else + cat->Name = ui->txtItemName->text().toStdString(); + cat->Description = ui->txtItemDesc->text().toStdString(); + cat->OnlyOneCodeEnabled = ui->chkItemOption->isChecked(); + + auto tvmodel = (QStandardItemModel*)ui->tvCodeList->model(); + auto tvitem = tvmodel->itemFromIndex(index); + tvitem->setText(ui->txtItemName->text()); + } + else if (itemtype == 2) + { + auto code = index.data(Qt::UserRole+1).value(); + + auto codeconv = convertCodeInput(); + if (codeconv.empty()) { - code.Name = item->text().toStdString(); + QMessageBox::critical(this, "melonDS", "Error: the code entered is empty or invalid."); + return; } - code.Enabled = (item->checkState() == Qt::Checked); + code->Name = ui->txtItemName->text().toStdString(); + code->Description = ui->txtItemDesc->text().toStdString(); + code->Enabled = ui->chkItemOption->isChecked(); + code->Code = codeconv; + + //updatingEnableChk = true; + + auto tvmodel = (QStandardItemModel*)ui->tvCodeList->model(); + auto tvitem = tvmodel->itemFromIndex(index); + tvitem->setText(ui->txtItemName->text()); + tvitem->setCheckState(code->Enabled ? Qt::Checked : Qt::Unchecked); + + //updatingEnableChk = false; } + else + assert(false); + + ui->btnNewCat->setEnabled(true); + ui->btnNewARCode->setEnabled(true); + ui->btnImportCheats->setEnabled(true); + ui->tvCodeList->setEnabled(true); + + ui->btnEditCode->show(); + ui->btnDeleteCode->setEnabled(true); + ui->btnSaveCode->hide(); + ui->btnCancelEdit->hide(); + + populateCheatInfo(); } -void CheatsDialog::on_txtCode_textChanged() +void CheatsDialog::on_btnCancelEdit_clicked() { - QModelIndexList indices = ui->tvCodeList->selectionModel()->selectedIndexes(); - if (indices.isEmpty()) - return; + ui->btnNewCat->setEnabled(true); + ui->btnNewARCode->setEnabled(true); + ui->btnImportCheats->setEnabled(true); + ui->tvCodeList->setEnabled(true); + + ui->btnEditCode->show(); + ui->btnDeleteCode->setEnabled(true); + ui->btnSaveCode->hide(); + ui->btnCancelEdit->hide(); + + populateCheatInfo(); +} + +void CheatsDialog::populateCheatList() +{ + auto model = (CheatListModel*)ui->tvCodeList->model(); + model->clear(); + + QStandardItem* root = model->invisibleRootItem(); + root->setData(0, Qt::UserRole); + root->setData(QVariant::fromValue(&codeFile->RootCat), Qt::UserRole+1); + model->populateCheatListCat(root, codeFile->RootCat); + + populateCheatInfo(); +} + +void CheatsDialog::populateCheatInfo() +{ + auto selmodel = ui->tvCodeList->selectionModel(); + if (!selmodel->hasSelection()) + { + ui->splitter->widget(1)->setEnabled(false); + + ui->btnDeleteCode->show(); + ui->btnDeleteCode->setEnabled(false); + ui->btnEditCode->show(); + ui->btnEditCode->setEnabled(false); + ui->btnSaveCode->hide(); + ui->btnCancelEdit->hide(); + + ui->txtItemName->clear(); + ui->txtItemName->setReadOnly(true); + ui->txtItemDesc->clear(); + ui->txtItemDesc->setReadOnly(true); + + ui->chkItemOption->setText("Enabled"); + ui->chkItemOption->setChecked(false); + ui->chkItemOption->setEnabled(false); + + ui->lblCode->show(); + ui->txtCode->show(); + ui->txtCode->setPlainText(""); + ui->txtCode->setReadOnly(true); - QStandardItem* item = ((QStandardItemModel*)ui->tvCodeList->model())->itemFromIndex(indices.first()); - QVariant data = item->data(); - if (!data.canConvert()) return; + } + + ui->splitter->widget(1)->setEnabled(true); - bool error = false; + auto index = selmodel->selectedIndexes()[0]; + int itemtype = index.data(Qt::UserRole).toInt(); + if (itemtype == 1) + { + auto cat = index.data(Qt::UserRole+1).value(); + + ui->btnDeleteCode->show(); + ui->btnDeleteCode->setEnabled(true); + ui->btnEditCode->show(); + ui->btnEditCode->setEnabled(true); + ui->btnSaveCode->hide(); + ui->btnCancelEdit->hide(); + + ui->txtItemName->setText(QString::fromStdString(cat->Name)); + ui->txtItemName->setReadOnly(true); + ui->txtItemDesc->setText(QString::fromStdString(cat->Description)); + ui->txtItemDesc->setReadOnly(true); + + ui->chkItemOption->setText("Only allow one code enabled"); + ui->chkItemOption->setChecked(cat->OnlyOneCodeEnabled); + ui->chkItemOption->setEnabled(false); + + ui->lblCode->hide(); + ui->txtCode->hide(); + } + else if (itemtype == 2) + { + auto code = index.data(Qt::UserRole+1).value(); + + ui->btnDeleteCode->show(); + ui->btnDeleteCode->setEnabled(true); + ui->btnEditCode->show(); + ui->btnEditCode->setEnabled(true); + ui->btnSaveCode->hide(); + ui->btnCancelEdit->hide(); + + ui->txtItemName->setText(QString::fromStdString(code->Name)); + ui->txtItemName->setReadOnly(true); + ui->txtItemDesc->setText(QString::fromStdString(code->Description)); + ui->txtItemDesc->setReadOnly(true); + + ui->chkItemOption->setText("Enabled"); + ui->chkItemOption->setChecked(code->Enabled); + ui->chkItemOption->setEnabled(false); + + ui->lblCode->show(); + ui->txtCode->show(); + + QString codestr = ""; + for (size_t i = 0; i < code->Code.size(); i += 2) + { + u32 c0 = code->Code[i+0]; + u32 c1 = code->Code[i+1]; + //codestr += QString("%1 %2\n").arg(c0, 8, 16, '0').arg(c1, 8, 16, '0').toUpper(); + codestr += QString::asprintf("%08X %08X\n", c0, c1); + } + ui->txtCode->setPlainText(codestr); + ui->txtCode->setReadOnly(true); + } +} + +std::vector CheatsDialog::convertCodeInput() +{ std::vector codeout; - codeout.reserve(64); - QString text = ui->txtCode->document()->toPlainText(); + QString text = ui->txtCode->toPlainText(); + text = text.replace((QChar)'o', (QChar)'0', Qt::CaseInsensitive); #if QT_VERSION >= QT_VERSION_CHECK(5, 14, 0) QStringList lines = text.split('\n', Qt::SkipEmptyParts); #else @@ -329,15 +558,13 @@ void CheatsDialog::on_txtCode_textChanged() if (line.length() > 17) { - error = true; - break; + return {}; } QStringList numbers = line.split(' '); if (numbers.length() != 2) { - error = true; - break; + return {}; } QStringList::iterator jt = numbers.begin(); @@ -352,32 +579,281 @@ void CheatsDialog::on_txtCode_textChanged() if (!c0good || !c1good) { - error = true; - break; + return {}; } codeout.push_back(c0); codeout.push_back(c1); } - ui->btnNewCat->setEnabled(!error); - ui->btnNewARCode->setEnabled(!error); - ui->btnDeleteCode->setEnabled(!error); - ui->tvCodeList->setEnabled(!error); - ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(!error); + return codeout; +} + + +void CheatListModel::populateCheatListCat(QStandardItem* parentitem, ARCodeCat& parentcat) +{ + for (auto& item : parentcat.Children) + { + if (std::holds_alternative(item)) + { + auto& cat = std::get(item); + + auto catitem = new QStandardItem(QString::fromStdString(cat.Name)); + catitem->setDropEnabled(true); + catitem->setData(1, Qt::UserRole); + catitem->setData(QVariant::fromValue(&cat), Qt::UserRole+1); + parentitem->appendRow(catitem); + + populateCheatListCat(catitem, cat); + } + else + { + auto& code = std::get(item); + + auto codeitem = new QStandardItem(QString::fromStdString(code.Name)); + codeitem->setCheckable(true); + codeitem->setCheckState(code.Enabled ? Qt::Checked : Qt::Unchecked); + codeitem->setDropEnabled(false); + codeitem->setData(2, Qt::UserRole); + codeitem->setData(QVariant::fromValue(&code), Qt::UserRole+1); + parentitem->appendRow(codeitem); + } + } +} + +QMimeData* CheatListModel::mimeData(const QModelIndexList& indexes) const +{ + QMimeData* mime = new QMimeData(); + QByteArray array; + + auto& index = indexes[0]; + int itemtype = index.data(Qt::UserRole).toInt(); + quintptr itemptr; + + auto parentlist = (QTreeView*)this->parent(); + unsigned char flags = (unsigned char)itemtype; + if (parentlist->isExpanded(index)) + flags |= 0x10; + + array.append((char)flags); + if (itemtype == 1) + { + auto cat = index.data(Qt::UserRole+1).value(); + itemptr = (quintptr)cat; + + } + else if (itemtype == 2) + { + auto code = index.data(Qt::UserRole+1).value(); + itemptr = (quintptr)code; + } + + for (int i = 0; i < sizeof(quintptr); i++) + { + array.append((char)(itemptr & 0xFF)); + itemptr >>= 8; + } + + mime->setData("application/x-qabstractitemmodeldatalist", array); + return mime; +} + +bool CheatListModel::canDropMimeData(const QMimeData* mime, Qt::DropAction action, int row, int column, const QModelIndex& parent) const +{ + if (action != Qt::MoveAction) + return false; + + QByteArray data = mime->data("application/x-qabstractitemmodeldatalist"); + if (data.length() != (1 + sizeof(quintptr))) + return false; + + int flags = (unsigned char)data[0]; + int itemtype = flags & 0xF; + + QStandardItem* parentitem = itemFromIndex(parent); + if (!parentitem) parentitem = invisibleRootItem(); + + if (itemtype == 1) + { + // if we are trying to drop a category into another category: reject this move + if (parentitem != invisibleRootItem()) + return false; + } + + return true; +} + +bool CheatListModel::dropMimeData(const QMimeData* mime, Qt::DropAction action, int row, int column, const QModelIndex& parent) +{ + if (action != Qt::MoveAction) + return false; + + QByteArray data = mime->data("application/x-qabstractitemmodeldatalist"); + if (data.length() != (1 + sizeof(quintptr))) + return false; + + int flags = (unsigned char)data[0]; + int itemtype = flags & 0xF; + + quintptr itemptr = 0; + for (int i = sizeof(quintptr) - 1; i >= 0; i--) + { + itemptr <<= 8; + itemptr |= (unsigned char)data[1 + i]; + } + + QStandardItem* parentitem = itemFromIndex(parent); + if (!parentitem) parentitem = invisibleRootItem(); + if (row < 0) row = parentitem->rowCount(); + + auto newparent = parentitem->data(Qt::UserRole+1).value(); + QStandardItem* newitem; + + if (itemtype == 1) + { + if (parentitem != invisibleRootItem()) + return false; + + ARCodeCat* cat = (ARCodeCat*)itemptr; + auto oldparent = cat->Parent; + + // retrieve iterator to insert new category at + auto newit = newparent->Children.begin(); + for (int i = 0; newit != newparent->Children.end(); i++) + { + if (i == row) break; + newit++; + } + + // create new category + ARCodeCat newcat = { + .Parent = newparent, + .Name = cat->Name, + .Description = cat->Description, + .OnlyOneCodeEnabled = cat->OnlyOneCodeEnabled, + .Children = cat->Children + }; + + // delete category from old parent + for (auto it = oldparent->Children.begin(); it != oldparent->Children.end(); it++) + { + auto& childitem = *it; + if (!std::holds_alternative(childitem)) + continue; + if (&std::get(childitem) != cat) + continue; + + oldparent->Children.erase(it); + break; + } + + // add it to new parent + newit = newparent->Children.emplace(newit, newcat); + ARCodeCat* newcatptr = &std::get(*newit); + + // reparent the children + for (auto& childitem : newcatptr->Children) + { + if (std::holds_alternative(childitem)) + { + auto& thiscat = std::get(childitem); + thiscat.Parent = newcatptr; + } + else + { + auto& thiscode = std::get(childitem); + thiscode.Parent = newcatptr; + } + } + + // create and add new item + auto catitem = new QStandardItem(QString::fromStdString(newcatptr->Name)); + catitem->setDropEnabled(true); + catitem->setData(1, Qt::UserRole); + catitem->setData(QVariant::fromValue(newcatptr), Qt::UserRole+1); + + parentitem->insertRow(row, catitem); + newitem = catitem; + + populateCheatListCat(catitem, *newcatptr); + } + else if (itemtype == 2) + { + ARCode* code = (ARCode*)itemptr; + auto oldparent = code->Parent; + + // retrieve iterator to insert new code at + auto newit = newparent->Children.begin(); + for (int i = 0; newit != newparent->Children.end(); i++) + { + if (i == row) break; + newit++; + } + + // create new code + ARCode newcode = { + .Parent = newparent, + .Name = code->Name, + .Description = code->Description, + .Enabled = code->Enabled, + .Code = code->Code + }; + + // delete code from old parent + for (auto it = oldparent->Children.begin(); it != oldparent->Children.end(); it++) + { + auto& childitem = *it; + if (!std::holds_alternative(childitem)) + continue; + if (&std::get(childitem) != code) + continue; + + oldparent->Children.erase(it); + break; + } + + // add it to new parent + newit = newparent->Children.emplace(newit, newcode); + ARCode* newcodeptr = &std::get(*newit); - if (error) return; + // create and add new item + auto codeitem = new QStandardItem(QString::fromStdString(newcodeptr->Name)); + codeitem->setCheckable(true); + codeitem->setCheckState(newcodeptr->Enabled ? Qt::Checked : Qt::Unchecked); + codeitem->setDropEnabled(false); + codeitem->setData(2, Qt::UserRole); + codeitem->setData(QVariant::fromValue(newcodeptr), Qt::UserRole+1); - ARCode& code = *(data.value()); - code.Code = codeout; + parentitem->insertRow(row, codeitem); + newitem = codeitem; + } + else + return false; + + auto parentlist = (QTreeView*)this->parent(); + auto newindex = newitem->index(); + if (flags & 0x10) + parentlist->expand(newindex); + + // quality of life: select the item after it's been moved + // we have to add a small delay here + // if we do this immediately, it will cause the dragdrop operation to fail + QTimer::singleShot(1, [=]() + { + parentlist->selectionModel()->select(newindex, QItemSelectionModel::ClearAndSelect); + parentlist->scrollTo(newindex); + }); + + return true; } + void ARCodeChecker::highlightBlock(const QString& text) { QTextCharFormat errformat; errformat.setForeground(Qt::red); { - QRegularExpression expr("^\\s*[0-9A-Fa-f]{1,8} [0-9A-Fa-f]{1,8}\\s*$"); + QRegularExpression expr("^\\s*[0-9A-Fa-foO]{1,8} [0-9A-Fa-foO]{1,8}\\s*$"); QRegularExpressionMatchIterator it = expr.globalMatch(text); if (!it.hasNext()) { diff --git a/src/frontend/qt_sdl/CheatsDialog.h b/src/frontend/qt_sdl/CheatsDialog.h index 9f666b118e..6c0902495d 100644 --- a/src/frontend/qt_sdl/CheatsDialog.h +++ b/src/frontend/qt_sdl/CheatsDialog.h @@ -27,14 +27,34 @@ #include "ARCodeFile.h" -Q_DECLARE_METATYPE(melonDS::ARCodeList::iterator) -Q_DECLARE_METATYPE(melonDS::ARCodeCatList::iterator) - namespace Ui { class CheatsDialog; } class CheatsDialog; +class CheatImportDialog; class EmuInstance; +namespace melonDS +{ + class ARDatabaseDAT; +} + +class CheatListModel : public QStandardItemModel +{ + Q_OBJECT + +public: + CheatListModel(QObject* parent = nullptr) : QStandardItemModel(parent) {} + + void populateCheatListCat(QStandardItem* parentitem, melonDS::ARCodeCat& parentcat); + + Qt::DropActions supportedDragActions() const override { return Qt::MoveAction; } + Qt::DropActions supportedDropActions() const override { return Qt::MoveAction; } + + QMimeData* mimeData(const QModelIndexList& indexes) const override; + bool canDropMimeData(const QMimeData* mime, Qt::DropAction action, int row, int column, const QModelIndex& parent) const override; + bool dropMimeData(const QMimeData* mime, Qt::DropAction action, int row, int column, const QModelIndex& parent) override; +}; + class ARCodeChecker : public QSyntaxHighlighter { Q_OBJECT @@ -74,25 +94,40 @@ class CheatsDialog : public QDialog } private slots: - void on_CheatsDialog_accepted(); - void on_CheatsDialog_rejected(); + void done(int r) override; void on_btnNewCat_clicked(); void on_btnNewARCode_clicked(); void on_btnDeleteCode_clicked(); + void on_btnImportCheats_clicked(); + void onImportCheatsFinished(int res); void onCheatSelectionChanged(const QItemSelection& sel, const QItemSelection& desel); void onCheatEntryModified(QStandardItem* item); - void on_txtCode_textChanged(); + void on_btnEditCode_clicked(); + void on_btnSaveCode_clicked(); + void on_btnCancelEdit_clicked(); private: Ui::CheatsDialog* ui; EmuInstance* emuInstance; + melonDS::u32 gameCode; + melonDS::u32 gameChecksum; + melonDS::ARCodeFile* codeFile; ARCodeChecker* codeChecker; + + melonDS::ARDatabaseDAT* importDB; + CheatImportDialog* importDlg; + + bool updatingEnableChk; + + void populateCheatList(); + void populateCheatInfo(); + std::vector convertCodeInput(); }; #endif // CHEATSDIALOG_H diff --git a/src/frontend/qt_sdl/CheatsDialog.ui b/src/frontend/qt_sdl/CheatsDialog.ui index d0bc586a37..5445b283f5 100644 --- a/src/frontend/qt_sdl/CheatsDialog.ui +++ b/src/frontend/qt_sdl/CheatsDialog.ui @@ -6,101 +6,252 @@ 0 0 - 609 - 417 + 711 + 511 Cheat code editor - melonDS - + - - - QLayout::SetDefaultConstraint + + + Qt::Orientation::Horizontal - - - - New category - - - - - - - New AR code - - - - - - - Delete - - - - - - - Qt::Horizontal - - - QSizePolicy::Expanding - - - - 40 - 20 - - - - - - - - - - - - - 2 - 0 - + + 2 + + + + + 0 - - - 200 - 0 - + + 4 - - true + + 0 - - - - - - - 3 - 0 - + + + + QLayout::SizeConstraint::SetDefaultConstraint + + + + + New category + + + + + + + New AR code + + + + + + + Import... + + + + + + + Qt::Orientation::Horizontal + + + QSizePolicy::Policy::Expanding + + + + 40 + 20 + + + + + + + + + + + 0 + 0 + + + + + 200 + 0 + + + + QAbstractItemView::EditTrigger::NoEditTriggers + + + true + + + QAbstractItemView::DragDropMode::InternalMove + + + Qt::DropAction::IgnoreAction + + + true + + + + + + + + + 4 - - - 200 - 0 - + + 0 - - - + + + + + + Edit + + + + + + + Save + + + + + + + Cancel edit + + + + + + + Qt::Orientation::Horizontal + + + + 40 + 20 + + + + + + + + Delete + + + + + + + + + + + Name: + + + + + + + + + + Description: + + + + + + + + + + + + + + + + + CheckBox + + + + + + + + 0 + 0 + + + + Code: + + + Qt::AlignmentFlag::AlignLeading|Qt::AlignmentFlag::AlignLeft|Qt::AlignmentFlag::AlignTop + + + + + + + + 0 + 0 + + + + + 200 + 0 + + + + + Noto Sans Mono + + + + + + + + + + + + 0 + 0 + + - QDialogButtonBox::Cancel|QDialogButtonBox::Ok + QDialogButtonBox::StandardButton::Close @@ -115,12 +266,12 @@ accept() - 304 - 396 + 649 + 23 - 304 - 208 + 349 + 230 @@ -131,12 +282,12 @@ reject() - 304 - 396 + 649 + 23 - 304 - 208 + 349 + 230 diff --git a/src/frontend/qt_sdl/EmuInstance.cpp b/src/frontend/qt_sdl/EmuInstance.cpp index 51dfaa6d16..d6d662fa93 100755 --- a/src/frontend/qt_sdl/EmuInstance.cpp +++ b/src/frontend/qt_sdl/EmuInstance.cpp @@ -99,6 +99,7 @@ EmuInstance::EmuInstance(int inst) : deleting(false), targetFPS = 60.0; } else targetFPS = val; + curFPS = targetFPS; val = globalCfg.GetDouble("FastForwardFPS"); if (val == 0.0) @@ -1318,6 +1319,7 @@ bool EmuInstance::updateConsole() noexcept jitargs, static_cast(globalCfg.GetInt("Audio.BitDepth")), static_cast(globalCfg.GetInt("Audio.Interpolation")), + (double) audioFreq, gdbargs, }; NDSArgs* args = &ndsargs; diff --git a/src/frontend/qt_sdl/EmuInstance.h b/src/frontend/qt_sdl/EmuInstance.h index c8e756ea6e..f143496573 100755 --- a/src/frontend/qt_sdl/EmuInstance.h +++ b/src/frontend/qt_sdl/EmuInstance.h @@ -232,7 +232,6 @@ class EmuInstance void setupMicInputData(); int audioGetNumSamplesOut(int outlen); - void audioResample(melonDS::s16* inbuf, int inlen, melonDS::s16* outbuf, int outlen, int volume); static void audioCallback(void* data, Uint8* stream, int len); int micGetNumSamplesIn(int inlen); diff --git a/src/frontend/qt_sdl/EmuInstanceAudio.cpp b/src/frontend/qt_sdl/EmuInstanceAudio.cpp index 0ada9388b1..40656de47f 100644 --- a/src/frontend/qt_sdl/EmuInstanceAudio.cpp +++ b/src/frontend/qt_sdl/EmuInstanceAudio.cpp @@ -26,6 +26,7 @@ using namespace melonDS; +#define INTERNAL_FRAME_RATE 59.8260982880808f // --- AUDIO OUTPUT ----------------------------------------------------------- @@ -40,7 +41,8 @@ void EmuInstance::audioInit() audioSyncLock = SDL_CreateMutex(); audioFreq = 48000; // TODO: make both of these configurable? - audioBufSize = 1024; + audioBufSize = 512; + SDL_AudioSpec whatIwant, whatIget; memset(&whatIwant, 0, sizeof(SDL_AudioSpec)); whatIwant.freq = audioFreq; @@ -49,7 +51,7 @@ void EmuInstance::audioInit() whatIwant.samples = audioBufSize; whatIwant.callback = audioCallback; whatIwant.userdata = this; - audioDevice = SDL_OpenAudioDevice(NULL, 0, &whatIwant, &whatIget, SDL_AUDIO_ALLOW_FREQUENCY_CHANGE | SDL_AUDIO_ALLOW_SAMPLES_CHANGE); + audioDevice = SDL_OpenAudioDevice(NULL, 0, &whatIwant, &whatIget, SDL_AUDIO_ALLOW_FREQUENCY_CHANGE); if (!audioDevice) { Platform::Log(Platform::LogLevel::Error, "Audio init failed: %s\n", SDL_GetError()); @@ -126,7 +128,7 @@ void EmuInstance::audioSync() if (audioDevice) { SDL_LockMutex(audioSyncLock); - while (nds->SPU.GetOutputSize() > audioBufSize) + while (nds->SPU.GetOutputSize() >= audioFreq / INTERNAL_FRAME_RATE) { int ret = SDL_CondWaitTimeout(audioSyncCond, audioSyncLock, 500); if (ret == SDL_MUTEX_TIMEDOUT) break; @@ -137,8 +139,7 @@ void EmuInstance::audioSync() int EmuInstance::audioGetNumSamplesOut(int outlen) { - //float f_len_in = (outlen * 32823.6328125 * (curFPS/60.0)) / (float)audioFreq; - float f_len_in = (outlen * 47743.4659091 * (curFPS/60.0)) / (float)audioFreq; + float f_len_in = outlen * (curFPS/targetFPS); f_len_in += audioSampleFrac; int len_in = (int)floor(f_len_in); audioSampleFrac = f_len_in - len_in; @@ -146,48 +147,20 @@ int EmuInstance::audioGetNumSamplesOut(int outlen) return len_in; } -void EmuInstance::audioResample(s16* inbuf, int inlen, s16* outbuf, int outlen, int volume) -{ - float res_incr = inlen / (float)outlen; - float res_timer = -0.5; - int res_pos = 0; - - for (int i = 0; i < outlen; i++) - { - s16 l1 = inbuf[res_pos * 2]; - s16 l2 = inbuf[res_pos * 2 + 2]; - s16 r1 = inbuf[res_pos * 2 + 1]; - s16 r2 = inbuf[res_pos * 2 + 3]; - - float l = (float)l1 + ((l2 - l1) * res_timer); - float r = (float)r1 + ((r2 - r1) * res_timer); - - outbuf[i*2 ] = (s16)(((s32)round(l) * volume) >> 8); - outbuf[i*2+1] = (s16)(((s32)round(r) * volume) >> 8); - - res_timer += res_incr; - while (res_timer >= 1.0) - { - res_timer -= 1.0; - res_pos++; - } - } -} - void EmuInstance::audioCallback(void* data, Uint8* stream, int len) { EmuInstance* inst = (EmuInstance*)data; len /= (sizeof(s16) * 2); - // resample incoming audio to match the output sample rate + double skew = std::clamp(inst->targetFPS / INTERNAL_FRAME_RATE, 0.995, 1.005); + inst->nds->SPU.SetOutputSkew(skew); int len_in = inst->audioGetNumSamplesOut(len); if (len_in > inst->audioBufSize) len_in = inst->audioBufSize; s16 buf_in[inst->audioBufSize*2]; - int num_in; SDL_LockMutex(inst->audioSyncLock); - num_in = inst->nds->SPU.ReadOutput(buf_in, len_in); + int num_in = inst->nds->SPU.ReadOutput((s16*) stream, len_in); SDL_CondSignal(inst->audioSyncCond); SDL_UnlockMutex(inst->audioSyncLock); @@ -197,18 +170,21 @@ void EmuInstance::audioCallback(void* data, Uint8* stream, int len) return; } + if (inst->audioVolume < 256) + { + s16* samples = (s16*) stream; + for (int i = 0; i < num_in * 2; i++) + samples[i] = ((s32) samples[i] * inst->audioVolume) >> 8; + } + int margin = 6; if (num_in < len_in-margin) { int last = num_in-1; for (int i = num_in; i < len_in-margin; i++) - ((u32*)buf_in)[i] = ((u32*)buf_in)[last]; - - num_in = len_in-margin; + ((u32*)stream)[i] = ((u32*)stream)[last]; } - - inst->audioResample(buf_in, num_in, (s16*)stream, len, inst->audioVolume); } @@ -249,7 +225,7 @@ void EmuInstance::micOpen() { mic = micDeviceName.c_str(); } - micDevice = SDL_OpenAudioDevice(mic, 1, &whatIwant, &whatIget, SDL_AUDIO_ALLOW_FREQUENCY_CHANGE | SDL_AUDIO_ALLOW_SAMPLES_CHANGE); + micDevice = SDL_OpenAudioDevice(mic, 1, &whatIwant, &whatIget, SDL_AUDIO_ALLOW_FREQUENCY_CHANGE); if (!micDevice) { Platform::Log(Platform::LogLevel::Error, "Mic init failed: %s\n", SDL_GetError()); diff --git a/src/frontend/qt_sdl/Platform.cpp b/src/frontend/qt_sdl/Platform.cpp index c9e2f07661..6ad06e5957 100644 --- a/src/frontend/qt_sdl/Platform.cpp +++ b/src/frontend/qt_sdl/Platform.cpp @@ -42,8 +42,11 @@ #include "SPI_Firmware.h" #ifdef __WIN32__ +#include +#define fdopen _fdopen #define fseek _fseeki64 #define ftell _ftelli64 +#define dup _dup #endif // __WIN32__ extern CameraManager* camManager[2]; @@ -136,20 +139,19 @@ FileHandle* OpenFile(const std::string& path, FileMode mode) std::string modeString = GetModeString(mode, QFile::exists(qpath)); QIODevice::OpenMode qmode = GetQMode(mode); QFile qfile{qpath}; - qfile.open(qmode); - FILE* file = fdopen(dup(qfile.handle()), modeString.c_str()); - qfile.close(); - - if (file) + if (qfile.open(qmode)) { - Log(LogLevel::Debug, "Opened \"%s\" with FileMode 0x%x (effective mode \"%s\")\n", path.c_str(), mode, modeString.c_str()); - return reinterpret_cast(file); - } - else - { - Log(LogLevel::Warn, "Failed to open \"%s\" with FileMode 0x%x (effective mode \"%s\")\n", path.c_str(), mode, modeString.c_str()); - return nullptr; + FILE* file = fdopen(dup(qfile.handle()), modeString.c_str()); + qfile.close(); + + if (file) + { + Log(LogLevel::Debug, "Opened \"%s\" with FileMode 0x%x (effective mode \"%s\")\n", path.c_str(), mode, modeString.c_str()); + return reinterpret_cast(file); + } } + Log(LogLevel::Warn, "Failed to open \"%s\" with FileMode 0x%x (effective mode \"%s\")\n", path.c_str(), mode, modeString.c_str()); + return nullptr; } std::string GetLocalFilePath(const std::string& filename) @@ -263,6 +265,11 @@ void FileRewind(FileHandle* file) rewind(reinterpret_cast(file)); } +u64 FilePosition(FileHandle* file) +{ + return ftell(reinterpret_cast(file)); +} + u64 FileRead(void* data, u64 size, u64 count, FileHandle* file) { return fread(data, size, count, reinterpret_cast(file)); diff --git a/src/frontend/qt_sdl/SaveManager.h b/src/frontend/qt_sdl/SaveManager.h index f9c8611072..6706c39f6a 100644 --- a/src/frontend/qt_sdl/SaveManager.h +++ b/src/frontend/qt_sdl/SaveManager.h @@ -20,7 +20,9 @@ #define SAVEMANAGER_H #include +#ifndef _MSC_VER #include +#endif #include #include #include diff --git a/src/frontend/qt_sdl/Screen.cpp b/src/frontend/qt_sdl/Screen.cpp index 17fd4b8644..6d58fff981 100755 --- a/src/frontend/qt_sdl/Screen.cpp +++ b/src/frontend/qt_sdl/Screen.cpp @@ -23,11 +23,7 @@ #include #include -#ifndef _WIN32 -#ifndef APPLE -#include -#endif -#endif + #include #include "OpenGLSupport.h" @@ -50,6 +46,14 @@ using namespace melonDS; +#if !defined(_WIN32) && !defined(APPLE) +#if QT_VERSION >= QT_VERSION_CHECK(6, 5, 0) +using namespace QNativeInterface; +#else +#include +#endif +#endif + const u32 kOSDMargin = 6; const int kLogoWidth = 192; @@ -1285,8 +1289,25 @@ std::optional ScreenPanelGL::getWindowInfo() wi.type = WindowInfo::Type::MacOS; wi.window_handle = reinterpret_cast(winId()); #else - QPlatformNativeInterface* pni = QGuiApplication::platformNativeInterface(); const QString platform_name = QGuiApplication::platformName(); + + #if QT_VERSION >= QT_VERSION_CHECK(6, 5, 0) + if (platform_name == QStringLiteral("xcb")) + { + wi.type = WindowInfo::Type::X11; + const QX11Application* x11 = qApp->nativeInterface(); + wi.display_connection = x11->display(); + wi.window_handle = reinterpret_cast(winId()); + } + else if (platform_name == QStringLiteral("wayland")) + { + wi.type = WindowInfo::Type::Wayland; + const QWaylandApplication* wl = qApp->nativeInterface(); + wi.display_connection = wl->display(); + wi.window_handle = reinterpret_cast(winId()); + } + #else + QPlatformNativeInterface* pni = QGuiApplication::platformNativeInterface(); if (platform_name == QStringLiteral("xcb")) { wi.type = WindowInfo::Type::X11; @@ -1303,9 +1324,9 @@ std::optional ScreenPanelGL::getWindowInfo() wi.display_connection = pni->nativeResourceForWindow("display", handle); wi.window_handle = pni->nativeResourceForWindow("surface", handle); } + #endif else { - //qCritical() << "Unknown PNI platform " << platform_name; Platform::Log(Platform::LogLevel::Error, "Unknown PNI platform %s\n", platform_name.toStdString().c_str()); return std::nullopt; } diff --git a/src/frontend/qt_sdl/TitleManagerDialog.cpp b/src/frontend/qt_sdl/TitleManagerDialog.cpp index a06f578194..f65ca1d419 100644 --- a/src/frontend/qt_sdl/TitleManagerDialog.cpp +++ b/src/frontend/qt_sdl/TitleManagerDialog.cpp @@ -119,7 +119,7 @@ void TitleManagerDialog::createTitleItem(u32 category, u32 titleid) // TODO: make it possible to select other languages? QString title = QString::fromUtf16(banner.EnglishTitle, 128); - title = title.left(title.indexOf('\0')); + title = title.left(title.indexOf(QChar('\0'))); title.replace("\n", " · "); char gamecode[5]; diff --git a/src/frontend/qt_sdl/Window.cpp b/src/frontend/qt_sdl/Window.cpp index b5a9b69064..9b3ab95c2d 100755 --- a/src/frontend/qt_sdl/Window.cpp +++ b/src/frontend/qt_sdl/Window.cpp @@ -47,9 +47,6 @@ #include #include #include -#ifndef APPLE -#include -#endif #endif #include "main.h" diff --git a/src/frontend/qt_sdl/main.cpp b/src/frontend/qt_sdl/main.cpp index 651426638f..f5bb3c9549 100644 --- a/src/frontend/qt_sdl/main.cpp +++ b/src/frontend/qt_sdl/main.cpp @@ -42,9 +42,6 @@ #include #include #include -#ifndef APPLE -#include -#endif #endif #include @@ -349,6 +346,10 @@ int main(int argc, char** argv) } } + // fix for Wayland OpenGL glitches + QGuiApplication::setAttribute(Qt::AA_NativeWindows, false); + QGuiApplication::setAttribute(Qt::AA_DontCreateNativeWidgetSiblings, true); + // default MP interface type is local MP // this will be changed if a LAN or netplay session is initiated setMPInterface(MPInterface_Local); @@ -401,24 +402,3 @@ int main(int argc, char** argv) SDL_Quit(); return ret; } - -#ifdef __WIN32__ - -#include - -int CALLBACK WinMain(HINSTANCE hinst, HINSTANCE hprev, LPSTR cmdline, int cmdshow) -{ - if (AttachConsole(ATTACH_PARENT_PROCESS) && GetStdHandle(STD_OUTPUT_HANDLE)) - { - freopen("CONOUT$", "w", stdout); - freopen("CONOUT$", "w", stderr); - } - - int ret = main(__argc, __argv); - - printf("\n\n>"); - - return ret; -} - -#endif diff --git a/src/melonDLDI.h b/src/melonDLDI.h index 271e07137d..ee77b78a4c 100644 --- a/src/melonDLDI.h +++ b/src/melonDLDI.h @@ -25,7 +25,7 @@ namespace melonDS { const u8 melonDLDI[] = { - 0xed, 0xa5, 0x8d, 0xbf, 0x20, 0x43, 0x68, 0x69, 0x73, 0x68, 0x6d, 0x00, 0x01, 0x09, 0x00, 0x00, + 0xed, 0xa5, 0x8d, 0xbf, 0x20, 0x43, 0x68, 0x69, 0x73, 0x68, 0x6d, 0x00, 0x01, 0x0a, 0x00, 0x00, 0x6d, 0x65, 0x6c, 0x6f, 0x6e, 0x44, 0x53, 0x20, 0x44, 0x4c, 0x44, 0x49, 0x20, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, diff --git a/src/net/Net_Slirp.cpp b/src/net/Net_Slirp.cpp index 3937c6bc16..c36ba84d4f 100644 --- a/src/net/Net_Slirp.cpp +++ b/src/net/Net_Slirp.cpp @@ -50,6 +50,9 @@ const u8 kServerMAC[6] = {0x00, 0xAB, 0x33, 0x28, 0x99, 0x44}; #ifdef __WIN32__ #define poll WSAPoll +#ifndef CLOCK_MONOTONIC +#define CLOCK_MONOTONIC 1 +#endif // https://stackoverflow.com/questions/5404277/porting-clock-gettime-to-windows diff --git a/src/net/libslirp/glib/glib.h b/src/net/libslirp/glib/glib.h index 3563f7f5e0..f48993e825 100644 --- a/src/net/libslirp/glib/glib.h +++ b/src/net/libslirp/glib/glib.h @@ -11,6 +11,11 @@ #if defined(WIN32) || defined(_WIN32) || defined(_MSC_VER) #define G_OS_WIN32 1 + +#include +#define open _open +#define close _close +typedef ptrdiff_t ssize_t; #endif #define G_LITTLE_ENDIAN 0 diff --git a/src/types.h b/src/types.h index 461c84f224..75b24f09ae 100644 --- a/src/types.h +++ b/src/types.h @@ -35,5 +35,12 @@ typedef int64_t s64; template using array2d = std::array, A>; + +#ifdef _MSC_VER +#define strcasecmp _stricmp +#define strncasecmp _strnicmp + +typedef ptrdiff_t ssize_t; +#endif } #endif // TYPES_H diff --git a/src/xxhash/xxh3.h b/src/xxhash/xxh3.h index 5d5faf8774..7e3ce68e0c 100644 --- a/src/xxhash/xxh3.h +++ b/src/xxhash/xxh3.h @@ -1,7 +1,7 @@ /* * xxHash - Extremely Fast Hash algorithm * Development source file for `xxh3` - * Copyright (C) 2019-2020 Yann Collet + * Copyright (C) 2019-2021 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * @@ -34,2357 +34,22 @@ */ /* - * Note: This file is separated for development purposes. - * It will be integrated into `xxhash.h` when development stage is completed. + * Note: This file used to host the source code of XXH3_* variants. + * during the development period. + * The source code is now properly integrated within xxhash.h. * - * Credit: most of the work on vectorial and asm variants comes from @easyaspi314 - */ - -#ifndef XXH3_H_1397135465 -#define XXH3_H_1397135465 - -/* === Dependencies === */ -#ifndef XXHASH_H_5627135585666179 -/* special: when including `xxh3.h` directly, turn on XXH_INLINE_ALL */ -# undef XXH_INLINE_ALL /* avoid redefinition */ -# define XXH_INLINE_ALL -#endif -#include "xxhash.h" - - -/* === Compiler specifics === */ - -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ -# define XXH_RESTRICT restrict -#else -/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ -# define XXH_RESTRICT /* disable */ -#endif - -#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ - || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ - || defined(__clang__) -# define XXH_likely(x) __builtin_expect(x, 1) -# define XXH_unlikely(x) __builtin_expect(x, 0) -#else -# define XXH_likely(x) (x) -# define XXH_unlikely(x) (x) -#endif - -#if defined(__GNUC__) -# if defined(__AVX2__) -# include -# elif defined(__SSE2__) -# include -# elif defined(__ARM_NEON__) || defined(__ARM_NEON) -# define inline __inline__ /* clang bug */ -# include -# undef inline -# endif -#elif defined(_MSC_VER) -# include -#endif - -/* - * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while - * remaining a true 64-bit/128-bit hash function. - * - * This is done by prioritizing a subset of 64-bit operations that can be - * emulated without too many steps on the average 32-bit machine. - * - * For example, these two lines seem similar, and run equally fast on 64-bit: - * - * xxh_u64 x; - * x ^= (x >> 47); // good - * x ^= (x >> 13); // bad - * - * However, to a 32-bit machine, there is a major difference. - * - * x ^= (x >> 47) looks like this: - * - * x.lo ^= (x.hi >> (47 - 32)); - * - * while x ^= (x >> 13) looks like this: - * - * // note: funnel shifts are not usually cheap. - * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); - * x.hi ^= (x.hi >> 13); - * - * The first one is significantly faster than the second, simply because the - * shift is larger than 32. This means: - * - All the bits we need are in the upper 32 bits, so we can ignore the lower - * 32 bits in the shift. - * - The shift result will always fit in the lower 32 bits, and therefore, - * we can ignore the upper 32 bits in the xor. - * - * Thanks to this optimization, XXH3 only requires these features to be efficient: - * - * - Usable unaligned access - * - A 32-bit or 64-bit ALU - * - If 32-bit, a decent ADC instruction - * - A 32 or 64-bit multiply with a 64-bit result - * - For the 128-bit variant, a decent byteswap helps short inputs. - * - * The first two are already required by XXH32, and almost all 32-bit and 64-bit - * platforms which can run XXH32 can run XXH3 efficiently. - * - * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one - * notable exception. - * - * First of all, Thumb-1 lacks support for the UMULL instruction which - * performs the important long multiply. This means numerous __aeabi_lmul - * calls. - * - * Second of all, the 8 functional registers are just not enough. - * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need - * Lo registers, and this shuffling results in thousands more MOVs than A32. - * - * A32 and T32 don't have this limitation. They can access all 14 registers, - * do a 32->64 multiply with UMULL, and the flexible operand allowing free - * shifts is helpful, too. - * - * Therefore, we do a quick sanity check. - * - * If compiling Thumb-1 for a target which supports ARM instructions, we will - * emit a warning, as it is not a "sane" platform to compile for. - * - * Usually, if this happens, it is because of an accident and you probably need - * to specify -march, as you likely meant to compile for a newer architecture. - */ -#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) -# warning "XXH3 is highly inefficient without ARM or Thumb-2." -#endif - -/* ========================================== - * Vectorization detection - * ========================================== */ -#define XXH_SCALAR 0 /* Portable scalar version */ -#define XXH_SSE2 1 /* SSE2 for Pentium 4 and all x86_64 */ -#define XXH_AVX2 2 /* AVX2 for Haswell and Bulldozer */ -#define XXH_NEON 3 /* NEON for most ARMv7-A and all AArch64 */ -#define XXH_VSX 4 /* VSX and ZVector for POWER8/z13 */ -#define XXH_AVX512 5 /* AVX512 for Skylake and Icelake */ - -#ifndef XXH_VECTOR /* can be defined on command line */ -# if defined(__AVX512F__) -# define XXH_VECTOR XXH_AVX512 -# elif defined(__AVX2__) -# define XXH_VECTOR XXH_AVX2 -# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) -# define XXH_VECTOR XXH_SSE2 -# elif defined(__GNUC__) /* msvc support maybe later */ \ - && (defined(__ARM_NEON__) || defined(__ARM_NEON)) \ - && (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) -# define XXH_VECTOR XXH_NEON -# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ - || (defined(__s390x__) && defined(__VEC__)) \ - && defined(__GNUC__) /* TODO: IBM XL */ -# define XXH_VECTOR XXH_VSX -# else -# define XXH_VECTOR XXH_SCALAR -# endif -#endif - -/* - * Controls the alignment of the accumulator. - * This is for compatibility with aligned vector loads, which are usually faster. - */ -#ifndef XXH_ACC_ALIGN -# if XXH_VECTOR == XXH_SCALAR /* scalar */ -# define XXH_ACC_ALIGN 8 -# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ -# define XXH_ACC_ALIGN 16 -# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ -# define XXH_ACC_ALIGN 32 -# elif XXH_VECTOR == XXH_NEON /* neon */ -# define XXH_ACC_ALIGN 16 -# elif XXH_VECTOR == XXH_VSX /* vsx */ -# define XXH_ACC_ALIGN 16 -# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ -# define XXH_ACC_ALIGN 64 -# endif -#endif - -/* - * UGLY HACK: - * GCC usually generates the best code with -O3 for xxHash. - * - * However, when targeting AVX2, it is overzealous in its unrolling resulting - * in code roughly 3/4 the speed of Clang. - * - * There are other issues, such as GCC splitting _mm256_loadu_si256 into - * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which - * only applies to Sandy and Ivy Bridge... which don't even support AVX2. - * - * That is why when compiling the AVX2 version, it is recommended to use either - * -O2 -mavx2 -march=haswell - * or - * -O2 -mavx2 -mno-avx256-split-unaligned-load - * for decent performance, or to use Clang instead. - * - * Fortunately, we can control the first one with a pragma that forces GCC into - * -O2, but the other one we can't control without "failed to inline always - * inline function due to target mismatch" warnings. - */ -#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ -# pragma GCC push_options -# pragma GCC optimize("-O2") -#endif - - -#if XXH_VECTOR == XXH_NEON -/* - * NEON's setup for vmlal_u32 is a little more complicated than it is on - * SSE2, AVX2, and VSX. - * - * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. - * - * To do the same operation, the 128-bit 'Q' register needs to be split into - * two 64-bit 'D' registers, performing this operation:: - * - * [ a | b ] - * | '---------. .--------' | - * | x | - * | .---------' '--------. | - * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] - * - * Due to significant changes in aarch64, the fastest method for aarch64 is - * completely different than the fastest method for ARMv7-A. - * - * ARMv7-A treats D registers as unions overlaying Q registers, so modifying - * D11 will modify the high half of Q5. This is similar to how modifying AH - * will only affect bits 8-15 of AX on x86. - * - * VZIP takes two registers, and puts even lanes in one register and odd lanes - * in the other. - * - * On ARMv7-A, this strangely modifies both parameters in place instead of - * taking the usual 3-operand form. - * - * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the - * lower and upper halves of the Q register to end up with the high and low - * halves where we want - all in one instruction. - * - * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } - * - * Unfortunately we need inline assembly for this: Instructions modifying two - * registers at once is not possible in GCC or Clang's IR, and they have to - * create a copy. - * - * aarch64 requires a different approach. - * - * In order to make it easier to write a decent compiler for aarch64, many - * quirks were removed, such as conditional execution. - * - * NEON was also affected by this. - * - * aarch64 cannot access the high bits of a Q-form register, and writes to a - * D-form register zero the high bits, similar to how writes to W-form scalar - * registers (or DWORD registers on x86_64) work. - * - * The formerly free vget_high intrinsics now require a vext (with a few - * exceptions) - * - * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent - * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one - * operand. - * - * The equivalent of the VZIP.32 on the lower and upper halves would be this - * mess: - * - * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } - * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } - * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } - * - * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): - * - * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); - * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); - * - * This is available on ARMv7-A, but is less efficient than a single VZIP.32. - */ - -/* - * Function-like macro: - * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) - * { - * outLo = (uint32x2_t)(in & 0xFFFFFFFF); - * outHi = (uint32x2_t)(in >> 32); - * in = UNDEFINED; - * } - */ -# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ - && defined(__GNUC__) \ - && !defined(__aarch64__) && !defined(__arm64__) -# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ - /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ - /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ - __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ - (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ - (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ - } while (0) -# else -# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - (outLo) = vmovn_u64 (in); \ - (outHi) = vshrn_n_u64 ((in), 32); \ - } while (0) -# endif -#endif /* XXH_VECTOR == XXH_NEON */ - -/* - * VSX and Z Vector helpers. - * - * This is very messy, and any pull requests to clean this up are welcome. - * - * There are a lot of problems with supporting VSX and s390x, due to - * inconsistent intrinsics, spotty coverage, and multiple endiannesses. - */ -#if XXH_VECTOR == XXH_VSX -# if defined(__s390x__) -# include -# else -# include -# endif - -# undef vector /* Undo the pollution */ - -typedef __vector unsigned long long xxh_u64x2; -typedef __vector unsigned char xxh_u8x16; -typedef __vector unsigned xxh_u32x4; - -# ifndef XXH_VSX_BE -# if defined(__BIG_ENDIAN__) \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -# define XXH_VSX_BE 1 -# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ -# warning "-maltivec=be is not recommended. Please use native endianness." -# define XXH_VSX_BE 1 -# else -# define XXH_VSX_BE 0 -# endif -# endif /* !defined(XXH_VSX_BE) */ - -# if XXH_VSX_BE -/* A wrapper for POWER9's vec_revb. */ -# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) -# define XXH_vec_revb vec_revb -# else -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) -{ - xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, - 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; - return vec_perm(val, val, vByteSwap); -} -# endif -# endif /* XXH_VSX_BE */ - -/* - * Performs an unaligned load and byte swaps it on big endian. - */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) -{ - xxh_u64x2 ret; - memcpy(&ret, ptr, sizeof(xxh_u64x2)); -# if XXH_VSX_BE - ret = XXH_vec_revb(ret); -# endif - return ret; -} - -/* - * vec_mulo and vec_mule are very problematic intrinsics on PowerPC - * - * These intrinsics weren't added until GCC 8, despite existing for a while, - * and they are endian dependent. Also, their meaning swap depending on version. - * */ -# if defined(__s390x__) - /* s390x is always big endian, no issue on this platform */ -# define XXH_vec_mulo vec_mulo -# define XXH_vec_mule vec_mule -# elif defined(__clang__) && __has_builtin(__builtin_altivec_vmuleuw) -/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ -# define XXH_vec_mulo __builtin_altivec_vmulouw -# define XXH_vec_mule __builtin_altivec_vmuleuw -# else -/* gcc needs inline assembly */ -/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) -{ - xxh_u64x2 result; - __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); - return result; -} -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) -{ - xxh_u64x2 result; - __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); - return result; -} -# endif /* XXH_vec_mulo, XXH_vec_mule */ -#endif /* XXH_VECTOR == XXH_VSX */ - - -/* prefetch - * can be disabled, by declaring XXH_NO_PREFETCH build macro */ -#if defined(XXH_NO_PREFETCH) -# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ -#else -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ -# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ -# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) -# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) -# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) -# else -# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ -# endif -#endif /* XXH_NO_PREFETCH */ - - -/* ========================================== - * XXH3 default settings - * ========================================== */ - -#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ - -#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) -# error "default keyset is not large enough" -#endif - -/* Pseudorandom secret taken directly from FARSH */ -XXH_ALIGN(64) static const xxh_u8 kSecret[XXH_SECRET_DEFAULT_SIZE] = { - 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, - 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, - 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, - 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, - 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, - 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, - 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, - 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, - - 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, - 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, - 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, - 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, -}; - -/* - * Calculates a 32-bit to 64-bit long multiply. - * - * Wraps __emulu on MSVC x86 because it tends to call __allmul when it doesn't - * need to (but it shouldn't need to anyways, it is about 7 instructions to do - * a 64x64 multiply...). Since we know that this will _always_ emit MULL, we - * use that instead of the normal method. - * - * If you are compiling for platforms like Thumb-1 and don't have a better option, - * you may also want to write your own long multiply routine here. - * - * XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) - * { - * return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); - * } - */ -#if defined(_MSC_VER) && defined(_M_IX86) -# include -# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) -#else -/* - * Downcast + upcast is usually better than masking on older compilers like - * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. - * - * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands - * and perform a full 64x64 multiply -- entirely redundant on 32-bit. - */ -# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) -#endif - -/* - * Calculates a 64->128-bit long multiply. - * - * Uses __uint128_t and _umul128 if available, otherwise uses a scalar version. - */ -static XXH128_hash_t -XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) -{ - /* - * GCC/Clang __uint128_t method. - * - * On most 64-bit targets, GCC and Clang define a __uint128_t type. - * This is usually the best way as it usually uses a native long 64-bit - * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. - * - * Usually. - * - * Despite being a 32-bit platform, Clang (and emscripten) define this type - * despite not having the arithmetic for it. This results in a laggy - * compiler builtin call which calculates a full 128-bit multiply. - * In that case it is best to use the portable one. - * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 - */ -#if defined(__GNUC__) && !defined(__wasm__) \ - && defined(__SIZEOF_INT128__) \ - || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) - - __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; - XXH128_hash_t r128; - r128.low64 = (xxh_u64)(product); - r128.high64 = (xxh_u64)(product >> 64); - return r128; - - /* - * MSVC for x64's _umul128 method. - * - * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); - * - * This compiles to single operand MUL on x64. - */ -#elif defined(_M_X64) || defined(_M_IA64) - -#ifndef _MSC_VER -# pragma intrinsic(_umul128) -#endif - xxh_u64 product_high; - xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); - XXH128_hash_t r128; - r128.low64 = product_low; - r128.high64 = product_high; - return r128; - -#else - /* - * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. - * - * This is a fast and simple grade school multiply, which is shown below - * with base 10 arithmetic instead of base 0x100000000. - * - * 9 3 // D2 lhs = 93 - * x 7 5 // D2 rhs = 75 - * ---------- - * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 - * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 - * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 - * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 - * --------- - * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 - * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 - * --------- - * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 - * - * The reasons for adding the products like this are: - * 1. It avoids manual carry tracking. Just like how - * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. - * This avoids a lot of complexity. - * - * 2. It hints for, and on Clang, compiles to, the powerful UMAAL - * instruction available in ARM's Digital Signal Processing extension - * in 32-bit ARMv6 and later, which is shown below: - * - * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) - * { - * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; - * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); - * *RdHi = (xxh_u32)(product >> 32); - * } - * - * This instruction was designed for efficient long multiplication, and - * allows this to be calculated in only 4 instructions at speeds - * comparable to some 64-bit ALUs. - * - * 3. It isn't terrible on other platforms. Usually this will be a couple - * of 32-bit ADD/ADCs. - */ - - /* First calculate all of the cross products. */ - xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); - xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); - xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); - xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); - - /* Now add the products together. These will never overflow. */ - xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; - xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; - xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); - - XXH128_hash_t r128; - r128.low64 = lower; - r128.high64 = upper; - return r128; -#endif -} - -/* - * Does a 64-bit to 128-bit multiply, then XOR folds it. - * - * The reason for the separate function is to prevent passing too many structs - * around by value. This will hopefully inline the multiply, but we don't force it. - */ -static xxh_u64 -XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) -{ - XXH128_hash_t product = XXH_mult64to128(lhs, rhs); - return product.low64 ^ product.high64; -} - -/* Seems to produce slightly better code on GCC for some reason. */ -XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) -{ - XXH_ASSERT(0 <= shift && shift < 64); - return v64 ^ (v64 >> shift); -} - -/* - * We don't need to (or want to) mix as much as XXH64. - * - * Short hashes are more evenly distributed, so it isn't necessary. - */ -static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) -{ - h64 = XXH_xorshift64(h64, 37); - h64 *= 0x165667919E3779F9ULL; - h64 = XXH_xorshift64(h64, 32); - return h64; -} - - -/* ========================================== - * Short keys - * ========================================== - * One of the shortcomings of XXH32 and XXH64 was that their performance was - * sub-optimal on short lengths. It used an iterative algorithm which strongly - * favored lengths that were a multiple of 4 or 8. - * - * Instead of iterating over individual inputs, we use a set of single shot - * functions which piece together a range of lengths and operate in constant time. - * - * Additionally, the number of multiplies has been significantly reduced. This - * reduces latency, especially when emulating 64-bit multiplies on 32-bit. - * - * Depending on the platform, this may or may not be faster than XXH32, but it - * is almost guaranteed to be faster than XXH64. - */ - -/* - * At very short lengths, there isn't enough input to fully hide secrets, or use - * the entire secret. - * - * There is also only a limited amount of mixing we can do before significantly - * impacting performance. - * - * Therefore, we use different sections of the secret and always mix two secret - * samples with an XOR. This should have no effect on performance on the - * seedless or withSeed variants because everything _should_ be constant folded - * by modern compilers. - * - * The XOR mixing hides individual parts of the secret and increases entropy. - * - * This adds an extra layer of strength for custom secrets. - */ -XXH_FORCE_INLINE XXH64_hash_t -XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(1 <= len && len <= 3); - XXH_ASSERT(secret != NULL); - /* - * len = 1: combined = { input[0], 0x01, input[0], input[0] } - * len = 2: combined = { input[1], 0x02, input[0], input[1] } - * len = 3: combined = { input[2], 0x03, input[0], input[1] } - */ - { xxh_u8 const c1 = input[0]; - xxh_u8 const c2 = input[len >> 1]; - xxh_u8 const c3 = input[len - 1]; - xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) - | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); - xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; - xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; - xxh_u64 const mixed = keyed * PRIME64_1; - return XXH3_avalanche(mixed); - } -} - -XXH_FORCE_INLINE XXH64_hash_t -XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len < 8); - seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; - { xxh_u32 const input1 = XXH_readLE32(input); - xxh_u32 const input2 = XXH_readLE32(input + len - 4); - xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; - xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); - xxh_u64 x = input64 ^ bitflip; - /* this mix is inspired by Pelle Evensen's rrmxmx */ - x ^= XXH_rotl64(x, 49) ^ XXH_rotl64(x, 24); - x *= 0x9FB21C651E98DF25ULL; - x ^= (x >> 35) + len ; - x *= 0x9FB21C651E98DF25ULL; - return XXH_xorshift64(x, 28); - } -} - -XXH_FORCE_INLINE XXH64_hash_t -XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(8 <= len && len <= 16); - { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; - xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; - xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; - xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; - xxh_u64 const acc = len - + XXH_swap64(input_lo) + input_hi - + XXH3_mul128_fold64(input_lo, input_hi); - return XXH3_avalanche(acc); - } -} - -XXH_FORCE_INLINE XXH64_hash_t -XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(len <= 16); - { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); - if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); - if (len) return XXH3_len_1to3_64b(input, len, secret, seed); - return XXH3_avalanche((PRIME64_1 + seed) ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); - } -} - -/* - * DISCLAIMER: There are known *seed-dependent* multicollisions here due to - * multiplication by zero, affecting hashes of lengths 17 to 240. - * - * However, they are very unlikely. - * - * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all - * unseeded non-cryptographic hashes, it does not attempt to defend itself - * against specially crafted inputs, only random inputs. - * - * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes - * cancelling out the secret is taken an arbitrary number of times (addressed - * in XXH3_accumulate_512), this collision is very unlikely with random inputs - * and/or proper seeding: - * - * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a - * function that is only called up to 16 times per hash with up to 240 bytes of - * input. - * - * This is not too bad for a non-cryptographic hash function, especially with - * only 64 bit outputs. - * - * The 128-bit variant (which trades some speed for strength) is NOT affected - * by this, although it is always a good idea to use a proper seed if you care - * about strength. - */ -XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, - const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) -{ -#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ - /* - * UGLY HACK: - * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in - * slower code. - * - * By forcing seed64 into a register, we disrupt the cost model and - * cause it to scalarize. See `XXH32_round()` - * - * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, - * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on - * GCC 9.2, despite both emitting scalar code. - * - * GCC generates much better scalar code than Clang for the rest of XXH3, - * which is why finding a more optimal codepath is an interest. - */ - __asm__ ("" : "+r" (seed64)); -#endif - { xxh_u64 const input_lo = XXH_readLE64(input); - xxh_u64 const input_hi = XXH_readLE64(input+8); - return XXH3_mul128_fold64( - input_lo ^ (XXH_readLE64(secret) + seed64), - input_hi ^ (XXH_readLE64(secret+8) - seed64) - ); - } -} - -/* For mid range keys, XXH3 uses a Mum-hash variant. */ -XXH_FORCE_INLINE XXH64_hash_t -XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; - XXH_ASSERT(16 < len && len <= 128); - - { xxh_u64 acc = len * PRIME64_1; - if (len > 32) { - if (len > 64) { - if (len > 96) { - acc += XXH3_mix16B(input+48, secret+96, seed); - acc += XXH3_mix16B(input+len-64, secret+112, seed); - } - acc += XXH3_mix16B(input+32, secret+64, seed); - acc += XXH3_mix16B(input+len-48, secret+80, seed); - } - acc += XXH3_mix16B(input+16, secret+32, seed); - acc += XXH3_mix16B(input+len-32, secret+48, seed); - } - acc += XXH3_mix16B(input+0, secret+0, seed); - acc += XXH3_mix16B(input+len-16, secret+16, seed); - - return XXH3_avalanche(acc); - } -} - -#define XXH3_MIDSIZE_MAX 240 - -XXH_NO_INLINE XXH64_hash_t -XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; - XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - - #define XXH3_MIDSIZE_STARTOFFSET 3 - #define XXH3_MIDSIZE_LASTOFFSET 17 - - { xxh_u64 acc = len * PRIME64_1; - int const nbRounds = (int)len / 16; - int i; - for (i=0; i<8; i++) { - acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); - } - acc = XXH3_avalanche(acc); - XXH_ASSERT(nbRounds >= 8); -#if defined(__clang__) /* Clang */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ - /* - * UGLY HACK: - * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. - * In everywhere else, it uses scalar code. - * - * For 64->128-bit multiplies, even if the NEON was 100% optimal, it - * would still be slower than UMAAL (see XXH_mult64to128). - * - * Unfortunately, Clang doesn't handle the long multiplies properly and - * converts them to the nonexistent "vmulq_u64" intrinsic, which is then - * scalarized into an ugly mess of VMOV.32 instructions. - * - * This mess is difficult to avoid without turning autovectorization - * off completely, but they are usually relatively minor and/or not - * worth it to fix. - * - * This loop is the easiest to fix, as unlike XXH32, this pragma - * _actually works_ because it is a loop vectorization instead of an - * SLP vectorization. - */ - #pragma clang loop vectorize(disable) -#endif - for (i=8 ; i < nbRounds; i++) { - acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); - } - /* last bytes */ - acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); - return XXH3_avalanche(acc); - } -} - - -/* === Long Keys === */ - -#define STRIPE_LEN 64 -#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ -#define ACC_NB (STRIPE_LEN / sizeof(xxh_u64)) - -typedef enum { XXH3_acc_64bits, XXH3_acc_128bits } XXH3_accWidth_e; - -/* - * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. - * - * It is a hardened version of UMAC, based off of FARSH's implementation. - * - * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD - * implementations, and it is ridiculously fast. - * - * We harden it by mixing the original input to the accumulators as well as the product. - * - * This means that in the (relatively likely) case of a multiply by zero, the - * original input is preserved. - * - * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve - * cross-pollination, as otherwise the upper and lower halves would be - * essentially independent. - * - * This doesn't matter on 64-bit hashes since they all get merged together in - * the end, so we skip the extra step. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ -XXH_FORCE_INLINE void -XXH3_accumulate_512( void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret, - XXH3_accWidth_e accWidth) -{ -#if (XXH_VECTOR == XXH_AVX512) - - XXH_ASSERT((((size_t)acc) & 63) == 0); - XXH_STATIC_ASSERT(STRIPE_LEN == sizeof(__m512i)); - { XXH_ALIGN(64) __m512i* const xacc = (__m512i *) acc; - - /* data_vec = input[0]; */ - __m512i const data_vec = _mm512_loadu_si512 (input); - /* key_vec = secret[0]; */ - __m512i const key_vec = _mm512_loadu_si512 (secret); - /* data_key = data_vec ^ key_vec; */ - __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); - if (accWidth == XXH3_acc_128bits) { - /* xacc[0] += swap(data_vec); */ - __m512i const data_swap = _mm512_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); - __m512i const sum = _mm512_add_epi64(*xacc, data_swap); - /* xacc[0] += product; */ - *xacc = _mm512_add_epi64(product, sum); - } else { /* XXH3_acc_64bits */ - /* xacc[0] += data_vec; */ - __m512i const sum = _mm512_add_epi64(*xacc, data_vec); - /* xacc[0] += product; */ - *xacc = _mm512_add_epi64(product, sum); - } - } - -#elif (XXH_VECTOR == XXH_AVX2) - - XXH_ASSERT((((size_t)acc) & 31) == 0); - { XXH_ALIGN(32) __m256i* const xacc = (__m256i *) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i* const xinput = (const __m256i *) input; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i* const xsecret = (const __m256i *) secret; - - size_t i; - for (i=0; i < STRIPE_LEN/sizeof(__m256i); i++) { - /* data_vec = xinput[i]; */ - __m256i const data_vec = _mm256_loadu_si256 (xinput+i); - /* key_vec = xsecret[i]; */ - __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); - /* data_key = data_vec ^ key_vec; */ - __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); - if (accWidth == XXH3_acc_128bits) { - /* xacc[i] += swap(data_vec); */ - __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); - __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); - /* xacc[i] += product; */ - xacc[i] = _mm256_add_epi64(product, sum); - } else { /* XXH3_acc_64bits */ - /* xacc[i] += data_vec; */ - __m256i const sum = _mm256_add_epi64(xacc[i], data_vec); - /* xacc[i] += product; */ - xacc[i] = _mm256_add_epi64(product, sum); - } - } } - -#elif (XXH_VECTOR == XXH_SSE2) - - /* SSE2 is just a half-scale version of the AVX2 version. */ - XXH_ASSERT((((size_t)acc) & 15) == 0); - { XXH_ALIGN(16) __m128i* const xacc = (__m128i *) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i* const xinput = (const __m128i *) input; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i* const xsecret = (const __m128i *) secret; - - size_t i; - for (i=0; i < STRIPE_LEN/sizeof(__m128i); i++) { - /* data_vec = xinput[i]; */ - __m128i const data_vec = _mm_loadu_si128 (xinput+i); - /* key_vec = xsecret[i]; */ - __m128i const key_vec = _mm_loadu_si128 (xsecret+i); - /* data_key = data_vec ^ key_vec; */ - __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); - if (accWidth == XXH3_acc_128bits) { - /* xacc[i] += swap(data_vec); */ - __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); - __m128i const sum = _mm_add_epi64(xacc[i], data_swap); - /* xacc[i] += product; */ - xacc[i] = _mm_add_epi64(product, sum); - } else { /* XXH3_acc_64bits */ - /* xacc[i] += data_vec; */ - __m128i const sum = _mm_add_epi64(xacc[i], data_vec); - /* xacc[i] += product; */ - xacc[i] = _mm_add_epi64(product, sum); - } - } } - -#elif (XXH_VECTOR == XXH_NEON) - - XXH_ASSERT((((size_t)acc) & 15) == 0); - { - XXH_ALIGN(16) uint64x2_t* const xacc = (uint64x2_t *) acc; - /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ - uint8_t const* const xinput = (const uint8_t *) input; - uint8_t const* const xsecret = (const uint8_t *) secret; - - size_t i; - for (i=0; i < STRIPE_LEN / sizeof(uint64x2_t); i++) { - /* data_vec = xinput[i]; */ - uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); - /* key_vec = xsecret[i]; */ - uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); - uint64x2_t data_key; - uint32x2_t data_key_lo, data_key_hi; - if (accWidth == XXH3_acc_64bits) { - /* xacc[i] += data_vec; */ - xacc[i] = vaddq_u64 (xacc[i], vreinterpretq_u64_u8(data_vec)); - } else { /* XXH3_acc_128bits */ - /* xacc[i] += swap(data_vec); */ - uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); - uint64x2_t const swapped = vextq_u64(data64, data64, 1); - xacc[i] = vaddq_u64 (xacc[i], swapped); - } - /* data_key = data_vec ^ key_vec; */ - data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); - /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); - * data_key_hi = (uint32x2_t) (data_key >> 32); - * data_key = UNDEFINED; */ - XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); - /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ - xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); - - } - } - -#elif (XXH_VECTOR == XXH_VSX) - xxh_u64x2* const xacc = (xxh_u64x2*) acc; /* presumed aligned */ - xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ - xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ - xxh_u64x2 const v32 = { 32, 32 }; - size_t i; - for (i = 0; i < STRIPE_LEN / sizeof(xxh_u64x2); i++) { - /* data_vec = xinput[i]; */ - xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); - /* key_vec = xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); - xxh_u64x2 const data_key = data_vec ^ key_vec; - /* shuffled = (data_key << 32) | (data_key >> 32); */ - xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); - /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ - xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); - xacc[i] += product; - - if (accWidth == XXH3_acc_64bits) { - xacc[i] += data_vec; - } else { /* XXH3_acc_128bits */ - /* swap high and low halves */ -#ifdef __s390x__ - xxh_u64x2 const data_swapped = vec_permi(data_vec, data_vec, 2); -#else - xxh_u64x2 const data_swapped = vec_xxpermdi(data_vec, data_vec, 2); -#endif - xacc[i] += data_swapped; - } - } - -#else /* scalar variant of Accumulator - universal */ - - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ - const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */ - const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ - size_t i; - XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); - for (i=0; i < ACC_NB; i++) { - xxh_u64 const data_val = XXH_readLE64(xinput + 8*i); - xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8); - - if (accWidth == XXH3_acc_64bits) { - xacc[i] += data_val; - } else { - xacc[i ^ 1] += data_val; /* swap adjacent lanes */ - } - xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); - } -#endif -} - -/* - * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. - * - * Multiplication isn't perfect, as explained by Google in HighwayHash: - * - * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to - * // varying degrees. In descending order of goodness, bytes - * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. - * // As expected, the upper and lower bytes are much worse. - * - * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 - * - * Since our algorithm uses a pseudorandom secret to add some variance into the - * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. - * - * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid - * extraction. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ -XXH_FORCE_INLINE void -XXH3_scrambleAcc(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) -{ -#if (XXH_VECTOR == XXH_AVX512) - - XXH_ASSERT((((size_t)acc) & 63) == 0); - XXH_STATIC_ASSERT(STRIPE_LEN == sizeof(__m512i)); - { XXH_ALIGN(64) __m512i* const xacc = (__m512i*) acc; - const __m512i prime32 = _mm512_set1_epi32((int)PRIME32_1); - - /* xacc[0] ^= (xacc[0] >> 47) */ - __m512i const acc_vec = *xacc; - __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); - __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); - /* xacc[0] ^= secret; */ - __m512i const key_vec = _mm512_loadu_si512 (secret); - __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); - - /* xacc[0] *= PRIME32_1; */ - __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); - __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); - *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); - } - -#elif (XXH_VECTOR == XXH_AVX2) - - XXH_ASSERT((((size_t)acc) & 31) == 0); - { XXH_ALIGN(32) __m256i* const xacc = (__m256i*) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i* const xsecret = (const __m256i *) secret; - const __m256i prime32 = _mm256_set1_epi32((int)PRIME32_1); - - size_t i; - for (i=0; i < STRIPE_LEN/sizeof(__m256i); i++) { - /* xacc[i] ^= (xacc[i] >> 47) */ - __m256i const acc_vec = xacc[i]; - __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); - __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); - /* xacc[i] ^= xsecret; */ - __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); - __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); - - /* xacc[i] *= PRIME32_1; */ - __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); - __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); - xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); - } - } - -#elif (XXH_VECTOR == XXH_SSE2) - - XXH_ASSERT((((size_t)acc) & 15) == 0); - { XXH_ALIGN(16) __m128i* const xacc = (__m128i*) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i* const xsecret = (const __m128i *) secret; - const __m128i prime32 = _mm_set1_epi32((int)PRIME32_1); - - size_t i; - for (i=0; i < STRIPE_LEN/sizeof(__m128i); i++) { - /* xacc[i] ^= (xacc[i] >> 47) */ - __m128i const acc_vec = xacc[i]; - __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); - __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); - /* xacc[i] ^= xsecret[i]; */ - __m128i const key_vec = _mm_loadu_si128 (xsecret+i); - __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); - - /* xacc[i] *= PRIME32_1; */ - __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); - __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); - xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); - } - } - -#elif (XXH_VECTOR == XXH_NEON) - - XXH_ASSERT((((size_t)acc) & 15) == 0); - - { uint64x2_t* xacc = (uint64x2_t*) acc; - uint8_t const* xsecret = (uint8_t const*) secret; - uint32x2_t prime = vdup_n_u32 (PRIME32_1); - - size_t i; - for (i=0; i < STRIPE_LEN/sizeof(uint64x2_t); i++) { - /* xacc[i] ^= (xacc[i] >> 47); */ - uint64x2_t acc_vec = xacc[i]; - uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); - uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); - - /* xacc[i] ^= xsecret[i]; */ - uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); - uint64x2_t data_key = veorq_u64(data_vec, vreinterpretq_u64_u8(key_vec)); - - /* xacc[i] *= PRIME32_1 */ - uint32x2_t data_key_lo, data_key_hi; - /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); - * data_key_hi = (uint32x2_t) (xacc[i] >> 32); - * xacc[i] = UNDEFINED; */ - XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); - { /* - * prod_hi = (data_key >> 32) * PRIME32_1; - * - * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will - * incorrectly "optimize" this: - * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); - * shifted = vshll_n_u32(tmp, 32); - * to this: - * tmp = "vmulq_u64"(a, b); // no such thing! - * shifted = vshlq_n_u64(tmp, 32); - * - * However, unlike SSE, Clang lacks a 64-bit multiply routine - * for NEON, and it scalarizes two 64-bit multiplies instead. - * - * vmull_u32 has the same timing as vmul_u32, and it avoids - * this bug completely. - * See https://bugs.llvm.org/show_bug.cgi?id=39967 - */ - uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); - /* xacc[i] = prod_hi << 32; */ - xacc[i] = vshlq_n_u64(prod_hi, 32); - /* xacc[i] += (prod_hi & 0xFFFFFFFF) * PRIME32_1; */ - xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); - } - } } - -#elif (XXH_VECTOR == XXH_VSX) - - XXH_ASSERT((((size_t)acc) & 15) == 0); - - { xxh_u64x2* const xacc = (xxh_u64x2*) acc; - const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; - /* constants */ - xxh_u64x2 const v32 = { 32, 32 }; - xxh_u64x2 const v47 = { 47, 47 }; - xxh_u32x4 const prime = { PRIME32_1, PRIME32_1, PRIME32_1, PRIME32_1 }; - size_t i; - for (i = 0; i < STRIPE_LEN / sizeof(xxh_u64x2); i++) { - /* xacc[i] ^= (xacc[i] >> 47); */ - xxh_u64x2 const acc_vec = xacc[i]; - xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); - - /* xacc[i] ^= xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); - xxh_u64x2 const data_key = data_vec ^ key_vec; - - /* xacc[i] *= PRIME32_1 */ - /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ - xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); - /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ - xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); - xacc[i] = prod_odd + (prod_even << v32); - } } - -#else /* scalar variant of Scrambler - universal */ - - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ - const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ - size_t i; - XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); - for (i=0; i < ACC_NB; i++) { - xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i); - xxh_u64 acc64 = xacc[i]; - acc64 = XXH_xorshift64(acc64, 47); - acc64 ^= key64; - acc64 *= PRIME32_1; - xacc[i] = acc64; - } - -#endif -} - -#define XXH_PREFETCH_DIST 384 - -#ifdef __clang__ // for clang -# define XXH_PREFETCH_DIST_AVX512_64 320 -# define XXH_PREFETCH_DIST_AVX512_128 320 -#else // for gcc -# define XXH_PREFETCH_DIST_AVX512_64 640 -# define XXH_PREFETCH_DIST_AVX512_128 512 -#endif - -/* - * XXH3_accumulate() - * Loops over XXH3_accumulate_512(). - * Assumption: nbStripes will not overflow the secret size - */ -XXH_FORCE_INLINE void -XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, - const xxh_u8* XXH_RESTRICT input, - const xxh_u8* XXH_RESTRICT secret, - size_t nbStripes, - XXH3_accWidth_e accWidth) -{ - size_t n; - for (n = 0; n < nbStripes; n++ ) { - const xxh_u8* const in = input + n*STRIPE_LEN; -#if (XXH_VECTOR == XXH_AVX512) - if (accWidth == XXH3_acc_64bits) XXH_PREFETCH(in + XXH_PREFETCH_DIST_AVX512_64); - else XXH_PREFETCH(in + XXH_PREFETCH_DIST_AVX512_128); -#else - XXH_PREFETCH(in + XXH_PREFETCH_DIST); -#endif - XXH3_accumulate_512(acc, - in, - secret + n*XXH_SECRET_CONSUME_RATE, - accWidth); - } -} - -XXH_FORCE_INLINE void -XXH3_hashLong_internal_loop( xxh_u64* XXH_RESTRICT acc, - const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH3_accWidth_e accWidth) -{ - size_t const nb_rounds = (secretSize - STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; - size_t const block_len = STRIPE_LEN * nb_rounds; - size_t const nb_blocks = len / block_len; - - size_t n; - - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - - for (n = 0; n < nb_blocks; n++) { - XXH3_accumulate(acc, input + n*block_len, secret, nb_rounds, accWidth); - XXH3_scrambleAcc(acc, secret + secretSize - STRIPE_LEN); - } - - /* last partial block */ - XXH_ASSERT(len > STRIPE_LEN); - { size_t const nbStripes = (len - (block_len * nb_blocks)) / STRIPE_LEN; - XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); - XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, accWidth); - - /* last stripe */ - if (len & (STRIPE_LEN - 1)) { - const xxh_u8* const p = input + len - STRIPE_LEN; - /* Do not align on 8, so that the secret is different from the scrambler */ -#define XXH_SECRET_LASTACC_START 7 - XXH3_accumulate_512(acc, p, secret + secretSize - STRIPE_LEN - XXH_SECRET_LASTACC_START, accWidth); - } } -} - -XXH_FORCE_INLINE xxh_u64 -XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) -{ - return XXH3_mul128_fold64( - acc[0] ^ XXH_readLE64(secret), - acc[1] ^ XXH_readLE64(secret+8) ); -} - -static XXH64_hash_t -XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) -{ - xxh_u64 result64 = start; - size_t i = 0; - - for (i = 0; i < 4; i++) { - result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); -#if defined(__clang__) /* Clang */ \ - && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ - /* - * UGLY HACK: - * Prevent autovectorization on Clang ARMv7-a. Exact same problem as - * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. - * XXH3_64bits, len == 256, Snapdragon 835: - * without hack: 2063.7 MB/s - * with hack: 2560.7 MB/s - */ - __asm__("" : "+r" (result64)); -#endif - } - - return XXH3_avalanche(result64); -} - -#define XXH3_INIT_ACC { PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3, \ - PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1 } - -XXH_FORCE_INLINE XXH64_hash_t -XXH3_hashLong_64b_internal(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize) -{ - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[ACC_NB] = XXH3_INIT_ACC; - - XXH3_hashLong_internal_loop(acc, input, len, secret, secretSize, XXH3_acc_64bits); - - /* converge into final hash */ - XXH_STATIC_ASSERT(sizeof(acc) == 64); - /* do not align on 8, so that the secret is different from the accumulator */ -#define XXH_SECRET_MERGEACCS_START 11 - XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * PRIME64_1); -} - -XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) -{ - if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); - memcpy(dst, &v64, sizeof(v64)); -} - -/* XXH3_initCustomSecret() : - * destination `customSecret` is presumed allocated and same size as `kSecret`. - */ -XXH_FORCE_INLINE void XXH3_initCustomSecret(xxh_u8* XXH_RESTRICT customSecret, xxh_u64 seed64) -{ - int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; - int i; - /* - * We need a separate pointer for the hack below. - * Any decent compiler will optimize this out otherwise. - */ - const xxh_u8 *kSecretPtr = kSecret; - - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); - -#if defined(__clang__) && defined(__aarch64__) - /* - * UGLY HACK: - * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are - * placed sequentially, in order, at the top of the unrolled loop. - * - * While MOVK is great for generating constants (2 cycles for a 64-bit - * constant compared to 4 cycles for LDR), long MOVK chains stall the - * integer pipelines: - * I L S - * MOVK - * MOVK - * MOVK - * MOVK - * ADD - * SUB STR - * STR - * By forcing loads from memory (as the asm line causes Clang to assume - * that kSecretPtr has been changed), the pipelines are used more efficiently: - * I L S - * LDR - * ADD LDR - * SUB STR - * STR - * XXH3_64bits_withSeed, len == 256, Snapdragon 835 - * without hack: 2654.4 MB/s - * with hack: 3202.9 MB/s - */ - __asm__("" : "+r" (kSecretPtr)); -#endif - /* - * Note: in debug mode, this overrides the asm optimization - * and Clang will emit MOVK chains again. - */ - XXH_ASSERT(kSecretPtr == kSecret); - - for (i=0; i < nbRounds; i++) { - /* - * The asm hack causes Clang to assume that kSecretPtr aliases with - * customSecret, and on aarch64, this prevented LDP from merging two - * loads together for free. Putting the loads together before the stores - * properly generates LDP. - */ - xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; - xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; - XXH_writeLE64(customSecret + 16*i, lo); - XXH_writeLE64(customSecret + 16*i + 8, hi); - } -} - - -/* - * It's important for performance that XXH3_hashLong is not inlined. Not sure - * why (uop cache maybe?), but the difference is large and easily measurable. - */ -XXH_NO_INLINE XXH64_hash_t -XXH3_hashLong_64b_defaultSecret(const xxh_u8* XXH_RESTRICT input, size_t len) -{ - return XXH3_hashLong_64b_internal(input, len, kSecret, sizeof(kSecret)); -} - -/* - * It's important for performance that XXH3_hashLong is not inlined. Not sure - * why (uop cache maybe?), but the difference is large and easily measurable. - */ -XXH_NO_INLINE XXH64_hash_t -XXH3_hashLong_64b_withSecret(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize) -{ - return XXH3_hashLong_64b_internal(input, len, secret, secretSize); -} - -/* - * XXH3_hashLong_64b_withSeed(): - * Generate a custom key based on alteration of default kSecret with the seed, - * and then use this key for long mode hashing. - * - * This operation is decently fast but nonetheless costs a little bit of time. - * Try to avoid it whenever possible (typically when seed==0). - * - * It's important for performance that XXH3_hashLong is not inlined. Not sure - * why (uop cache maybe?), but the difference is large and easily measurable. - */ -XXH_NO_INLINE XXH64_hash_t -XXH3_hashLong_64b_withSeed(const xxh_u8* input, size_t len, XXH64_hash_t seed) -{ - XXH_ALIGN(8) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; - if (seed==0) return XXH3_hashLong_64b_defaultSecret(input, len); - XXH3_initCustomSecret(secret, seed); - return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret)); -} - -/* === Public entry point === */ - -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) -{ - if (len <= 16) - return XXH3_len_0to16_64b((const xxh_u8*)input, len, kSecret, 0); - if (len <= 128) - return XXH3_len_17to128_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0); - return XXH3_hashLong_64b_defaultSecret((const xxh_u8*)input, len); -} - -XXH_PUBLIC_API XXH64_hash_t -XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - /* - * If an action is to be taken if `secret` conditions are not respected, - * it should be done here. - * For now, it's a contract pre-condition. - * Adding a check and a branch here would cost performance at every hash. - */ - if (len <= 16) - return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, 0); - if (len <= 128) - return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0); - return XXH3_hashLong_64b_withSecret((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize); -} - -XXH_PUBLIC_API XXH64_hash_t -XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) -{ - if (len <= 16) - return XXH3_len_0to16_64b((const xxh_u8*)input, len, kSecret, seed); - if (len <= 128) - return XXH3_len_17to128_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed); - return XXH3_hashLong_64b_withSeed((const xxh_u8*)input, len, seed); -} - -/* === XXH3 streaming === */ - - -/* - * Malloc's a pointer that is always aligned to align. - * - * This must be freed with `XXH_alignedFree()`. - * - * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte - * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 - * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * xxh3.h is no longer useful, + * but it is still provided for compatibility with source code + * which used to include it directly. * - * This underalignment previously caused a rather obvious crash which went - * completely unnoticed due to XXH3_createState() not actually being tested. - * Credit to RedSpah for noticing this bug. + * Programs are now highly discouraged to include xxh3.h. + * Include `xxhash.h` instead, which is the officially supported interface. * - * The alignment is done manually: Functions like posix_memalign or _mm_malloc - * are avoided: To maintain portability, we would have to write a fallback - * like this anyways, and besides, testing for the existence of library - * functions without relying on external build tools is impossible. - * - * The method is simple: Overallocate, manually align, and store the offset - * to the original behind the returned pointer. - * - * Align must be a power of 2 and 8 <= align <= 128. - */ -static void* XXH_alignedMalloc(size_t s, size_t align) -{ - XXH_ASSERT(align <= 128 && align >= 8); /* range check */ - XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ - XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ - { /* Overallocate to make room for manual realignment and an offset byte */ - xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); - if (base != NULL) { - /* - * Get the offset needed to align this pointer. - * - * Even if the returned pointer is aligned, there will always be - * at least one byte to store the offset to the original pointer. - */ - size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ - /* Add the offset for the now-aligned pointer */ - xxh_u8* ptr = base + offset; - - XXH_ASSERT((size_t)ptr % align == 0); - - /* Store the offset immediately before the returned pointer. */ - ptr[-1] = (xxh_u8)offset; - return ptr; - } - return NULL; - } -} -/* - * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass - * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + * In the future, xxh3.h will start to generate warnings, then errors, + * then it will be removed from source package and from include directory. */ -static void XXH_alignedFree(void* p) -{ - if (p != NULL) { - xxh_u8* ptr = (xxh_u8*)p; - /* Get the offset byte we added in XXH_malloc. */ - xxh_u8 offset = ptr[-1]; - /* Free the original malloc'd pointer */ - xxh_u8* base = ptr - offset; - XXH_free(base); - } -} -XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) -{ - return (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); -} - -XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) -{ - XXH_alignedFree(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API void -XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) -{ - memcpy(dst_state, src_state, sizeof(*dst_state)); -} - -static void -XXH3_64bits_reset_internal(XXH3_state_t* statePtr, - XXH64_hash_t seed, - const xxh_u8* secret, size_t secretSize) -{ - XXH_ASSERT(statePtr != NULL); - memset(statePtr, 0, sizeof(*statePtr)); - statePtr->acc[0] = PRIME32_3; - statePtr->acc[1] = PRIME64_1; - statePtr->acc[2] = PRIME64_2; - statePtr->acc[3] = PRIME64_3; - statePtr->acc[4] = PRIME64_4; - statePtr->acc[5] = PRIME32_2; - statePtr->acc[6] = PRIME64_5; - statePtr->acc[7] = PRIME32_1; - statePtr->seed = seed; - XXH_ASSERT(secret != NULL); - statePtr->secret = secret; - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - statePtr->secretLimit = (XXH32_hash_t)(secretSize - STRIPE_LEN); - statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; -} - -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset(XXH3_state_t* statePtr) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_64bits_reset_internal(statePtr, 0, kSecret, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_64bits_reset_internal(statePtr, 0, (const xxh_u8*)secret, secretSize); - if (secret == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_64bits_reset_internal(statePtr, seed, kSecret, XXH_SECRET_DEFAULT_SIZE); - XXH3_initCustomSecret(statePtr->customSecret, seed); - statePtr->secret = statePtr->customSecret; - return XXH_OK; -} - -XXH_FORCE_INLINE void -XXH3_consumeStripes( xxh_u64* acc, - XXH32_hash_t* nbStripesSoFarPtr, XXH32_hash_t nbStripesPerBlock, - const xxh_u8* input, size_t totalStripes, - const xxh_u8* secret, size_t secretLimit, - XXH3_accWidth_e accWidth) -{ - XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); - if (nbStripesPerBlock - *nbStripesSoFarPtr <= totalStripes) { - /* need a scrambling operation */ - size_t const nbStripes = nbStripesPerBlock - *nbStripesSoFarPtr; - XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, accWidth); - XXH3_scrambleAcc(acc, secret + secretLimit); - XXH3_accumulate(acc, input + nbStripes * STRIPE_LEN, secret, totalStripes - nbStripes, accWidth); - *nbStripesSoFarPtr = (XXH32_hash_t)(totalStripes - nbStripes); - } else { - XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, totalStripes, accWidth); - *nbStripesSoFarPtr += (XXH32_hash_t)totalStripes; - } -} - -/* - * Both XXH3_64bits_update and XXH3_128bits_update use this routine. - */ -XXH_FORCE_INLINE XXH_errorcode -XXH3_update(XXH3_state_t* state, const xxh_u8* input, size_t len, XXH3_accWidth_e accWidth) -{ - if (input==NULL) -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - return XXH_OK; -#else - return XXH_ERROR; -#endif - - { const xxh_u8* const bEnd = input + len; - - state->totalLen += len; - - if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ - XXH_memcpy(state->buffer + state->bufferedSize, input, len); - state->bufferedSize += (XXH32_hash_t)len; - return XXH_OK; - } - /* input is now > XXH3_INTERNALBUFFER_SIZE */ - - #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / STRIPE_LEN) - XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % STRIPE_LEN == 0); /* clean multiple */ - - /* - * There is some input left inside the internal buffer. - * Fill it, then consume it. - */ - if (state->bufferedSize) { - size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; - XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); - input += loadSize; - XXH3_consumeStripes(state->acc, - &state->nbStripesSoFar, state->nbStripesPerBlock, - state->buffer, XXH3_INTERNALBUFFER_STRIPES, - state->secret, state->secretLimit, - accWidth); - state->bufferedSize = 0; - } - - /* Consume input by full buffer quantities */ - if (input+XXH3_INTERNALBUFFER_SIZE <= bEnd) { - const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; - do { - XXH3_consumeStripes(state->acc, - &state->nbStripesSoFar, state->nbStripesPerBlock, - input, XXH3_INTERNALBUFFER_STRIPES, - state->secret, state->secretLimit, - accWidth); - input += XXH3_INTERNALBUFFER_SIZE; - } while (input<=limit); - } - - if (input < bEnd) { /* Some remaining input: buffer it */ - XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); - state->bufferedSize = (XXH32_hash_t)(bEnd-input); - } - } - return XXH_OK; -} +/* Simulate the same impact as including the old xxh3.h source file */ -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) -{ - return XXH3_update(state, (const xxh_u8*)input, len, XXH3_acc_64bits); -} - - -XXH_FORCE_INLINE void -XXH3_digest_long (XXH64_hash_t* acc, const XXH3_state_t* state, XXH3_accWidth_e accWidth) -{ - /* - * Digest on a local copy. This way, the state remains unaltered, and it can - * continue ingesting more input afterwards. - */ - memcpy(acc, state->acc, sizeof(state->acc)); - if (state->bufferedSize >= STRIPE_LEN) { - size_t const totalNbStripes = state->bufferedSize / STRIPE_LEN; - XXH32_hash_t nbStripesSoFar = state->nbStripesSoFar; - XXH3_consumeStripes(acc, - &nbStripesSoFar, state->nbStripesPerBlock, - state->buffer, totalNbStripes, - state->secret, state->secretLimit, - accWidth); - if (state->bufferedSize % STRIPE_LEN) { /* one last partial stripe */ - XXH3_accumulate_512(acc, - state->buffer + state->bufferedSize - STRIPE_LEN, - state->secret + state->secretLimit - XXH_SECRET_LASTACC_START, - accWidth); - } - } else { /* bufferedSize < STRIPE_LEN */ - if (state->bufferedSize) { /* one last stripe */ - xxh_u8 lastStripe[STRIPE_LEN]; - size_t const catchupSize = STRIPE_LEN - state->bufferedSize; - memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); - memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); - XXH3_accumulate_512(acc, - lastStripe, - state->secret + state->secretLimit - XXH_SECRET_LASTACC_START, - accWidth); - } } -} - -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) -{ - if (state->totalLen > XXH3_MIDSIZE_MAX) { - XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[ACC_NB]; - XXH3_digest_long(acc, state, XXH3_acc_64bits); - return XXH3_mergeAccs(acc, - state->secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)state->totalLen * PRIME64_1); - } - /* len <= XXH3_MIDSIZE_MAX: short code */ - if (state->seed) - return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); - return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), - state->secret, state->secretLimit + STRIPE_LEN); -} - -/* ========================================== - * XXH3 128 bits (a.k.a XXH128) - * ========================================== - * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, - * even without counting the significantly larger output size. - * - * For example, extra steps are taken to avoid the seed-dependent collisions - * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). - * - * This strength naturally comes at the cost of some speed, especially on short - * lengths. Note that longer hashes are about as fast as the 64-bit version - * due to it using only a slight modification of the 64-bit loop. - * - * XXH128 is also more oriented towards 64-bit machines. It is still extremely - * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). - */ - -XXH_FORCE_INLINE XXH128_hash_t -XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - /* A doubled version of 1to3_64b with different constants. */ - XXH_ASSERT(input != NULL); - XXH_ASSERT(1 <= len && len <= 3); - XXH_ASSERT(secret != NULL); - /* - * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } - * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } - * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } - */ - { xxh_u8 const c1 = input[0]; - xxh_u8 const c2 = input[len >> 1]; - xxh_u8 const c3 = input[len - 1]; - xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) - | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); - xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); - xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; - xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; - xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; - xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; - xxh_u64 const mixedl = keyed_lo * PRIME64_1; - xxh_u64 const mixedh = keyed_hi * PRIME64_5; - XXH128_hash_t h128; - h128.low64 = XXH3_avalanche(mixedl); - h128.high64 = XXH3_avalanche(mixedh); - return h128; - } -} - -XXH_FORCE_INLINE XXH128_hash_t -XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len <= 8); - seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; - { xxh_u32 const input_lo = XXH_readLE32(input); - xxh_u32 const input_hi = XXH_readLE32(input + len - 4); - xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); - xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; - xxh_u64 const keyed = input_64 ^ bitflip; - - /* Shift len to the left to ensure it is even, this avoids even multiplies. */ - XXH128_hash_t m128 = XXH_mult64to128(keyed, PRIME64_1 + (len << 2)); - - m128.high64 += (m128.low64 << 1); - m128.low64 ^= (m128.high64 >> 3); - - m128.low64 = XXH_xorshift64(m128.low64, 35); - m128.low64 *= 0x9FB21C651E98DF25ULL; - m128.low64 = XXH_xorshift64(m128.low64, 28); - m128.high64 = XXH3_avalanche(m128.high64); - return m128; - } -} - -XXH_FORCE_INLINE XXH128_hash_t -XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(9 <= len && len <= 16); - { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; - xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; - xxh_u64 const input_lo = XXH_readLE64(input); - xxh_u64 input_hi = XXH_readLE64(input + len - 8); - XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, PRIME64_1); - /* - * Put len in the middle of m128 to ensure that the length gets mixed to - * both the low and high bits in the 128x64 multiply below. - */ - m128.low64 += (xxh_u64)(len - 1) << 54; - input_hi ^= bitfliph; - /* - * Add the high 32 bits of input_hi to the high 32 bits of m128, then - * add the long product of the low 32 bits of input_hi and PRIME32_2 to - * the high 64 bits of m128. - * - * The best approach to this operation is different on 32-bit and 64-bit. - */ - if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ - /* - * 32-bit optimized version, which is more readable. - * - * On 32-bit, it removes an ADC and delays a dependency between the two - * halves of m128.high64, but it generates an extra mask on 64-bit. - */ - m128.high64 += (input_hi & 0xFFFFFFFF00000000) + XXH_mult32to64((xxh_u32)input_hi, PRIME32_2); - } else { - /* - * 64-bit optimized (albeit more confusing) version. - * - * Uses some properties of addition and multiplication to remove the mask: - * - * Let: - * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) - * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) - * c = PRIME32_2 - * - * a + (b * c) - * Inverse Property: x + y - x == y - * a + (b * (1 + c - 1)) - * Distributive Property: x * (y + z) == (x * y) + (x * z) - * a + (b * 1) + (b * (c - 1)) - * Identity Property: x * 1 == x - * a + b + (b * (c - 1)) - * - * Substitute a, b, and c: - * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (PRIME32_2 - 1)) - * - * Since input_hi.hi + input_hi.lo == input_hi, we get this: - * input_hi + ((xxh_u64)input_hi.lo * (PRIME32_2 - 1)) - */ - m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, PRIME32_2 - 1); - } - /* m128 ^= XXH_swap64(m128 >> 64); */ - m128.low64 ^= XXH_swap64(m128.high64); - - { /* 128x64 multiply: h128 = m128 * PRIME64_2; */ - XXH128_hash_t h128 = XXH_mult64to128(m128.low64, PRIME64_2); - h128.high64 += m128.high64 * PRIME64_2; - - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = XXH3_avalanche(h128.high64); - return h128; - } } -} - -/* - * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN - */ -XXH_FORCE_INLINE XXH128_hash_t -XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(len <= 16); - { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); - if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); - if (len) return XXH3_len_1to3_128b(input, len, secret, seed); - { XXH128_hash_t h128; - xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); - xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); - h128.low64 = XXH3_avalanche((PRIME64_1 + seed) ^ bitflipl); - h128.high64 = XXH3_avalanche((PRIME64_2 - seed) ^ bitfliph); - return h128; - } } -} - -/* - * A bit slower than XXH3_mix16B, but handles multiply by zero better. - */ -XXH_FORCE_INLINE XXH128_hash_t -XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, - const xxh_u8* secret, XXH64_hash_t seed) -{ - acc.low64 += XXH3_mix16B (input_1, secret+0, seed); - acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); - acc.high64 += XXH3_mix16B (input_2, secret+16, seed); - acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); - return acc; -} - - -XXH_FORCE_INLINE XXH128_hash_t -XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; - XXH_ASSERT(16 < len && len <= 128); - - { XXH128_hash_t acc; - acc.low64 = len * PRIME64_1; - acc.high64 = 0; - if (len > 32) { - if (len > 64) { - if (len > 96) { - acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); - } - acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); - } - acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); - } - acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); - { XXH128_hash_t h128; - h128.low64 = acc.low64 + acc.high64; - h128.high64 = (acc.low64 * PRIME64_1) - + (acc.high64 * PRIME64_4) - + ((len - seed) * PRIME64_2); - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); - return h128; - } - } -} - -XXH_NO_INLINE XXH128_hash_t -XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; - XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - - { XXH128_hash_t acc; - int const nbRounds = (int)len / 32; - int i; - acc.low64 = len * PRIME64_1; - acc.high64 = 0; - for (i=0; i<4; i++) { - acc = XXH128_mix32B(acc, - input + (32 * i), - input + (32 * i) + 16, - secret + (32 * i), - seed); - } - acc.low64 = XXH3_avalanche(acc.low64); - acc.high64 = XXH3_avalanche(acc.high64); - XXH_ASSERT(nbRounds >= 4); - for (i=4 ; i < nbRounds; i++) { - acc = XXH128_mix32B(acc, - input + (32 * i), - input + (32 * i) + 16, - secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), - seed); - } - /* last bytes */ - acc = XXH128_mix32B(acc, - input + len - 16, - input + len - 32, - secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, - 0ULL - seed); - - { XXH128_hash_t h128; - h128.low64 = acc.low64 + acc.high64; - h128.high64 = (acc.low64 * PRIME64_1) - + (acc.high64 * PRIME64_4) - + ((len - seed) * PRIME64_2); - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); - return h128; - } - } -} - -XXH_FORCE_INLINE XXH128_hash_t -XXH3_hashLong_128b_internal(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize) -{ - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[ACC_NB] = XXH3_INIT_ACC; - - XXH3_hashLong_internal_loop(acc, input, len, secret, secretSize, XXH3_acc_128bits); - - /* converge into final hash */ - XXH_STATIC_ASSERT(sizeof(acc) == 64); - XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - { XXH128_hash_t h128; - h128.low64 = XXH3_mergeAccs(acc, - secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)len * PRIME64_1); - h128.high64 = XXH3_mergeAccs(acc, - secret + secretSize - - sizeof(acc) - XXH_SECRET_MERGEACCS_START, - ~((xxh_u64)len * PRIME64_2)); - return h128; - } -} - -/* - * It's important for performance that XXH3_hashLong is not inlined. Not sure - * why (uop cache maybe?), but the difference is large and easily measurable. - */ -XXH_NO_INLINE XXH128_hash_t -XXH3_hashLong_128b_defaultSecret(const xxh_u8* input, size_t len) -{ - return XXH3_hashLong_128b_internal(input, len, kSecret, sizeof(kSecret)); -} - -/* - * It's important for performance that XXH3_hashLong is not inlined. Not sure - * why (uop cache maybe?), but the difference is large and easily measurable. - */ -XXH_NO_INLINE XXH128_hash_t -XXH3_hashLong_128b_withSecret(const xxh_u8* input, size_t len, - const xxh_u8* secret, size_t secretSize) -{ - return XXH3_hashLong_128b_internal(input, len, secret, secretSize); -} - -/* - * It's important for performance that XXH3_hashLong is not inlined. Not sure - * why (uop cache maybe?), but the difference is large and easily measurable. - */ -XXH_NO_INLINE XXH128_hash_t -XXH3_hashLong_128b_withSeed(const xxh_u8* input, size_t len, XXH64_hash_t seed) -{ - XXH_ALIGN(8) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; - if (seed == 0) return XXH3_hashLong_128b_defaultSecret(input, len); - XXH3_initCustomSecret(secret, seed); - return XXH3_hashLong_128b_internal(input, len, secret, sizeof(secret)); -} - - -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) -{ - if (len <= 16) - return XXH3_len_0to16_128b((const xxh_u8*)input, len, kSecret, 0); - if (len <= 128) - return XXH3_len_17to128_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0); - return XXH3_hashLong_128b_defaultSecret((const xxh_u8*)input, len); -} - -XXH_PUBLIC_API XXH128_hash_t -XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - /* - * If an action is to be taken if `secret` conditions are not respected, - * it should be done here. - * For now, it's a contract pre-condition. - * Adding a check and a branch here would cost performance at every hash. - */ - if (len <= 16) - return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, 0); - if (len <= 128) - return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0); - return XXH3_hashLong_128b_withSecret((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize); -} - -XXH_PUBLIC_API XXH128_hash_t -XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) -{ - if (len <= 16) - return XXH3_len_0to16_128b((const xxh_u8*)input, len, kSecret, seed); - if (len <= 128) - return XXH3_len_17to128_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed); - return XXH3_hashLong_128b_withSeed((const xxh_u8*)input, len, seed); -} - -XXH_PUBLIC_API XXH128_hash_t -XXH128(const void* input, size_t len, XXH64_hash_t seed) -{ - return XXH3_128bits_withSeed(input, len, seed); -} - - -/* === XXH3 128-bit streaming === */ - -/* - * All the functions are actually the same as for 64-bit streaming variant. - * The only difference is the finalizatiom routine. - */ - -static void -XXH3_128bits_reset_internal(XXH3_state_t* statePtr, - XXH64_hash_t seed, - const xxh_u8* secret, size_t secretSize) -{ - XXH3_64bits_reset_internal(statePtr, seed, secret, secretSize); -} - -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset(XXH3_state_t* statePtr) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_128bits_reset_internal(statePtr, 0, kSecret, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_128bits_reset_internal(statePtr, 0, (const xxh_u8*)secret, secretSize); - if (secret == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_128bits_reset_internal(statePtr, seed, kSecret, XXH_SECRET_DEFAULT_SIZE); - XXH3_initCustomSecret(statePtr->customSecret, seed); - statePtr->secret = statePtr->customSecret; - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) -{ - return XXH3_update(state, (const xxh_u8*)input, len, XXH3_acc_128bits); -} - -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) -{ - if (state->totalLen > XXH3_MIDSIZE_MAX) { - XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[ACC_NB]; - XXH3_digest_long(acc, state, XXH3_acc_128bits); - XXH_ASSERT(state->secretLimit + STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - { XXH128_hash_t h128; - h128.low64 = XXH3_mergeAccs(acc, - state->secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)state->totalLen * PRIME64_1); - h128.high64 = XXH3_mergeAccs(acc, - state->secret + state->secretLimit + STRIPE_LEN - - sizeof(acc) - XXH_SECRET_MERGEACCS_START, - ~((xxh_u64)state->totalLen * PRIME64_2)); - return h128; - } - } - /* len <= XXH3_MIDSIZE_MAX : short code */ - if (state->seed) - return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); - return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), - state->secret, state->secretLimit + STRIPE_LEN); -} - -/* 128-bit utility functions */ - -#include /* memcmp, memcpy */ - -/* return : 1 is equal, 0 if different */ -XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) -{ - /* note : XXH128_hash_t is compact, it has no padding byte */ - return !(memcmp(&h1, &h2, sizeof(h1))); -} - -/* This prototype is compatible with stdlib's qsort(). - * return : >0 if *h128_1 > *h128_2 - * <0 if *h128_1 < *h128_2 - * =0 if *h128_1 == *h128_2 */ -XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) -{ - XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; - XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; - int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); - /* note : bets that, in most cases, hash values are different */ - if (hcmp) return hcmp; - return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); -} - - -/*====== Canonical representation ======*/ -XXH_PUBLIC_API void -XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) { - hash.high64 = XXH_swap64(hash.high64); - hash.low64 = XXH_swap64(hash.low64); - } - memcpy(dst, &hash.high64, sizeof(hash.high64)); - memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); -} - -XXH_PUBLIC_API XXH128_hash_t -XXH128_hashFromCanonical(const XXH128_canonical_t* src) -{ - XXH128_hash_t h; - h.high64 = XXH_readBE64(src); - h.low64 = XXH_readBE64(src->digest + 8); - return h; -} - -/* Pop our optimization override from above */ -#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ -# pragma GCC pop_options -#endif - -#endif /* XXH3_H_1397135465 */ +#define XXH_INLINE_ALL +#include "xxhash.h" diff --git a/src/xxhash/xxhash.c b/src/xxhash/xxhash.c index 0fae88c5d6..e60cc37f13 100644 --- a/src/xxhash/xxhash.c +++ b/src/xxhash/xxhash.c @@ -1,6 +1,6 @@ /* * xxHash - Extremely Fast Hash algorithm - * Copyright (C) 2012-2020 Yann Collet + * Copyright (C) 2012-2023 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * @@ -32,12 +32,11 @@ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ - /* * xxhash.c instantiates functions defined in xxhash.h */ -#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ -#define XXH_IMPLEMENTATION /* access definitions */ +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ #include "xxhash.h" diff --git a/src/xxhash/xxhash.h b/src/xxhash/xxhash.h index 67a5887f39..78fc2e8dbf 100644 --- a/src/xxhash/xxhash.h +++ b/src/xxhash/xxhash.h @@ -1,7 +1,7 @@ /* * xxHash - Extremely Fast Hash algorithm * Header File - * Copyright (C) 2012-2020 Yann Collet + * Copyright (C) 2012-2023 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * @@ -33,43 +33,213 @@ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ -/* TODO: update */ -/* Notice extracted from xxHash homepage: - -xxHash is an extremely fast hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -Note: SMHasher's CRC32 implementation is not the fastest one. -Other speed-oriented implementations can be faster, -especially in combination with PCLMUL instruction: -https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 - -A 64-bit version, named XXH64, is available since r35. -It offers much better speed, but for 64-bit applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ #if defined (__cplusplus) extern "C" { @@ -79,21 +249,80 @@ extern "C" { * INLINE mode ******************************/ /*! - * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * * Use these build macros to inline xxhash into the target unit. * Inlining improves performance on small inputs, especially when the length is * expressed as a compile-time constant: * - * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html * * It also keeps xxHash symbols private to the unit, so they are not exported. * * Usage: + * @code{.c} * #define XXH_INLINE_ALL * #include "xxhash.h" - * + * @endcode * Do not compile and link xxhash.o as a separate object, as it is not useful. */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ && !defined(XXH_INLINE_ALL_31684351384) /* this section should be traversed only once */ @@ -104,7 +333,7 @@ extern "C" { /* make all functions private */ # undef XXH_PUBLIC_API # if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# define XXH_PUBLIC_API static __inline __attribute__((__unused__)) # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # define XXH_PUBLIC_API static inline # elif defined(_MSC_VER) @@ -116,29 +345,80 @@ extern "C" { /* * This part deals with the special case where a unit wants to inline xxHash, - * but "xxhash.h" has previously been included without XXH_INLINE_ALL, such - * as part of some previously included *.h header file. + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. * Without further action, the new include would just be ignored, * and functions would effectively _not_ be inlined (silent failure). * The following macros solve this situation by prefixing all inlined names, * avoiding naming collision with previous inclusions. */ -# ifdef XXH_NAMESPACE -# error "XXH_INLINE_ALL with XXH_NAMESPACE is not supported" - /* - * Note: Alternative: #undef all symbols (it's a pretty large list). - * Without #error: it compiles, but functions are actually not inlined. - */ -# endif + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ # define XXH_NAMESPACE XXH_INLINE_ /* - * Some identifiers (enums, type names) are not symbols, but they must - * still be renamed to avoid redeclaration. + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. * Alternative solution: do not redeclare them. - * However, this requires some #ifdefs, and is a more dispersed action. - * Meanwhile, renaming can be achieved in a single block + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. */ -# define XXH_IPREF(Id) XXH_INLINE_ ## Id +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id # define XXH_OK XXH_IPREF(XXH_OK) # define XXH_ERROR XXH_IPREF(XXH_ERROR) # define XXH_errorcode XXH_IPREF(XXH_errorcode) @@ -157,17 +437,15 @@ extern "C" { # undef XXHASH_H_STATIC_13879238742 #endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ - - /* **************************************************************** * Stable API *****************************************************************/ #ifndef XXHASH_H_5627135585666179 #define XXHASH_H_5627135585666179 1 -/* specific declaration modes for Windows */ +/*! @brief Marks a global symbol. */ #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) -# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) # ifdef XXH_EXPORT # define XXH_PUBLIC_API __declspec(dllexport) # elif XXH_IMPORT @@ -178,23 +456,11 @@ extern "C" { # endif #endif -/*! - * XXH_NAMESPACE, aka Namespace Emulation: - * - * If you want to include _and expose_ xxHash functions from within your own - * library, but also want to avoid symbol collisions with other libraries which - * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix - * any public symbol from xxhash library with the value of XXH_NAMESPACE - * (therefore, avoid empty or numeric values). - * - * Note that no change is required within the calling program as long as it - * includes `xxhash.h`: Regular symbol names will be automatically translated - * by this header. - */ #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) @@ -204,6 +470,7 @@ extern "C" { # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) @@ -213,1753 +480,6759 @@ extern "C" { # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif #endif +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((__const__)) +# define XXH_PUREF __attribute__((__pure__)) +# define XXH_MALLOCF __attribute__((__malloc__)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif /* ************************************* * Version ***************************************/ #define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 7 -#define XXH_VERSION_RELEASE 4 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 3 +/*! @brief Version number, encoded as two digits each */ #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); /* **************************** -* Definitions +* Common basic types ******************************/ #include /* size_t */ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; /*-********************************************************************** * 32-bit hash ************************************************************************/ -#if !defined (__VMS) \ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include +# ifdef _AIX +# include +# else +# include +# endif typedef uint32_t XXH32_hash_t; + #else # include # if UINT_MAX == 0xFFFFFFFFUL typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; # else -# if ULONG_MAX == 0xFFFFFFFFUL - typedef unsigned long XXH32_hash_t; -# else -# error "unsupported platform: need a 32-bit type" -# endif +# error "unsupported platform: need a 32-bit type" # endif #endif /*! - * XXH32(): - * Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". - * The memory between input & input+length must be valid (allocated and read-accessible). - * "seed" can be used to alter the result predictably. - * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. * - * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ */ -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); -/******* Streaming *******/ +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); -/* - * Streaming functions generate the xxHash value from an incrememtal input. - * This method is slower than single-call functions, due to state management. - * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. * - * An XXH state must first be allocated using `XXH*_createState()`. + * @see XXH32_state_s for details. + * @see @ref streaming_example "Streaming Example" + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. * - * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. * - * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * @note Must be freed with XXH32_freeState(). * - * The function returns an error code, with 0 meaning OK, and any other value - * meaning there is an error. + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. * - * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. - * This function returns the nn-bits hash as an int or long long. + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). * - * It's still possible to continue inserting input into the hash state after a - * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + * @see @ref streaming_example "Streaming Example" * - * When done, release the state using `XXH*_freeState()`. */ - -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @see @ref streaming_example "Streaming Example" + */ XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); -/******* Canonical representation *******/ +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -/* - * The default return values from XXH functions are unsigned 32 and 64 bit - * integers. - * This the simplest and fastest format for further post-processing. +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. * - * However, this leaves open the question of what is the order on the byte level, - * since little and big endian conventions will store the same number differently. + * @param statePtr The state struct to calculate the hash from. * - * The canonical representation settles this issue by mandating big-endian - * convention, the same convention as human-readable numbers (large digits first). + * @pre + * @p statePtr must not be `NULL`. * - * When writing hash values to storage, sending them over a network, or printing - * them, it's highly recommended to use the canonical representation to ensure - * portability across a wider range of systems, present and future. + * @return The calculated 32-bit xxHash32 value from that state. * - * The following functions allow transformation of hash values to and from - * canonical format. + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); +/******* Canonical representation *******/ +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; -#ifndef XXH_NO_LONG_LONG -/*-********************************************************************** -* 64-bit hash -************************************************************************/ -#if !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint64_t XXH64_hash_t; -#else - /* the following type must have a width of 64-bit */ - typedef unsigned long long XXH64_hash_t; -#endif +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); /*! - * XXH64(): - * Returns the 64-bit hash of sequence of length @length stored at memory - * address @input. - * @seed can be used to alter the result predictably. + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. * - * This function usually runs faster on 64-bit systems, but slower on 32-bit - * systems (see benchmark). + * @pre + * @p src must not be `NULL`. * - * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, XXH64_hash_t seed); - -/******* Streaming *******/ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/******* Canonical representation *******/ -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -#endif /* XXH_NO_LONG_LONG */ +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ -#endif /* XXHASH_H_5627135585666179 */ +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ -#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) -#define XXHASH_H_STATIC_13879238742 -/* **************************************************************************** - * This section contains declarations which are not guaranteed to remain stable. - * They may change in future versions, becoming incompatible with a different - * version of the library. - * These declarations should only be used with static linking. - * Never use them in association with dynamic linking! - ***************************************************************************** */ +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ +/*! @cond Doxygen ignores this part */ /* - * These definitions are only present to allow static allocation of an XXH - * state, for example, on the stack or in a struct. - * Never **ever** access members directly. + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((__noescape__)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ -struct XXH32_state_s { - XXH32_hash_t total_len_32; - XXH32_hash_t large_len; - XXH32_hash_t v1; - XXH32_hash_t v2; - XXH32_hash_t v3; - XXH32_hash_t v4; - XXH32_hash_t mem32[4]; - XXH32_hash_t memsize; - XXH32_hash_t reserved; /* never read nor write, might be removed in a future version */ -}; /* typedef'd to XXH32_state_t */ +/*! + * @} + * @ingroup public + * @{ + */ -#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif -struct XXH64_state_s { - XXH64_hash_t total_len; - XXH64_hash_t v1; - XXH64_hash_t v2; - XXH64_hash_t v3; - XXH64_hash_t v4; - XXH64_hash_t mem64[4]; - XXH32_hash_t memsize; - XXH32_hash_t reserved32; /* required for padding anyway */ - XXH64_hash_t reserved64; /* never read nor write, might be removed in a future version */ -}; /* typedef'd to XXH64_state_t */ +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); -/*-********************************************************************** -* XXH3 -* New experimental hash -************************************************************************/ +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + * @see @ref streaming_example "Streaming Example" + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -/* ************************************************************************ - * XXH3 is a new hash algorithm featuring: - * - Improved speed for both small and large inputs - * - True 64-bit and 128-bit outputs - * - SIMD acceleration - * - Improved 32-bit viability +/*! + * @brief Allocates an @ref XXH64_state_t. * - * Speed analysis methodology is explained here: + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. * - * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * @note Must be freed with XXH64_freeState(). * - * In general, expect XXH3 to run about ~2x faster on large inputs and >3x - * faster on small ones compared to XXH64, though exact differences depend on - * the platform. + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. * - * The algorithm is portable: Like XXH32 and XXH64, it generates the same hash - * on all platforms. + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). * - * It benefits greatly from SIMD and 64-bit arithmetic, but does not require it. + * @return @ref XXH_OK. * - * Almost all 32-bit and 64-bit targets that can run XXH32 smoothly can run - * XXH3 at competitive speeds, even if XXH64 runs slowly. Further details are - * explained in the implementation. + * @note @p statePtr must be allocated with XXH64_createState(). * - * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, - * ZVector and scalar targets. This can be controlled with the XXH_VECTOR macro. + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. * - * XXH3 offers 2 variants, _64bits and _128bits. - * When only 64 bits are needed, prefer calling the _64bits variant, as it - * reduces the amount of mixing, resulting in faster speed on small inputs. + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. * - * It's also generally simpler to manipulate a scalar return type than a struct. + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. * - * The 128-bit version adds additional strength, but it is slightly slower. + * @pre + * @p statePtr must not be `NULL`. * - * The XXH3 algorithm is still in development. - * The results it produces may still change in future versions. + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. * - * Results produced by v0.7.x are not comparable with results from v0.7.y. - * However, the API is completely stable, and it can safely be used for - * ephemeral data (local sessions). + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). * - * Avoid storing values in long-term storage until the algorithm is finalized. + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. * - * Since v0.7.3, XXH3 has reached "release candidate" status, meaning that, if - * everything remains fine, its current format will be "frozen" and become the - * final one. + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. * - * After which, return values of XXH3 and XXH128 will no longer change in - * future versions. + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * XXH3's return values will be officially finalized upon reaching v0.8.0. + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. * - * The API supports one-shot hashing, streaming mode, and custom secrets. + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); -#ifdef XXH_NAMESPACE -# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) -# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) -# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ -# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) -# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) -# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; -# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) -# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) -# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) -# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) -# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) -#endif +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); -/* XXH3_64bits(): - * default 64-bit variant, using default secret and default seed of 0. - * It's the fastest variant. */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); +#ifndef XXH_NO_XXH3 -/* - * XXH3_64bits_withSecret(): - * It's possible to provide any blob of bytes as a "secret" to generate the hash. - * This makes it more difficult for an external actor to prepare an intentional - * collision. - * The secret *must* be large enough (>= XXH3_SECRET_SIZE_MIN). - * It should consist of random bytes. - * Avoid trivial sequences, such as repeating sequences and especially '\0', - * as this can cancel out itself. - * Failure to respect these conditions will result in a poor quality hash. +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. */ -#define XXH3_SECRET_SIZE_MIN 136 -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); -/* - * XXH3_64bits_withSeed(): - * This variant generates a custom secret on the fly based on the default - * secret, altered using the `seed` value. - * While this operation is decently fast, note that it's not completely free. - * Note: seed==0 produces the same results as XXH3_64bits(). +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Unless set explicitly, determined automatically. */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +# define XXH_SCALAR 0 /*!< Portable scalar version */ +# define XXH_SSE2 1 /*!< SSE2 for Pentium 4, Opteron, all x86_64. */ +# define XXH_AVX2 2 /*!< AVX2 for Haswell and Bulldozer */ +# define XXH_AVX512 3 /*!< AVX512 for Skylake and Icelake */ +# define XXH_NEON 4 /*!< NEON for most ARMv7-A, all AArch64, and WASM SIMD128 */ +# define XXH_VSX 5 /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +# define XXH_SVE 6 /*!< SVE for some ARMv8-A and ARMv9-A */ +# define XXH_LSX 7 /*!< LSX (128-bit SIMD) for LoongArch64 */ -/* streaming 64-bit */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11+ */ -# include -# define XXH_ALIGN(n) alignas(n) -#elif defined(__GNUC__) -# define XXH_ALIGN(n) __attribute__ ((aligned(n))) -#elif defined(_MSC_VER) -# define XXH_ALIGN(n) __declspec(align(n)) -#else -# define XXH_ALIGN(n) /* disabled */ -#endif +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); -/* Old GCC versions only accept the attribute after the type in structures. */ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ - && defined(__GNUC__) -# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) -#else -# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type -#endif +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); -typedef struct XXH3_state_s XXH3_state_t; +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 -#define XXH3_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ -#define XXH3_INTERNALBUFFER_SIZE 256 -struct XXH3_state_s { - XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); - /* used to store a custom secret generated from the seed. Makes state larger. - * Design might change */ - XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); - XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); - XXH32_hash_t bufferedSize; - XXH32_hash_t nbStripesPerBlock; - XXH32_hash_t nbStripesSoFar; - XXH32_hash_t secretLimit; - XXH32_hash_t reserved32; - XXH32_hash_t reserved32_2; - XXH64_hash_t totalLen; - XXH64_hash_t seed; - XXH64_hash_t reserved64; - /* note: there is some padding after due to alignment on 64 bytes */ - const unsigned char* secret; -}; /* typedef'd to XXH3_state_t */ +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); -#undef XXH_ALIGN_MEMBER +/******* Streaming *******/ +#ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. * As a consequence, streaming is slower than one-shot hashing. - * For better performance, prefer one-shot functions whenever possible. + * For better performance, prefer one-shot functions whenever applicable. */ -XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); -XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); - -/* - * XXH3_64bits_reset(): - * Initialize with the default parameters. - * The result will be equivalent to `XXH3_64bits()`. - */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); -/* - * XXH3_64bits_reset_withSeed(): - * Generate a custom secret from `seed`, and store it into `statePtr`. - * digest will be equivalent to `XXH3_64bits_withSeed()`. +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); -/* - * XXH3_64bits_reset_withSecret(): - * `secret` is referenced, and must outlive the hash streaming session, so - * be careful when using stack arrays. - * `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`. +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + * @see @ref streaming_example "Streaming Example" + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); -/* 128-bit */ +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); -#ifdef XXH_NAMESPACE -# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) -# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) -# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) -# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); -# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) -# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) -# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) -# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) -# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ -# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) -# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) -# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) -# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) -#endif +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ typedef struct { - XXH64_hash_t low64; - XXH64_hash_t high64; + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ } XXH128_hash_t; -XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); /* == XXH128() */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ -/* Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ /*! - * XXH128_isEqual(): - * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. */ -XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); /*! - * XXH128_cmp(): + * @brief Compares two @ref XXH128_hash_t * * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. * - * return: >0 if *h128_1 > *h128_2 - * <0 if *h128_1 < *h128_2 - * =0 if *h128_1 == *h128_2 + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 */ -XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); /******* Canonical representation *******/ -typedef struct { unsigned char digest[16]; } XXH128_canonical_t; -XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); -XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); +#endif /* !XXH_NO_XXH3 */ #endif /* XXH_NO_LONG_LONG */ -#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) -# define XXH_IMPLEMENTATION +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t acc[4]; /*!< Accumulator lanes */ + unsigned char buffer[16]; /*!< Internal buffer for partial reads. */ + XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t acc[4]; /*!< Accumulator lanes */ + unsigned char buffer[32]; /*!< Internal buffer for partial reads.. */ + XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# define XXH_ALIGN(n) _Alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ #endif -#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 -/* ======================================================================== */ -/* ======================================================================== */ -/* ======================================================================== */ +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ +#undef XXH_ALIGN_MEMBER -/*-********************************************************************** - * xxHash implementation - *-********************************************************************** - * xxHash's implementation used to be found in xxhash.c. +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. * - * However, code inlining requires the implementation to be visible to the - * compiler, usually within the header. + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. * - * As a workaround, xxhash.c used to be included within xxhash.h. This caused - * some issues with some build systems, especially ones which treat .c files - * as source files. + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * Therefore, the implementation is now directly integrated within xxhash.h. - * Another small advantage is that xxhash.c is no longer needed in /include. - ************************************************************************/ + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); -#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ - || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) -# define XXH_IMPLEM_13a8737387 -/* ************************************* -* Tuning parameters -***************************************/ +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + /*! - * XXH_FORCE_MEMORY_ACCESS: - * By default, access to unaligned memory is controlled by `memcpy()`, which is - * safe and portable. + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. * - * Unfortunately, on some target/compiler combinations, the generated assembly - * is sub-optimal. + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_SIZE_MIN. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. * - * The below switch allow to select a different access method for improved - * performance. - * Method 0 (default): - * Use `memcpy()`. Safe and portable. - * Method 1: - * `__attribute__((packed))` statement. It depends on compiler extensions - * and is therefore not portable. - * This method is safe if your compiler supports it, and *generally* as - * fast or faster than `memcpy`. - * Method 2: - * Direct access via cast. This method doesn't depend on the compiler but - * violates the C standard. - * It can generate buggy code on targets which do not support unaligned - * memory accesses. - * But in some circumstances, it's the only known way to get the most - * performance (ie GCC + ARMv6) - * Method 3: - * Byteshift. This can generate the best code on old compilers which don't - * inline small `memcpy()` calls, and it might also be faster on big-endian - * systems which lack a native byteswap instruction. - * See https://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2 > 3) + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if !defined(__clang__) && defined(__GNUC__) && defined(__ARM_FEATURE_UNALIGNED) && defined(__ARM_ARCH) && (__ARM_ARCH == 6) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif !defined(__clang__) && ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && (defined(__ARM_ARCH) && __ARM_ARCH >= 7))) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_DEFAULT_SIZE bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_DEFAULT_SIZE]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either: + * - @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * - @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); + +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The memory segment to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret used to alter hash result predictably. + * @param secretSize The length of @p secret, in bytes (must be >= XXH3_SECRET_SIZE_MIN) + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed(): contract is the same. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed(). Contract is identical. + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed(). Contract is identical. + * + * Note: there was a bug in an earlier version of this function (<= v0.8.2) + * that would make it generate an incorrect hash value + * when @p seed == 0 and @p length < XXH3_MIDSIZE_MAX + * and @p secret is different from XXH3_generateSecret_fromSeed(). + * As stated in the contract, the correct hash result must be + * the same as XXH3_128bits_withSeed() when @p length <= XXH3_MIDSIZE_MAX. + * Results generated by this older version are wrong, hence not comparable. + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((__unused__)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((__always_inline__, __unused__)) +# define XXH_NO_INLINE static __attribute__((__noinline__)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if defined(XXH_INLINE_ALL) +# define XXH_STATIC XXH_FORCE_INLINE +#else +# define XXH_STATIC static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((__packed__)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((__aligned__(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +#elif XXH_HAS_BUILTIN(__builtin_stdc_rotate_left) +# define XXH_rotl32 __builtin_stdc_rotate_left +# define XXH_rotl64 __builtin_stdc_rotate_left +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Sets up the initial accumulator state for XXH32(). + */ +XXH_FORCE_INLINE void +XXH32_initAccs(xxh_u32 *acc, xxh_u32 seed) +{ + XXH_ASSERT(acc != NULL); + acc[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + acc[1] = seed + XXH_PRIME32_2; + acc[2] = seed + 0; + acc[3] = seed - XXH_PRIME32_1; +} + +/*! + * @internal + * @brief Consumes a block of data for XXH32(). + * + * @return the end input pointer. + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH32_consumeLong( + xxh_u32 *XXH_RESTRICT acc, + xxh_u8 const *XXH_RESTRICT input, + size_t len, + XXH_alignment align +) +{ + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + XXH_ASSERT(acc != NULL); + XXH_ASSERT(input != NULL); + XXH_ASSERT(len >= 16); + do { + acc[0] = XXH32_round(acc[0], XXH_get32bits(input)); input += 4; + acc[1] = XXH32_round(acc[1], XXH_get32bits(input)); input += 4; + acc[2] = XXH32_round(acc[2], XXH_get32bits(input)); input += 4; + acc[3] = XXH32_round(acc[3], XXH_get32bits(input)); input += 4; + } while (input < limit); + + return input; +} + +/*! + * @internal + * @brief Merges the accumulator lanes together for XXH32() + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_mergeAccs(const xxh_u32 *acc) +{ + XXH_ASSERT(acc != NULL); + return XXH_rotl32(acc[0], 1) + XXH_rotl32(acc[1], 7) + + XXH_rotl32(acc[2], 12) + XXH_rotl32(acc[3], 18); +} + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + xxh_u32 acc[4]; + XXH32_initAccs(acc, seed); + + input = XXH32_consumeLong(acc, input, len, align); + + h32 = XXH32_mergeAccs(acc); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + XXH32_initAccs(statePtr->acc, seed); + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + XXH_ASSERT(state->bufferedSize < sizeof(state->buffer)); + if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */ + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + { const xxh_u8* xinput = (const xxh_u8*)input; + const xxh_u8* const bEnd = xinput + len; + + if (state->bufferedSize) { /* non-empty buffer: complete first */ + XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize); + xinput += sizeof(state->buffer) - state->bufferedSize; + /* then process one round */ + (void)XXH32_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned); + state->bufferedSize = 0; + } + + XXH_ASSERT(xinput <= bEnd); + if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) { + /* Process the remaining data */ + xinput = XXH32_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned); + } + + if (xinput < bEnd) { + /* Copy the leftover to the tmp buffer */ + XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput)); + state->bufferedSize = (unsigned)(bEnd-xinput); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH32_mergeAccs(state->acc); + } else { + h32 = state->acc[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, state->buffer, state->bufferedSize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((__packed__)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((__aligned__(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Sets up the initial accumulator state for XXH64(). + */ +XXH_FORCE_INLINE void +XXH64_initAccs(xxh_u64 *acc, xxh_u64 seed) +{ + XXH_ASSERT(acc != NULL); + acc[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + acc[1] = seed + XXH_PRIME64_2; + acc[2] = seed + 0; + acc[3] = seed - XXH_PRIME64_1; +} + +/*! + * @internal + * @brief Consumes a block of data for XXH64(). + * + * @return the end input pointer. + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH64_consumeLong( + xxh_u64 *XXH_RESTRICT acc, + xxh_u8 const *XXH_RESTRICT input, + size_t len, + XXH_alignment align +) +{ + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + XXH_ASSERT(acc != NULL); + XXH_ASSERT(input != NULL); + XXH_ASSERT(len >= 32); + do { + /* reroll on 32-bit */ + if (sizeof(void *) < sizeof(xxh_u64)) { + size_t i; + for (i = 0; i < 4; i++) { + acc[i] = XXH64_round(acc[i], XXH_get64bits(input)); + input += 8; + } + } else { + acc[0] = XXH64_round(acc[0], XXH_get64bits(input)); input += 8; + acc[1] = XXH64_round(acc[1], XXH_get64bits(input)); input += 8; + acc[2] = XXH64_round(acc[2], XXH_get64bits(input)); input += 8; + acc[3] = XXH64_round(acc[3], XXH_get64bits(input)); input += 8; + } + } while (input < limit); + + return input; +} + +/*! + * @internal + * @brief Merges the accumulator lanes together for XXH64() + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_mergeAccs(const xxh_u64 *acc) +{ + XXH_ASSERT(acc != NULL); + { + xxh_u64 h64 = XXH_rotl64(acc[0], 1) + XXH_rotl64(acc[1], 7) + + XXH_rotl64(acc[2], 12) + XXH_rotl64(acc[3], 18); + /* reroll on 32-bit */ + if (sizeof(void *) < sizeof(xxh_u64)) { + size_t i; + for (i = 0; i < 4; i++) { + h64 = XXH64_mergeRound(h64, acc[i]); + } + } else { + h64 = XXH64_mergeRound(h64, acc[0]); + h64 = XXH64_mergeRound(h64, acc[1]); + h64 = XXH64_mergeRound(h64, acc[2]); + h64 = XXH64_mergeRound(h64, acc[3]); + } + return h64; + } +} + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +XXH_STATIC XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { /* Process a large block of data */ + xxh_u64 acc[4]; + XXH64_initAccs(acc, seed); + + input = XXH64_consumeLong(acc, input, len, align); + + h64 = XXH64_mergeAccs(acc); + } else { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (xxh_u64) len; + + return XXH64_finalize(h64, input, len, align); +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + XXH64_initAccs(statePtr->acc, seed); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + state->total_len += len; + + XXH_ASSERT(state->bufferedSize <= sizeof(state->buffer)); + if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */ + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + { const xxh_u8* xinput = (const xxh_u8*)input; + const xxh_u8* const bEnd = xinput + len; + + if (state->bufferedSize) { /* non-empty buffer => complete first */ + XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize); + xinput += sizeof(state->buffer) - state->bufferedSize; + /* and process one round */ + (void)XXH64_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned); + state->bufferedSize = 0; + } + + XXH_ASSERT(xinput <= bEnd); + if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) { + /* Process the remaining data */ + xinput = XXH64_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned); + } + + if (xinput < bEnd) { + /* Copy the leftover to the tmp buffer */ + XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput)); + state->bufferedSize = (unsigned)(bEnd-xinput); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH64_mergeAccs(state->acc); + } else { + h64 = state->acc[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, state->buffer, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# elif defined(__loongarch_sx) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# elif defined(__loongarch_sx) +# define XXH_VECTOR XXH_LSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_LSX /* lsx */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((__may_alias__)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + #endif -/*! - *XXH_ACCEPT_NULL_INPUT_POINTER: - * If the input pointer is NULL, xxHash's default behavior is to dereference it, - * triggering a segfault. - * When this macro is enabled, xxHash actively checks the input for a null pointer. - * If it is, the result for null input pointers is the same as a zero-length input. - */ -#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ -# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ #endif -/*! - * XXH_FORCE_ALIGN_CHECK: - * This is a minor performance trick, only useful with lots of very small keys. - * It means: check for aligned/unaligned input. - * The check costs one initial branch per hash; - * Set it to 0 when the input is guaranteed to be aligned or when alignment - * doesn't matter for performance. - * - * This option does not affect XXH3. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + #endif +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + /*! - * XXH_NO_INLINE_HINTS: + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. * - * By default, xxHash tries to force the compiler to inline almost all internal - * functions. + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. * - * This can usually improve performance due to reduced jumping and improved - * constant folding, but significantly increases the size of the binary which - * might not be favorable. + * @see XXH3_NEON_LANES for configuring this and details about this optimization. * - * Additionally, sometimes the forced inlining can be detrimental to performance, - * depending on the architecture. + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. * - * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the - * compiler full control on whether to inline or not. + * Additionally, there is an optimization for 4 lanes at once noted below. * - * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using - * -fno-inline with GCC or Clang, this will automatically be defined. - */ -#ifndef XXH_NO_INLINE_HINTS -# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ - || defined(__NO_INLINE__) /* -O0, -fno-inline */ -# define XXH_NO_INLINE_HINTS 1 -# else -# define XXH_NO_INLINE_HINTS 0 -# endif -#endif - -/*! - * XXH_REROLL: - * Whether to reroll XXH32_finalize, and XXH64_finalize, - * instead of using an unrolled jump table/if statement loop. + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. * - * This is automatically defined on -Os/-Oz on GCC and Clang. - */ -#ifndef XXH_REROLL -# if defined(__OPTIMIZE_SIZE__) -# define XXH_REROLL 1 -# else -# define XXH_REROLL 0 -# endif -#endif - - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/*! - * Modify the local functions below should you wish to use some other memory - * routines for malloc() and free() + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. */ -#include - -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free(void* p) { free(p); } -/*! and for memcpy() */ -#include -static void* XXH_memcpy(void* dest, const void* src, size_t size) +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) { - return memcpy(dest,src,size); + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } } +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) -#include /* ULLONG_MAX */ - - -/* ************************************* -* Compiler Specific Options -***************************************/ -#ifdef _MSC_VER /* Visual Studio warning fix */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); #endif -#if XXH_NO_INLINE_HINTS /* disable inlining hints */ -# define XXH_FORCE_INLINE static -# define XXH_NO_INLINE static -#elif defined(_MSC_VER) /* Visual Studio */ -# define XXH_FORCE_INLINE static __forceinline -# define XXH_NO_INLINE static __declspec(noinline) + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; #else -# if defined (__cplusplus) \ - || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define XXH_FORCE_INLINE static inline __attribute__((always_inline)) -# define XXH_NO_INLINE static __attribute__((noinline)) -# else -# define XXH_FORCE_INLINE static inline -# define XXH_NO_INLINE static -# endif -# else -# define XXH_FORCE_INLINE static -# define XXH_NO_INLINE static -# endif /* __STDC_VERSION__ */ + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} #endif +#if (XXH_VECTOR == XXH_VSX) - -/* ************************************* -* Debug -***************************************/ -/* - * DEBUGLEVEL is expected to be defined externally, typically via the compiler's - * command line options. The value must be a number. - */ -#ifndef DEBUGLEVEL -# define DEBUGLEVEL 0 +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); #endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} -#if (DEBUGLEVEL>=1) -# include /* note: can still be disabled with NDEBUG */ -# define XXH_ASSERT(c) assert(c) -#else -# define XXH_ASSERT(c) ((void)0) #endif -/* note: use after variable declarations */ -#define XXH_STATIC_ASSERT(c) do { enum { XXH_sa = 1/(int)(!!(c)) }; } while (0) +#if (XXH_VECTOR == XXH_SVE) +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} -/* ************************************* -* Basic Types -***************************************/ -#if !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t xxh_u8; -#else - typedef unsigned char xxh_u8; #endif -typedef XXH32_hash_t xxh_u32; +#if (XXH_VECTOR == XXH_LSX) +#define _LSX_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w)) -/* *** Memory access *** */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_lsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + __m128i* const xacc = (__m128i *) acc; + const __m128i* const xinput = (const __m128i *) input; + const __m128i* const xsecret = (const __m128i *) secret; + + for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = __lsx_vld(xinput + i, 0); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = __lsx_vld(xsecret + i, 0); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = __lsx_vxor_v(data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32); + // __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = __lsx_vmulwev_d_wu(data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = __lsx_vshuf4i_w(data_vec, _LSX_SHUFFLE(1, 0, 3, 2)); + __m128i const sum = __lsx_vadd_d(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = __lsx_vadd_d(product, sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(lsx) -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) -/* - * Manual byteshift. Best for old compilers which don't inline memcpy. - * We actually directly use XXH_readLE32 and XXH_readBE32. - */ -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) +XXH_FORCE_INLINE void +XXH3_scrambleAcc_lsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + __m128i* const xacc = (__m128i*) acc; + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = __lsx_vreplgr2vr_w((int)XXH_PRIME32_1); + + for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = __lsx_vsrli_d(acc_vec, 47); + __m128i const data_vec = __lsx_vxor_v(acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = __lsx_vld(xsecret + i, 0); + __m128i const data_key = __lsx_vxor_v(data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = __lsx_vsrli_d(data_key, 32); + __m128i const prod_lo = __lsx_vmulwev_d_wu(data_key, prime32); + __m128i const prod_hi = __lsx_vmulwev_d_wu(data_key_hi, prime32); + xacc[i] = __lsx_vadd_d(prod_lo, __lsx_vslli_d(prod_hi, 32)); + } + } +} -/* - * Force direct memory access. Only works on CPU which support unaligned memory - * access in hardware. - */ -static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } +#endif -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) +/* scalar variants - universal */ +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) /* - * __pack instructions are safer but compiler specific, hence potentially - * problematic for some compilers. + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). * - * Currently only defined for GCC and ICC. + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. */ -typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; -static xxh_u32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } - +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} #else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif -/* - * Portable and safe solution. Generally efficient. - * see: https://stackoverflow.com/a/32095106/646947 +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. */ -static xxh_u32 XXH_read32(const void* memPtr) +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) { - xxh_u32 val; - memcpy(&val, memPtr, sizeof(val)); - return val; + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } } -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* *** Endianess *** */ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) /*! - * XXH_CPU_LITTLE_ENDIAN: - * Defined to 1 if the target is little endian, or 0 if it is big endian. - * It can be defined externally, for example on the compiler command line. + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). * - * If it is not defined, a runtime check (which is usually constant folded) - * is used instead. + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. */ -#ifndef XXH_CPU_LITTLE_ENDIAN -/* - * Try to detect endianness automatically, to avoid the nonstandard behavior - * in `XXH_isLittleEndian()` +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read */ -# if defined(_WIN32) /* Windows is always little endian */ \ - || defined(__LITTLE_ENDIAN__) \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -# define XXH_CPU_LITTLE_ENDIAN 1 -# elif defined(__BIG_ENDIAN__) \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -# define XXH_CPU_LITTLE_ENDIAN 0 -# else -static int XXH_isLittleEndian(void) +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { /* - * Nonstandard, but well-defined behavior in practice. - * Don't use static: it is detrimental to performance. + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. */ - const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; - return one.c[0]; -} -# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() -# endif + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); #endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_LSX) +#define XXH3_accumulate_512 XXH3_accumulate_512_lsx +#define XXH3_accumulate XXH3_accumulate_lsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_lsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -#ifndef __has_builtin -# define __has_builtin(x) 0 #endif -#if !defined(NO_CLANG_BUILTIN) && __has_builtin(__builtin_rotateleft32) \ - && __has_builtin(__builtin_rotateleft64) -# define XXH_rotl32 __builtin_rotateleft32 -# define XXH_rotl64 __builtin_rotateleft64 -/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ -#elif defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) -# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #endif -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static xxh_u32 XXH_swap32 (xxh_u32 x) +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); } -#endif +static XXH_PUREF XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + return XXH3_avalanche(result64); +} -/* - * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. - * - * This is ideal for older compilers which don't inline memcpy. - */ -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 -XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +static XXH_PUREF XXH64_hash_t +XXH3_finalizeLong_64b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 len) { - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[0] - | ((xxh_u32)bytePtr[1] << 8) - | ((xxh_u32)bytePtr[2] << 16) - | ((xxh_u32)bytePtr[3] << 24); + return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, len * XXH_PRIME64_1); } -XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) { - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[3] - | ((xxh_u32)bytePtr[2] << 8) - | ((xxh_u32)bytePtr[1] << 16) - | ((xxh_u32)bytePtr[0] << 24); + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_finalizeLong_64b(acc, (const xxh_u8*)secret, (xxh_u64)len); } -#else -XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) { - return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); } -static xxh_u32 XXH_readBE32(const void* ptr) +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) { - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); } -#endif -XXH_FORCE_INLINE xxh_u32 -XXH_readLE32_align(const void* ptr, XXH_alignment align) +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) { - if (align==XXH_unaligned) { - return XXH_readLE32(ptr); - } else { - return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); } } - -/* ************************************* -* Misc -***************************************/ -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} -/* ******************************************************************* -* 32-bit hash functions -*********************************************************************/ -static const xxh_u32 PRIME32_1 = 0x9E3779B1U; /* 0b10011110001101110111100110110001 */ -static const xxh_u32 PRIME32_2 = 0x85EBCA77U; /* 0b10000101111010111100101001110111 */ -static const xxh_u32 PRIME32_3 = 0xC2B2AE3DU; /* 0b11000010101100101010111000111101 */ -static const xxh_u32 PRIME32_4 = 0x27D4EB2FU; /* 0b00100111110101001110101100101111 */ -static const xxh_u32 PRIME32_5 = 0x165667B1U; /* 0b00010110010101100110011110110001 */ +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); -static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) { - acc += input * PRIME32_2; - acc = XXH_rotl32(acc, 13); - acc *= PRIME32_1; -#if defined(__GNUC__) && defined(__SSE4_1__) && !defined(XXH_ENABLE_AUTOVECTORIZE) + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); /* - * UGLY HACK: - * This inline assembly hack forces acc into a normal register. This is the - * only thing that prevents GCC and Clang from autovectorizing the XXH32 - * loop (pragmas and attributes don't work for some resason) without globally - * disabling SSE4.1. - * - * The reason we want to avoid vectorization is because despite working on - * 4 integers at a time, there are multiple factors slowing XXH32 down on - * SSE4: - * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on - * newer chips!) making it slightly slower to multiply four integers at - * once compared to four integers independently. Even when pmulld was - * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE - * just to multiply unless doing a long operation. - * - * - Four instructions are required to rotate, - * movqda tmp, v // not required with VEX encoding - * pslld tmp, 13 // tmp <<= 13 - * psrld v, 19 // x >>= 19 - * por v, tmp // x |= tmp - * compared to one for scalar: - * roll v, 13 // reliably fast across the board - * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason - * - * - Instruction level parallelism is actually more beneficial here because - * the SIMD actually serializes this operation: While v1 is rotating, v2 - * can load data, while v3 can multiply. SSE forces them to operate - * together. - * - * How this hack works: - * __asm__("" // Declare an assembly block but don't declare any instructions - * : // However, as an Input/Output Operand, - * "+r" // constrain a read/write operand (+) as a general purpose register (r). - * (acc) // and set acc as the operand - * ); - * - * Because of the 'r', the compiler has promised that seed will be in a - * general purpose register and the '+' says that it will be 'read/write', - * so it has to assume it has changed. It is like volatile without all the - * loads and stores. - * - * Since the argument has to be in a normal register (not an SSE register), - * each time XXH32_round is called, it is impossible to vectorize. + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. */ - __asm__("" : "+r" (acc)); -#endif - return acc; + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); } -/* mix all bits */ -static xxh_u32 XXH32_avalanche(xxh_u32 h32) + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) { - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - return(h32); + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); } -#define XXH_get32bits(p) XXH_readLE32_align(p, align) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} -static xxh_u32 -XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) { -#define PROCESS1 do { \ - h32 += (*ptr++) * PRIME32_5; \ - h32 = XXH_rotl32(h32, 11) * PRIME32_1; \ -} while (0) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} -#define PROCESS4 do { \ - h32 += XXH_get32bits(ptr) * PRIME32_3; \ - ptr += 4; \ - h32 = XXH_rotl32(h32, 17) * PRIME32_4; \ -} while (0) +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} - /* Compact rerolled version */ - if (XXH_REROLL) { - len &= 15; - while (len >= 4) { - PROCESS4; - len -= 4; - } - while (len > 0) { - PROCESS1; - --len; - } - return XXH32_avalanche(h32); - } else { - switch(len&15) /* or switch(bEnd - p) */ { - case 12: PROCESS4; - /* fallthrough */ - case 8: PROCESS4; - /* fallthrough */ - case 4: PROCESS4; - return XXH32_avalanche(h32); - - case 13: PROCESS4; - /* fallthrough */ - case 9: PROCESS4; - /* fallthrough */ - case 5: PROCESS4; - PROCESS1; - return XXH32_avalanche(h32); - - case 14: PROCESS4; - /* fallthrough */ - case 10: PROCESS4; - /* fallthrough */ - case 6: PROCESS4; - PROCESS1; - PROCESS1; - return XXH32_avalanche(h32); - - case 15: PROCESS4; - /* fallthrough */ - case 11: PROCESS4; - /* fallthrough */ - case 7: PROCESS4; - /* fallthrough */ - case 3: PROCESS1; - /* fallthrough */ - case 2: PROCESS1; - /* fallthrough */ - case 1: PROCESS1; - /* fallthrough */ - case 0: return XXH32_avalanche(h32); + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to @align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; } - XXH_ASSERT(0); - return h32; /* reaching this point is deemed impossible */ + return NULL; } } - -XXH_FORCE_INLINE xxh_u32 -XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) { - const xxh_u8* bEnd = input + len; - xxh_u32 h32; - -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - if (input==NULL) { - len=0; - bEnd=input=(const xxh_u8*)(size_t)16; - } -#endif - - if (len>=16) { - const xxh_u8* const limit = bEnd - 15; - xxh_u32 v1 = seed + PRIME32_1 + PRIME32_2; - xxh_u32 v2 = seed + PRIME32_2; - xxh_u32 v3 = seed + 0; - xxh_u32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; - v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; - v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; - v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; - } while (input < limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) - + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); } - - h32 += (xxh_u32)len; - - return XXH32_finalize(h32, input, len&15, align); } - - -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) { -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, (const xxh_u8*)input, len); - return XXH32_digest(&state); - -#else - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); - } } - - return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); -#endif + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; } +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} -/******* Hash streaming *******/ +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) { - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; } -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) { - XXH_free(statePtr); + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) { - memcpy(dstState, srcState, sizeof(*dstState)); + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; } -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) { - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - /* do not write into reserved, planned to be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ return XXH_OK; } +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; -XXH_PUBLIC_API XXH_errorcode -XXH32_update(XXH32_state_t* state, const void* input, size_t len) + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) { - if (input==NULL) -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (input==NULL) { + XXH_ASSERT(len == 0); return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); #else - return XXH_ERROR; + xxh_u64* XXH_RESTRICT const acc = state->acc; #endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); - { const xxh_u8* p = (const xxh_u8*)input; - const xxh_u8* const bEnd = p + len; - - state->total_len_32 += (XXH32_hash_t)len; - state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); - state->memsize += (XXH32_hash_t)len; + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; return XXH_OK; } - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const xxh_u32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32)); - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const xxh_u8* const limit = bEnd - 16; - xxh_u32 v1 = state->v1; - xxh_u32 v2 = state->v2; - xxh_u32 v3 = state->v3; - xxh_u32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif } return XXH_OK; } +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* state) +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) { - xxh_u32 h32; + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) - + XXH_rotl32(state->v2, 7) - + XXH_rotl32(state->v3, 12) - + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} - h32 += state->total_len_32; +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_finalizeLong_64b(acc, secret, (xxh_u64)state->totalLen); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ - return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } } +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); -/******* Canonical representation *******/ + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} /* - * The default return values from XXH functions are unsigned 32 and 64 bit - * integers. - * - * The canonical representation uses big endian convention, the same convention - * as human-readable numbers (large digits first). - * - * This way, hash values can be written into a file or buffer, remaining - * comparable across different systems. - * - * The following functions allow transformation of hash values to and from their - * canonical format. + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN */ -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } } -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) { - return XXH_readBE32(src); + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; } -#ifndef XXH_NO_LONG_LONG +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); -/* ******************************************************************* -* 64-bit hash functions -*********************************************************************/ + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; -/******* Memory access *******/ +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} -typedef XXH64_hash_t xxh_u64; +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} +static XXH_PUREF XXH128_hash_t +XXH3_finalizeLong_128b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, xxh_u64 len) +{ + XXH128_hash_t h128; + h128.low64 = XXH3_finalizeLong_64b(acc, secret, len); + h128.high64 = XXH3_mergeAccs(acc, secret + secretSize + - XXH_STRIPE_LEN - XXH_SECRET_MERGEACCS_START, + ~(len * XXH_PRIME64_2)); + return h128; +} -/*! - * XXH_REROLL_XXH64: - * Whether to reroll the XXH64_finalize() loop. - * - * Just like XXH32, we can unroll the XXH64_finalize() loop. This can be a - * performance gain on 64-bit hosts, as only one jump is required. - * - * However, on 32-bit hosts, because arithmetic needs to be done with two 32-bit - * registers, and 64-bit arithmetic needs to be simulated, it isn't beneficial - * to unroll. The code becomes ridiculously large (the largest function in the - * binary on i386!), and rerolling it saves anywhere from 3kB to 20kB. It is - * also slightly faster because it fits into cache better and is more likely - * to be inlined by the compiler. - * - * If XXH_REROLL is defined, this is ignored and the loop is always rerolled. - */ -#ifndef XXH_REROLL_XXH64 -# if (defined(__ILP32__) || defined(_ILP32)) /* ILP32 is often defined on 32-bit GCC family */ \ - || !(defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) /* x86-64 */ \ - || defined(_M_ARM64) || defined(__aarch64__) || defined(__arm64__) /* aarch64 */ \ - || defined(__PPC64__) || defined(__PPC64LE__) || defined(__ppc64__) || defined(__powerpc64__) /* ppc64 */ \ - || defined(__mips64__) || defined(__mips64)) /* mips64 */ \ - || (!defined(SIZE_MAX) || SIZE_MAX < ULLONG_MAX) /* check limits */ -# define XXH_REROLL_XXH64 1 -# else -# define XXH_REROLL_XXH64 0 -# endif -#endif /* !defined(XXH_REROLL_XXH64) */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) -/* - * Manual byteshift. Best for old compilers which don't inline memcpy. - * We actually directly use XXH_readLE64 and XXH_readBE64. - */ -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static xxh_u64 XXH_read64(const void* memPtr) { return *(const xxh_u64*) memPtr; } + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_finalizeLong_128b(acc, secret, secretSize, (xxh_u64)len); +} -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} /* - * __pack instructions are safer, but compiler specific, hence potentially - * problematic for some compilers. + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. * - * Currently only defined for GCC and ICC. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. */ -typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; -static xxh_u64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} -#else +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} /* - * Portable and safe solution. Generally efficient. - * see: https://stackoverflow.com/a/32095106/646947 + * It's important for performance that XXH3_hashLong is not inlined. */ -static xxh_u64 XXH_read64(const void* memPtr) +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) { - xxh_u64 val; - memcpy(&val, memPtr, sizeof(val)); - return val; + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); } -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap64 _byteswap_uint64 -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap64 __builtin_bswap64 -#else -static xxh_u64 XXH_swap64 (xxh_u64 x) +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) { - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); } -#endif -/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* === Public XXH128 API === */ -XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) { - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[0] - | ((xxh_u64)bytePtr[1] << 8) - | ((xxh_u64)bytePtr[2] << 16) - | ((xxh_u64)bytePtr[3] << 24) - | ((xxh_u64)bytePtr[4] << 32) - | ((xxh_u64)bytePtr[5] << 40) - | ((xxh_u64)bytePtr[6] << 48) - | ((xxh_u64)bytePtr[7] << 56); + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); } -XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) { - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[7] - | ((xxh_u64)bytePtr[6] << 8) - | ((xxh_u64)bytePtr[5] << 16) - | ((xxh_u64)bytePtr[4] << 24) - | ((xxh_u64)bytePtr[3] << 32) - | ((xxh_u64)bytePtr[2] << 40) - | ((xxh_u64)bytePtr[1] << 48) - | ((xxh_u64)bytePtr[0] << 56); + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); } -#else -XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) { - return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); } -static xxh_u64 XXH_readBE64(const void* ptr) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) { - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); } -#endif -XXH_FORCE_INLINE xxh_u64 -XXH_readLE64_align(const void* ptr, XXH_alignment align) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) { - if (align==XXH_unaligned) - return XXH_readLE64(ptr); - else - return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); + return XXH3_128bits_withSeed(input, len, seed); } -/******* xxh64 *******/ - -static const xxh_u64 PRIME64_1 = 0x9E3779B185EBCA87ULL; /* 0b1001111000110111011110011011000110000101111010111100101010000111 */ -static const xxh_u64 PRIME64_2 = 0xC2B2AE3D27D4EB4FULL; /* 0b1100001010110010101011100011110100100111110101001110101101001111 */ -static const xxh_u64 PRIME64_3 = 0x165667B19E3779F9ULL; /* 0b0001011001010110011001111011000110011110001101110111100111111001 */ -static const xxh_u64 PRIME64_4 = 0x85EBCA77C2B2AE63ULL; /* 0b1000010111101011110010100111011111000010101100101010111001100011 */ -static const xxh_u64 PRIME64_5 = 0x27D4EB2F165667C5ULL; /* 0b0010011111010100111010110010111100010110010101100110011111000101 */ +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ -static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) { - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; + return XXH3_64bits_reset(statePtr); } -static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) { - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); } -static xxh_u64 XXH64_avalanche(xxh_u64 h64) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) { - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - return h64; + return XXH3_64bits_reset_withSeed(statePtr, seed); } - -#define XXH_get64bits(p) XXH_readLE64_align(p, align) - -static xxh_u64 -XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) { -#define PROCESS1_64 do { \ - h64 ^= (*ptr++) * PRIME64_5; \ - h64 = XXH_rotl64(h64, 11) * PRIME64_1; \ -} while (0) - -#define PROCESS4_64 do { \ - h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * PRIME64_1; \ - ptr += 4; \ - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; \ -} while (0) - -#define PROCESS8_64 do { \ - xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); \ - ptr += 8; \ - h64 ^= k1; \ - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ -} while (0) - - /* Rerolled version for 32-bit targets is faster and much smaller. */ - if (XXH_REROLL || XXH_REROLL_XXH64) { - len &= 31; - while (len >= 8) { - PROCESS8_64; - len -= 8; - } - if (len >= 4) { - PROCESS4_64; - len -= 4; - } - while (len > 0) { - PROCESS1_64; - --len; - } - return XXH64_avalanche(h64); - } else { - switch(len & 31) { - case 24: PROCESS8_64; - /* fallthrough */ - case 16: PROCESS8_64; - /* fallthrough */ - case 8: PROCESS8_64; - return XXH64_avalanche(h64); - - case 28: PROCESS8_64; - /* fallthrough */ - case 20: PROCESS8_64; - /* fallthrough */ - case 12: PROCESS8_64; - /* fallthrough */ - case 4: PROCESS4_64; - return XXH64_avalanche(h64); - - case 25: PROCESS8_64; - /* fallthrough */ - case 17: PROCESS8_64; - /* fallthrough */ - case 9: PROCESS8_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 29: PROCESS8_64; - /* fallthrough */ - case 21: PROCESS8_64; - /* fallthrough */ - case 13: PROCESS8_64; - /* fallthrough */ - case 5: PROCESS4_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 26: PROCESS8_64; - /* fallthrough */ - case 18: PROCESS8_64; - /* fallthrough */ - case 10: PROCESS8_64; - PROCESS1_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 30: PROCESS8_64; - /* fallthrough */ - case 22: PROCESS8_64; - /* fallthrough */ - case 14: PROCESS8_64; - /* fallthrough */ - case 6: PROCESS4_64; - PROCESS1_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 27: PROCESS8_64; - /* fallthrough */ - case 19: PROCESS8_64; - /* fallthrough */ - case 11: PROCESS8_64; - PROCESS1_64; - PROCESS1_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 31: PROCESS8_64; - /* fallthrough */ - case 23: PROCESS8_64; - /* fallthrough */ - case 15: PROCESS8_64; - /* fallthrough */ - case 7: PROCESS4_64; - /* fallthrough */ - case 3: PROCESS1_64; - /* fallthrough */ - case 2: PROCESS1_64; - /* fallthrough */ - case 1: PROCESS1_64; - /* fallthrough */ - case 0: return XXH64_avalanche(h64); - } - } - /* impossible to reach */ - XXH_ASSERT(0); - return 0; /* unreachable, but some compilers complain without it */ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); } -XXH_FORCE_INLINE xxh_u64 -XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) { - const xxh_u8* bEnd = input + len; - xxh_u64 h64; - -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - if (input==NULL) { - len=0; - bEnd=input=(const xxh_u8*)(size_t)32; - } -#endif - - if (len>=32) { - const xxh_u8* const limit = bEnd - 32; - xxh_u64 v1 = seed + PRIME64_1 + PRIME64_2; - xxh_u64 v2 = seed + PRIME64_2; - xxh_u64 v3 = seed + 0; - xxh_u64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; - v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; - v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; - v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; - } while (input<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); + return XXH3_64bits_update(state, input, len); +} - } else { - h64 = seed + PRIME64_5; +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_finalizeLong_128b(acc, secret, state->secretLimit + XXH_STRIPE_LEN, (xxh_u64)state->totalLen); } - - h64 += (xxh_u64) len; - - return XXH64_finalize(h64, input, len, align); + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->useSeed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); } +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ +#include /* memcmp, memcpy */ -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed) +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, (const xxh_u8*)input, len); - return XXH64_digest(&state); - -#else - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); - } } - - return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); - -#endif + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); } -/******* Hash Streaming *******/ - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) { - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); } -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) { - XXH_free(statePtr); - return XXH_OK; + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); } -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) { - memcpy(dstState, srcState, sizeof(*dstState)); + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; } -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed) + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) { - XXH64_state_t state; /* use a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - /* do not write into reserved64, might be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64)); - return XXH_OK; + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); } +/*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode -XXH64_update (XXH64_state_t* state, const void* input, size_t len) +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) { - if (input==NULL) -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - return XXH_OK; +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); #else - return XXH_ERROR; + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; #endif - { const xxh_u8* p = (const xxh_u8*)input; - const xxh_u8* const bEnd = p + len; - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); - state->memsize += (xxh_u32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const xxh_u8* const limit = bEnd - 32; - xxh_u64 v1 = state->v1; - xxh_u64 v2 = state->v2; - xxh_u64 v3 = state->v3; - xxh_u64 v4 = state->v4; + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif - do { - v1 = XXH64_round(v1, XXH_readLE64(p)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; ntotal_len >= 32) { - xxh_u64 const v1 = state->v1; - xxh_u64 const v2 = state->v2; - xxh_u64 const v3 = state->v3; - xxh_u64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 /*seed*/ + PRIME64_5; - } - - h64 += (xxh_u64) state->total_len; - - return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); -} - - -/******* Canonical representation *******/ - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed) { - return XXH_readBE64(src); + XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + XXH3_initCustomSecret(secret, seed); + XXH_ASSERT(secretBuffer != NULL); + memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE); } -/* ********************************************************************* -* XXH3 -* New generation hash designed for speed on small keys and vectorization -************************************************************************ */ - -#include "xxh3.h" - +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif #endif /* XXH_NO_LONG_LONG */ +#endif /* XXH_NO_XXH3 */ +/*! + * @} + */ #endif /* XXH_IMPLEMENTATION */ #if defined (__cplusplus) -} +} /* extern "C" */ #endif diff --git a/vcpkg.json b/vcpkg.json index 68968b4702..f356de5ca4 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,4 +1,14 @@ { + "vcpkg-configuration": { + "default-registry": { + "kind": "git", + "repository": "https://github.com/Microsoft/vcpkg", + "baseline": "dd3097e305afa53f7b4312371f62058d2e665320", + "reference": "dd3097e305afa53f7b4312371f62058d2e665320" + }, + "overlay-ports": [ "./cmake/overlay-ports" ], + "overlay-triplets": [ "./cmake/overlay-triplets" ] + }, "default-features": ["qt6"], "dependencies": [ "sdl2", @@ -18,7 +28,9 @@ "name": "libslirp", "platform": "linux" }, - "faad2" + "faad2", + "pkgconf", + { "name": "dirent", "platform": "windows" } ], "features": { "qt6": {