diff --git a/.github/actions/load_conan2/action.yml b/.github/actions/load_conan2/action.yml new file mode 100644 index 00000000..c3a0f8fe --- /dev/null +++ b/.github/actions/load_conan2/action.yml @@ -0,0 +1,63 @@ +name: 'Load Conan Cache' +description: 'Loads Local Conan Cache' +inputs: + testing: + description: 'Support building tests' + required: true + key_prefix: + description: 'Cache prefix' + required: true + default: 'Deps' + fail_on_cache_miss: + description: 'Fail if key missing' + required: false + default: false + path: + description: 'Recipe path' + required: false + default: '.' + load_any: + description: 'Load cache miss' + required: false + default: 'False' +outputs: + cache-hit: + description: 'Cache match found' + value: ${{ steps.restore-cache.outputs.cache-hit }} +runs: + using: "composite" + steps: + - id: hash-key-primary + shell: bash + run: | + echo "key=${{ inputs.path }}/conanfile.py" >> $GITHUB_OUTPUT + + - id: hash-key-3rd + shell: bash + run: | + echo "keys=${{ inputs.path }}/3rd_party/**/conanfile.py" >> $GITHUB_OUTPUT + + - name: Restore Cache + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: | + ~/conan_cache_save.tgz + key: ${{ inputs.key_prefix }}-${{ hashFiles(steps.hash-key-primary.outputs.key, steps.hash-key-3rd.outputs.keys) }} + fail-on-cache-miss: ${{ inputs.fail_on_cache_miss }} + + - name: Restore Testing Cache + uses: actions/cache/restore@v4 + with: + path: | + ~/conan_cache_save.tgz + key: ${{ inputs.key_prefix }}-${{ hashFiles(steps.hash-key-primary.outputs.key, steps.hash-key-3rd.outputs.keys) }} + restore-keys: ${{ inputs.key_prefix }}- + if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && (( github.event_name == 'pull_request' && inputs.testing == 'True' ) || ( inputs.load_any == 'True' )) }} + + - name: Restore Tarball + shell: bash + run: | + conan cache restore ~/conan_cache_save.tgz + if: ${{ ((steps.restore-cache.outputs.cache-hit == 'true') || ( inputs.load_any == 'True' ))}} + diff --git a/.github/actions/setup_conan2/action.yml b/.github/actions/setup_conan2/action.yml new file mode 100644 index 00000000..a0e41c4c --- /dev/null +++ b/.github/actions/setup_conan2/action.yml @@ -0,0 +1,32 @@ +name: 'Setup Conan v2' +description: 'Sets up Conan v2 for Sisl v13+ Builds' +inputs: + platform: + description: 'Platform conan will be building on' + required: true + default: 'ubuntu-24.04' +runs: + using: "composite" + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Setup Conan and Export Recipes + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install conan~=2.0 + python -m pip install gcovr + conan profile detect --name default + + - name: Fixup libstdc++ + shell: bash + run: | + # Set std::string to non-CoW C++11 version + sed -i 's,compiler.libcxx=libstdc++$,compiler.libcxx=libstdc++11,g' ~/.conan2/profiles/default + # Set C++ version to C++20 + sed -i 's,compiler.cppstd=.*$,compiler.cppstd=20,g' ~/.conan2/profiles/default + if: ${{ inputs.platform == 'ubuntu-24.04' }} + diff --git a/.github/actions/store_conan2/action.yml b/.github/actions/store_conan2/action.yml new file mode 100644 index 00000000..fdb29ee6 --- /dev/null +++ b/.github/actions/store_conan2/action.yml @@ -0,0 +1,22 @@ +name: 'Store Conan Cache' +description: 'Cleans Local Conan Cache and Persists Dirty Packages' +inputs: + key_prefix: + description: 'Cache prefix' + required: true + default: 'Deps' +runs: + using: "composite" + steps: + - name: Setup Conan and Export Recipes + shell: bash + run: | + conan cache save --no-source --file ~/conan_cache_save.tgz -l ~/pkglist.json + + - name: Save Cache + uses: actions/cache/save@v4 + with: + path: | + ~/conan_cache_save.tgz + key: ${{ inputs.key_prefix }}-${{ hashFiles('conanfile.py', '3rd_party/**/conanfile.py') }} + diff --git a/.github/workflows/build_dependencies.yml b/.github/workflows/build_dependencies.yml index 0f00efef..f5baac70 100644 --- a/.github/workflows/build_dependencies.yml +++ b/.github/workflows/build_dependencies.yml @@ -5,7 +5,7 @@ on: inputs: platform: required: false - default: 'ubuntu-22.04' + default: 'ubuntu-24.04' type: string branch: required: true @@ -16,10 +16,6 @@ on: malloc-impl: required: true type: string - prerelease: - required: false - type: string - default: 'False' tooling: required: false type: string @@ -34,11 +30,8 @@ on: required: true type: choice options: - - ubuntu-22.04 - - ubuntu-20.04 - - macos-13 - - macos-12 - default: 'ubuntu-22.04' + - ubuntu-24.04 + default: 'ubuntu-24.04' branch: required: true type: string @@ -57,14 +50,6 @@ on: - libc - tcmalloc - jemalloc - prerelease: - description: 'Fault Instrumentation' - required: false - type: choice - options: - - 'True' - - 'False' - default: 'False' tooling: required: false type: choice @@ -94,73 +79,69 @@ jobs: - name: Retrieve Recipe uses: actions/checkout@main with: - repository: eBay/sisl + repository: ebay/sisl ref: ${{ inputs.branch }} if: ${{ inputs.testing == 'False' }} - - name: Load Conan Cache - id: restore-cache - uses: eBay/sisl/.github/actions/load_conan@master - with: - testing: ${{ inputs.testing }} - key_prefix: SislDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }} - - name: Setup Conan - uses: eBay/sisl/.github/actions/setup_conan@master + uses: ebay/sisl/.github/actions/setup_conan2@dev/v13.x with: platform: ${{ inputs.platform }} if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }} + - name: Load Conan Cache + id: restore-cache + uses: ebay/sisl/.github/actions/load_conan2@dev/v13.x + with: + testing: ${{ inputs.testing }} + key_prefix: SislDeps13-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }} + - name: Prepare Recipes run: | - ./prepare.sh - cached_pkgs=$(ls -1d ~/.conan/data/*/*/*/*/export 2>/dev/null | sed 's,.*data/,,' | cut -d'/' -f1,2 | paste -sd',' - -) - echo "::info:: Pre-cached: ${cached_pkgs}" + ./prepare_v2.sh if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }} - name: Build Cache run: | conan install \ - -o prerelease=${{ inputs.prerelease }} \ - -o malloc_impl=${{ inputs.malloc-impl }} \ - -s build_type=${{ inputs.build-type }} \ + -o sisl/*:malloc_impl=${{ inputs.malloc-impl }} \ + -s:h build_type=${{ inputs.build-type }} \ + --format=json \ --build missing \ - . + . > ~/build.json + conan list --graph ~/build.json --graph-binaries=build --format=json > ~/pkglist.json if: ${{ steps.restore-cache.outputs.cache-hit != 'true' }} - name: Save Conan Cache - uses: eBay/sisl/.github/actions/store_conan@master + uses: ebay/sisl/.github/actions/store_conan2@dev/v13.x with: - key_prefix: SislDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }} + key_prefix: SislDeps13-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }} if: ${{ github.event_name != 'pull_request' && steps.restore-cache.outputs.cache-hit != 'true' }} - name: Create and Test Package run: | sanitize=$([[ "${{ inputs.tooling }}" == "Sanitize" ]] && echo "True" || echo "False") conan create \ - -o sisl:prerelease=${{ inputs.prerelease }} \ - -o sisl:malloc_impl=${{ inputs.malloc-impl }} \ - -o sisl:sanitize=${sanitize} \ - -s build_type=${{ inputs.build-type }} \ + -o sisl/*:malloc_impl=${{ inputs.malloc-impl }} \ + -o sisl/*:sanitize=${sanitize} \ + -s:h build_type=${{ inputs.build-type }} \ --build missing \ . if: ${{ inputs.testing == 'True' && inputs.tooling != 'Coverage' }} - name: Code Coverage Run run: | - conan install \ - -o prerelease=${{ inputs.prerelease }} \ - -o malloc_impl=${{ inputs.malloc-impl }} \ - -o coverage=True \ - -s build_type=${{ inputs.build-type }} \ + conan build \ + -o sisl/*:malloc_impl=${{ inputs.malloc-impl }} \ + -o sisl/*:coverage=True \ + -s:h build_type=${{ inputs.build-type }} \ --build missing \ . - conan build . if: ${{ inputs.testing == 'True' && inputs.tooling == 'Coverage' }} - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - gcov: true + verbose: true if: ${{ inputs.testing == 'True' && inputs.tooling == 'Coverage' }} diff --git a/.github/workflows/merge_build.yml b/.github/workflows/merge_build.yml index 729f88d1..87f21c07 100644 --- a/.github/workflows/merge_build.yml +++ b/.github/workflows/merge_build.yml @@ -1,9 +1,10 @@ -name: Sisl Build +name: Sisl v13 Build on: workflow_dispatch: push: branches: + - dev/v13.x - stable/v12.x - master @@ -12,14 +13,11 @@ jobs: strategy: fail-fast: false matrix: - platform: ["ubuntu-22.04"] + platform: ["ubuntu-24.04"] build-type: ["Debug", "Release"] malloc-impl: ["libc", "tcmalloc"] - prerelease: ["True", "False"] tooling: ["Sanitize", "Coverage", "None"] exclude: - - build-type: Debug - prerelease: "False" - build-type: Debug tooling: None - build-type: Debug @@ -36,29 +34,28 @@ jobs: branch: ${{ github.ref }} build-type: ${{ matrix.build-type }} malloc-impl: ${{ matrix.malloc-impl }} - prerelease: ${{ matrix.prerelease }} tooling: ${{ matrix.tooling }} testing: 'True' - ChainBuild: - runs-on: "ubuntu-22.04" - steps: - - name: Start IOManager Build - run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.CHAIN_BUILD_TOKEN }}"\ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/eBay/iomanager/actions/workflows/merge_build.yml/dispatches \ - -d '{"ref":"master","inputs":{}}' - if: ${{ github.ref == 'refs/heads/master' }} - - name: Start NuraftMesg Build - run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.CHAIN_BUILD_TOKEN }}"\ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/eBay/nuraft_mesg/actions/workflows/conan_build.yml/dispatches \ - -d '{"ref":"main","inputs":{}}' - if: ${{ github.ref == 'refs/heads/master' }} + # ChainBuild: + # runs-on: "ubuntu-22.04" + # steps: + # - name: Start IOManager Build + # run: | + # curl -L \ + # -X POST \ + # -H "Accept: application/vnd.github+json" \ + # -H "Authorization: Bearer ${{ secrets.CHAIN_BUILD_TOKEN }}"\ + # -H "X-GitHub-Api-Version: 2022-11-28" \ + # https://api.github.com/repos/eBay/iomanager/actions/workflows/merge_build.yml/dispatches \ + # -d '{"ref":"master","inputs":{}}' + # if: ${{ github.ref == 'refs/heads/master' }} + # - name: Start NuraftMesg Build + # run: | + # curl -L \ + # -X POST \ + # -H "Accept: application/vnd.github+json" \ + # -H "Authorization: Bearer ${{ secrets.CHAIN_BUILD_TOKEN }}"\ + # -H "X-GitHub-Api-Version: 2022-11-28" \ + # https://api.github.com/repos/eBay/nuraft_mesg/actions/workflows/conan_build.yml/dispatches \ + # -d '{"ref":"main","inputs":{}}' + # if: ${{ github.ref == 'refs/heads/master' }} diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index e8c5fe24..c47e7e7a 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -1,9 +1,10 @@ -name: Sisl Build +name: Sisl v13 Build on: workflow_dispatch: pull_request: branches: + - dev/v13.x - stable/v12.x - master @@ -12,14 +13,11 @@ jobs: strategy: fail-fast: false matrix: - platform: ["ubuntu-22.04"] + platform: ["ubuntu-24.04"] build-type: ["Debug", "Release"] malloc-impl: ["libc", "tcmalloc"] - prerelease: ["True", "False"] tooling: ["Sanitize", "Coverage", "None"] exclude: - - build-type: Debug - prerelease: "False" - build-type: Debug tooling: None - build-type: Debug @@ -36,6 +34,5 @@ jobs: branch: ${{ github.ref }} build-type: ${{ matrix.build-type }} malloc-impl: ${{ matrix.malloc-impl }} - prerelease: ${{ matrix.prerelease }} tooling: ${{ matrix.tooling }} testing: 'True' diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 2c4d3aca..79733e2b 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent { label 'sds-builder-v5' } + agent { label 'sds-builder-v8' } environment { ARTIFACTORY_PASS = credentials('ARTIFACTORY_PASS') @@ -37,7 +37,6 @@ pipeline { sh "conan create ${BUILD_MISSING} -s:h build_type=Debug -o ${PROJECT}/*:sanitize=True ${CONAN_FLAGS} . ; \ conan create ${BUILD_MISSING} -s:h build_type=Debug ${CONAN_FLAGS} . ; \ conan create ${BUILD_MISSING} -s:h build_type=RelWithDebInfo -o ${PROJECT}/*:malloc_impl=tcmalloc ${CONAN_FLAGS} . ; \ - conan create ${BUILD_MISSING} -s:h build_type=RelWithDebInfo -o ${PROJECT}/*:prerelease=True -o ${PROJECT}/*:malloc_impl=tcmalloc ${CONAN_FLAGS} . ; \ " } } diff --git a/3rd_party/folly/conan_deps.cmake b/3rd_party/folly/conan_deps.cmake index bdc0907f..075e362b 100644 --- a/3rd_party/folly/conan_deps.cmake +++ b/3rd_party/folly/conan_deps.cmake @@ -1,39 +1,45 @@ +cmake_minimum_required(VERSION 3.17) # Set the dependency flags expected by https://github.com/facebook/folly/blob/v2023.12.18.00/CMake/folly-deps.cmake -macro(custom_find_package name var) - find_package(${name} ${ARGN} - # Allow only Conan packages - NO_DEFAULT_PATH - PATHS ${CMAKE_PREFIX_PATH} - ) - set(${var}_FOUND TRUE) - set(${var}_VERSION ${${name}_VERSION}) - set(${var}_VERSION_STRING ${${name}_VERSION_STRING}) - set(${var}_INCLUDE_DIRS ${${name}_INCLUDE_DIRS}) - set(${var}_INCLUDE_DIR ${${name}_INCLUDE_DIR}) - set(${var}_INCLUDE ${${name}_INCLUDE_DIR}) - set(${var}_LIB ${${name}_LIBRARIES}) - set(${var}_LIBRARY ${${name}_LIBRARIES}) - set(${var}_LIBRARIES ${${name}_LIBRARIES}) - set(${var}_DEFINITIONS ${${name}_DEFINITIONS}) +macro(custom_find_package package_name variable_prefix) + find_package(${package_name} REQUIRED CONFIG ${ARGN}) + list(APPEND FROM FOUND VERSION VERSION_STRING INCLUDE_DIRS INCLUDE_DIR INCLUDE_DIR LIBRARIES LIBRARIES LIBRARIES DEFINITIONS) + list(APPEND TO FOUND VERSION VERSION_STRING INCLUDE_DIRS INCLUDE_DIR INCLUDE LIB LIBRARY LIBRARIES DEFINITIONS) + + foreach (from_substr to_substr IN ZIP_LISTS FROM TO) + set(src_var ${package_name}_${from_substr}) + set(dst_var ${variable_prefix}_${to_substr}) + if (NOT DEFINED ${src_var}) + continue() + endif() + if ((DEFINED ${dst_var}) AND ("${${dst_var}}" STREQUAL "${${src_var}}")) + # if they're equal, skip + continue() + endif() + message(DEBUG "custom_find_package definining ${dst_var} with ${src_var} contents: ${${src_var}}") + set(${dst_var} ${${src_var}}) + endforeach() endmacro() custom_find_package(BZip2 BZIP2) -custom_find_package(Backtrace BACKTRACE) custom_find_package(DoubleConversion DOUBLE_CONVERSION REQUIRED) custom_find_package(Gflags LIBGFLAGS) custom_find_package(Glog GLOG) custom_find_package(LZ4 LZ4) -custom_find_package(LibAIO LIBAIO) -custom_find_package(LibDwarf LIBDWARF) custom_find_package(LibEvent LIBEVENT REQUIRED) custom_find_package(LibLZMA LIBLZMA) -custom_find_package(LibUnwind LIBUNWIND) -custom_find_package(LibUring LIBURING) -custom_find_package(Libiberty LIBIBERTY) custom_find_package(Libsodium LIBSODIUM) custom_find_package(OpenSSL OPENSSL REQUIRED) custom_find_package(Snappy SNAPPY) custom_find_package(ZLIB ZLIB) custom_find_package(Zstd ZSTD) custom_find_package(fmt FMT REQUIRED) + +if (NOT MSVC) + custom_find_package(LibDwarf LIBDWARF) +endif() +if (UNIX AND NOT APPLE) + custom_find_package(LibUring LIBURING) + custom_find_package(LibUnwind LIBUNWIND) + custom_find_package(Libiberty LIBIBERTY) +endif() \ No newline at end of file diff --git a/3rd_party/folly/conandata.yml b/3rd_party/folly/conandata.yml index 93d9352f..32374b2d 100644 --- a/3rd_party/folly/conandata.yml +++ b/3rd_party/folly/conandata.yml @@ -1,4 +1,7 @@ sources: + "nu2.2024.08.12.00": + url: "https://github.com/facebook/folly/releases/download/v2024.08.12.00/folly-v2024.08.12.00.tar.gz" + sha256: "18d7be721721db547cb9c5dd5cc50df05cd88b0a8e683e3126ec6f9ce2c41c4d" "nu2.2023.12.18.00": url: "https://github.com/facebook/folly/releases/download/v2023.12.18.00/folly-v2023.12.18.00.tar.gz" sha256: "57ce880e3ae7b4d4fe0980be64da9e6ca7dd09e2de477670bf984e11cf7739f2" @@ -6,6 +9,10 @@ sources: url: "https://github.com/facebook/folly/releases/download/v2022.10.31.00/folly-v2022.10.31.00.tar.gz" sha256: "d7749f78eee2a327c1fa6b4a290e4bcd7115cdd7f7ef59f9e043ed59e597ab30" patches: + "nu2.2024.08.12.00": + - patch_file: "patches/2024-001-timespec.patch" + patch_description: "Fix liburing inclusion of timespec" + patch_type: "conan" "nu2.2023.12.18.00": - patch_file: "patches/2023-001-compiler-flags.patch" patch_description: "Do not hard-code debug flag for all build types" diff --git a/3rd_party/folly/conanfile.py b/3rd_party/folly/conanfile.py index 360474a8..3a83da8b 100755 --- a/3rd_party/folly/conanfile.py +++ b/3rd_party/folly/conanfile.py @@ -1,14 +1,16 @@ from conan import ConanFile from conan.errors import ConanInvalidConfiguration, ConanException from conan.tools.apple import is_apple_os -from conan.tools.build import can_run, check_min_cppstd +from conan.tools.build import check_min_cppstd, cross_building +from conan.tools.env import VirtualBuildEnv from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout -from conan.tools.files import apply_conandata_patches, export_conandata_patches, get, copy, rmdir, replace_in_file, save -from conan.tools.microsoft import is_msvc, msvc_runtime_flag +from conan.tools.files import apply_conandata_patches, export_conandata_patches, get, copy, rmdir, replace_in_file, save, rm +from conan.tools.microsoft import is_msvc, is_msvc_static_runtime from conan.tools.scm import Version import os -required_conan_version = ">=1.53.0" + +required_conan_version = ">=2.1" class FollyConan(ConanFile): @@ -24,12 +26,10 @@ class FollyConan(ConanFile): options = { "shared": [True, False], "fPIC": [True, False], - "use_sse4_2": [True, False], } default_options = { "shared": False, "fPIC": True, - "use_sse4_2": False } @property @@ -38,12 +38,13 @@ def _min_cppstd(self): @property def _compilers_minimum_version(self): + # https://github.com/facebook/folly/blob/v2024.07.08.00/folly/Portability.h return { - "gcc": "7", - "Visual Studio": "16", + "gcc": "10", + "clang": "10", + "apple-clang": "14", "msvc": "192", - "clang": "6", - "apple-clang": "10", + "Visual Studio": "16", } def export_sources(self): @@ -53,38 +54,40 @@ def export_sources(self): def config_options(self): if self.settings.os == "Windows": del self.options.fPIC - if str(self.settings.arch) not in ["x86", "x86_64"]: - del self.options.use_sse4_2 def configure(self): if self.options.shared: self.options.rm_safe("fPIC") + if is_msvc(self): + # Folly does not support shared library on Windows: https://github.com/facebook/folly/issues/962 + self.package_type = "static-library" + del self.options.shared def layout(self): cmake_layout(self, src_folder="src") def requirements(self): - self.requires("boost/1.83.0", transitive_headers=True, transitive_libs=True) + self.requires("boost/1.85.0", transitive_headers=True, transitive_libs=True) self.requires("bzip2/1.0.8") self.requires("double-conversion/3.3.0", transitive_headers=True, transitive_libs=True) self.requires("gflags/2.2.2") - self.requires("glog/0.6.0", transitive_headers=True, transitive_libs=True) + self.requires("glog/0.7.1", transitive_headers=True, transitive_libs=True) self.requires("libevent/2.1.12", transitive_headers=True, transitive_libs=True) self.requires("openssl/[>=1.1 <4]") - self.requires("lz4/1.9.4", transitive_libs=True) - self.requires("snappy/1.1.10") + self.requires("lz4/1.10.0", transitive_libs=True) + self.requires("snappy/1.2.1") self.requires("zlib/[>=1.2.11 <2]") self.requires("zstd/1.5.5", transitive_libs=True) if not is_msvc(self): - self.requires("libdwarf/20191104") - self.requires("libsodium/1.0.19") - self.requires("xz_utils/5.4.5") - # FIXME: Causing compilation issues on clang: self.requires("jemalloc/5.2.1") + self.requires("libdwarf/0.9.1") + self.requires("libsodium/1.0.20") + self.requires("xz_utils/[>=5.4.5 <6]") if self.settings.os in ["Linux", "FreeBSD"]: self.requires("libiberty/9.1.0") - self.requires("libunwind/1.7.2") + self.requires("libunwind/1.8.0") if self.settings.os == "Linux": - self.requires("liburing/[>=2.1]") + self.requires("liburing/2.6") + # INFO: Folly does not support fmt 11 on MSVC: https://github.com/facebook/folly/issues/2250 self.requires("fmt/10.2.1", transitive_headers=True, transitive_libs=True) def build_requirements(self): @@ -98,114 +101,72 @@ def _required_boost_components(self): @property def _required_boost_conan_components(self): return [f"boost::{comp}" for comp in self._required_boost_components] - + @property def _required_boost_cmake_targets(self): return [f"Boost::{comp}" for comp in self._required_boost_components] - + def validate(self): if self.settings.compiler.cppstd: check_min_cppstd(self, self._min_cppstd) minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) if minimum_version and Version(self.settings.compiler.version) < minimum_version: - raise ConanInvalidConfiguration( - f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." - ) + raise ConanInvalidConfiguration(f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support.") - if is_apple_os(self) and self.settings.arch != "x86_64": - raise ConanInvalidConfiguration("Conan currently requires a 64bit target architecture for Folly on Macos") - - if is_apple_os(self): - raise ConanInvalidConfiguration("Current recipe doesn't support Macos. Contributions are welcome.") - if self.settings.os == "Windows" and self.settings.arch != "x86_64": - raise ConanInvalidConfiguration("Folly requires a 64bit target architecture on Windows") - - if (is_apple_os(self) or self.settings.os == "Windows") and self.options.shared: - raise ConanInvalidConfiguration(f"Folly could not be built on {self.settings.os} as shared library") + raise ConanInvalidConfiguration(f"{self.ref} Folly requires a 64bit target architecture on Windows.") - if self.settings.os == "Windows": - raise ConanInvalidConfiguration(f"{self.ref} could not be built on {self.settings.os}. PR's are welcome.") - - if self.settings.compiler == "clang" and self.options.shared: - raise ConanInvalidConfiguration(f"Folly {self.version} could not be built by clang as a shared library") + if self.settings.compiler == "apple-clang" and Version(self.settings.compiler.version) < "14.0": + # https://github.com/facebook/folly/issues/2266 + raise ConanInvalidConfiguration(f"{self.ref} could not be built by apple-clang < 14.0. Use apple-clang >= 14.0") - glog = self.dependencies["glog"] - if self.options.shared and not glog.options.shared: - raise ConanInvalidConfiguration(f"If Folly is built as shared lib, glog must be a shared lib too.") - boost = self.dependencies["boost"] if boost.options.header_only: - raise ConanInvalidConfiguration("Folly could not be built with a header only Boost") + raise ConanInvalidConfiguration(f"{self.ref} could not be built with a header only Boost. Use -o 'boost/*:header_only=False'") miss_boost_required_comp = any(getattr(boost.options, f"without_{boost_comp}", True) for boost_comp in self._required_boost_components) if miss_boost_required_comp: required_components = ", ".join(self._required_boost_components) - raise ConanInvalidConfiguration(f"Folly requires these boost components: {required_components}") - - if self.options.get_safe("use_sse4_2") and str(self.settings.arch) not in ['x86', 'x86_64']: - raise ConanInvalidConfiguration(f"{self.ref} can use the option use_sse4_2 only on x86 and x86_64 archs.") + raise ConanInvalidConfiguration(f"{self.ref} requires these Boost components: {required_components}. Try with '-o boost/*:without_{required_components}=False'") def source(self): get(self, **self.conan_data["sources"][self.version], strip_root=False) - def _cppstd_flag_value(self, cppstd): - cppstd = str(cppstd) - if cppstd.startswith("gnu"): - prefix = "gnu" - year = cppstd[3:] - else: - prefix = "c" - year = cppstd - if is_msvc(self): - prefix = "" - if year > "17": - year = "latest" - return f"{prefix}++{year}" - def generate(self): + env = VirtualBuildEnv(self) + env.generate() tc = CMakeToolchain(self) + tc.variables["CMAKE_PROJECT_folly_INCLUDE"] = "conan_deps.cmake" + # Folly fails to check Gflags: https://github.com/conan-io/conan/issues/12012 + tc.variables["CMAKE_TRY_COMPILE_CONFIGURATION"] = str(self.settings.build_type) - tc.cache_variables["CMAKE_PROJECT_folly_INCLUDE"] = os.path.join(self.source_folder, "conan_deps.cmake") - - if can_run(self): + if is_apple_os(self) and cross_building(self): + # INFO: Folly fails to configure Mac M1 -> Mac Intel: + # CMake Error: try_run() invoked in cross-compiling mode, please set the following cache variables appropriately: HAVE_VSNPRINTF_ERRORS_EXITCODE (advanced) for var in ["FOLLY_HAVE_UNALIGNED_ACCESS", "FOLLY_HAVE_LINUX_VDSO", "FOLLY_HAVE_WCHAR_SUPPORT", "HAVE_VSNPRINTF_ERRORS"]: - tc.variables[f"{var}_EXITCODE"] = "0" - tc.variables[f"{var}_EXITCODE__TRYRUN_OUTPUT"] = "" - - if self.options.get_safe("use_sse4_2") and str(self.settings.arch) in ["x86", "x86_64"]: - tc.preprocessor_definitions["FOLLY_SSE"] = "4" - tc.preprocessor_definitions["FOLLY_SSE_MINOR"] = "2" - if not is_msvc(self): - cflags = "-mfma" - else: - cflags = "/arch:FMA" - tc.blocks["cmake_flags_init"].template += ( - f'string(APPEND CMAKE_CXX_FLAGS_INIT " {cflags}")\n' - f'string(APPEND CMAKE_C_FLAGS_INIT " {cflags}")\n' - ) + tc.cache_variables[f"{var}_EXITCODE"] = 0 # Folly is not respecting this from the helper https://github.com/conan-io/conan-center-index/pull/15726/files#r1097068754 - tc.variables["CMAKE_POSITION_INDEPENDENT_CODE"] = self.options.get_safe("fPIC", True) + tc.cache_variables["CMAKE_POSITION_INDEPENDENT_CODE"] = self.options.get_safe("fPIC", True) # Relocatable shared lib on Macos - tc.cache_variables["CMAKE_POLICY_DEFAULT_CMP0042"] = "NEW" - # Honor CMAKE_REQUIRED_LIBRARIES in check_include_file_xxx - tc.cache_variables["CMAKE_POLICY_DEFAULT_CMP0075"] = "NEW" + if is_apple_os(self): + tc.cache_variables["CMAKE_POLICY_DEFAULT_CMP0042"] = "NEW" # Honor BUILD_SHARED_LIBS from conan_toolchain (see https://github.com/conan-io/conan/issues/11840) tc.cache_variables["CMAKE_POLICY_DEFAULT_CMP0077"] = "NEW" # Honor Boost_ROOT set by boost recipe tc.cache_variables["CMAKE_POLICY_DEFAULT_CMP0074"] = "NEW" tc.cache_variables["CMAKE_POLICY_VERSION_MINIMUM"] = "3.5" # CMake 4 support + #if Version(self.version) > "2024.08.12.00": # pylint: disable=conan-unreachable-upper-version + # raise ConanException("CMAKE_POLICY_VERSION_MINIMUM hardcoded to 3.5, check if new version supports CMake 4") + - cxx_std_value = self._cppstd_flag_value(self.settings.get_safe("compiler.cppstd", self._min_cppstd)) # 2019.10.21.00 -> either MSVC_ flags or CXX_STD if is_msvc(self): - tc.variables["MSVC_LANGUAGE_VERSION"] = cxx_std_value - tc.variables["MSVC_ENABLE_ALL_WARNINGS"] = False - tc.variables["MSVC_USE_STATIC_RUNTIME"] = "MT" in msvc_runtime_flag(self) + cxx_std_value = "c++latest" if str(self.settings.compiler.cppstd) > "17" else f"c++{str(self.settings.compiler.cppstd)}" + tc.cache_variables["MSVC_LANGUAGE_VERSION"] = cxx_std_value + tc.cache_variables["MSVC_ENABLE_ALL_WARNINGS"] = False + tc.cache_variables["MSVC_USE_STATIC_RUNTIME"] = is_msvc_static_runtime(self) tc.preprocessor_definitions["NOMINMAX"] = "" - else: - tc.variables["CXX_STD"] = cxx_std_value if not self.dependencies["boost"].options.header_only: tc.cache_variables["BOOST_LINK_STATIC"] = not self.dependencies["boost"].options.shared @@ -214,14 +175,12 @@ def generate(self): tc.generate() deps = CMakeDeps(self) - # deps.set_property("backtrace", "cmake_file_name", "Backtrace") deps.set_property("boost", "cmake_file_name", "Boost") deps.set_property("bzip2", "cmake_file_name", "BZip2") deps.set_property("double-conversion", "cmake_file_name", "DoubleConversion") deps.set_property("fmt", "cmake_file_name", "fmt") deps.set_property("gflags", "cmake_file_name", "Gflags") deps.set_property("glog", "cmake_file_name", "Glog") - # deps.set_property("libaio", "cmake_file_name", "LibAIO") deps.set_property("libdwarf", "cmake_file_name", "LibDwarf") deps.set_property("libevent", "cmake_file_name", "LibEvent") deps.set_property("libiberty", "cmake_file_name", "Libiberty") @@ -237,13 +196,19 @@ def generate(self): deps.generate() def _patch_sources(self): + # Make sure will consume Conan dependencies apply_conandata_patches(self) folly_deps = os.path.join(self.source_folder, "CMake", "folly-deps.cmake") - replace_in_file(self, folly_deps, " MODULE", " ") + replace_in_file(self, folly_deps, " MODULE", " REQUIRED CONFIG") replace_in_file(self, folly_deps, "${Boost_LIBRARIES}", f"{' '.join(self._required_boost_cmake_targets)}") replace_in_file(self, folly_deps, "OpenSSL 1.1.1", "OpenSSL") # Disable example save(self, os.path.join(self.source_folder, "folly", "logging", "example", "CMakeLists.txt"), "") + # Disable custom find modules to use Conan CMakeDeps instead + rm(self, "Find*.cmake", os.path.join(self.source_folder, "CMake")) + rm(self, "Find*.cmake", os.path.join(self.source_folder, "build", "fbcode_builder", "CMake")) + # Skip generating .pc file to avoid Windows errors when trying to compile with pkg-config + replace_in_file(self, os.path.join(self.source_folder, "CMakeLists.txt"), "gen_pkgconfig_vars(FOLLY_PKGCONFIG folly_deps)", "") def build(self): self._patch_sources() @@ -275,36 +240,30 @@ def package_info(self): "openssl::openssl", "bzip2::bzip2", "snappy::snappy", - "liburing::liburing", "zlib::zlib", "zstd::zstd", "libsodium::libsodium", - "xz_utils::xz_utils" + "xz_utils::xz_utils", ] if not is_msvc(self): self.cpp_info.components["libfolly"].requires.append("libdwarf::libdwarf") if self.settings.os in ["Linux", "FreeBSD"]: self.cpp_info.components["libfolly"].requires.extend(["libiberty::libiberty", "libunwind::libunwind"]) + if self.settings.os == "Linux": + self.cpp_info.components["libfolly"].requires.append("liburing::liburing") self.cpp_info.components["libfolly"].system_libs.extend(["pthread", "dl", "rt"]) self.cpp_info.components["libfolly"].defines.extend(["FOLLY_HAVE_ELF", "FOLLY_HAVE_DWARF"]) elif self.settings.os == "Windows": self.cpp_info.components["libfolly"].system_libs.extend(["ws2_32", "iphlpapi", "crypt32"]) - if str(self.settings.compiler.libcxx) == "libstdc++" or ( - self.settings.compiler == "apple-clang" and - Version(self.settings.compiler.version.value) == "9.0" and - self.settings.compiler.libcxx == "libc++"): + if self.settings.get_safe("compiler.libcxx") == "libstdc++" or \ + (self.settings.compiler == "apple-clang" and Version(self.settings.compiler.version.value) == "9.0" and \ + self.settings.get_safe("compiler.libcxx") == "libc++"): self.cpp_info.components["libfolly"].system_libs.append("atomic") if self.settings.compiler == "apple-clang" and Version(self.settings.compiler.version.value) >= "11.0": self.cpp_info.components["libfolly"].system_libs.append("c++abi") - if self.settings.compiler == "gcc" and Version(self.settings.compiler.version) < "9": - self.cpp_info.components["libfolly"].system_libs.append("stdc++fs") - - if self.settings.compiler == "clang" and Version(self.settings.compiler.version) < "9": - self.cpp_info.components["libfolly"].system_libs.append("stdc++fs" if self.settings.compiler.libcxx in ["libstdc++", "libstdc++11"] else "c++fs") - self.cpp_info.components["follybenchmark"].set_property("cmake_target_name", "Folly::follybenchmark") self.cpp_info.components["follybenchmark"].set_property("pkg_config_name", "libfollybenchmark") self.cpp_info.components["follybenchmark"].libs = ["follybenchmark"] @@ -330,27 +289,3 @@ def package_info(self): self.cpp_info.components["folly_exception_counter"].set_property("pkg_config_name", "libfolly_exception_counter") self.cpp_info.components["folly_exception_counter"].libs = ["folly_exception_counter"] self.cpp_info.components["folly_exception_counter"].requires = ["folly_exception_tracer"] - - # TODO: to remove in conan v2 once cmake_find_package_* & pkg_config generators removed - self.cpp_info.filenames["cmake_find_package"] = "folly" - self.cpp_info.filenames["cmake_find_package_multi"] = "folly" - self.cpp_info.names["cmake_find_package"] = "Folly" - self.cpp_info.names["cmake_find_package_multi"] = "Folly" - self.cpp_info.names["pkg_config"] = "libfolly" - self.cpp_info.components["libfolly"].names["cmake_find_package"] = "folly" - self.cpp_info.components["libfolly"].names["cmake_find_package_multi"] = "folly" - - # TODO: to remove in conan v2 once cmake_find_package_* & pkg_config generators removed - self.cpp_info.components["follybenchmark"].names["cmake_find_package"] = "follybenchmark" - self.cpp_info.components["follybenchmark"].names["cmake_find_package_multi"] = "follybenchmark" - self.cpp_info.components["folly_test_util"].names["cmake_find_package"] = "folly_test_util" - self.cpp_info.components["folly_test_util"].names["cmake_find_package_multi"] = "folly_test_util" - - if self.settings.os in ["Linux", "FreeBSD"]: - # TODO: to remove in conan v2 once cmake_find_package_* & pkg_config generators removed - self.cpp_info.components["folly_exception_tracer_base"].names["cmake_find_package"] = "folly_exception_tracer_base" - self.cpp_info.components["folly_exception_tracer_base"].names["cmake_find_package_multi"] = "folly_exception_tracer_base" - self.cpp_info.components["folly_exception_tracer"].names["cmake_find_package"] = "folly_exception_tracer" - self.cpp_info.components["folly_exception_tracer"].names["cmake_find_package_multi"] = "folly_exception_tracer" - self.cpp_info.components["folly_exception_counter"].names["cmake_find_package"] = "folly_exception_counter" - self.cpp_info.components["folly_exception_counter"].names["cmake_find_package_multi"] = "folly_exception_counter" diff --git a/3rd_party/folly/patches/2024-001-timespec.patch b/3rd_party/folly/patches/2024-001-timespec.patch new file mode 100644 index 00000000..03d9ef33 --- /dev/null +++ b/3rd_party/folly/patches/2024-001-timespec.patch @@ -0,0 +1,36 @@ +diff -Naur a/folly/io/async/AsyncSocket.cpp b/folly/io/async/AsyncSocket.cpp +--- a/folly/io/async/AsyncSocket.cpp 2024-08-12 09:28:44.000000000 +0000 ++++ b/folly/io/async/AsyncSocket.cpp 2025-10-22 16:13:34.439374937 +0000 +@@ -17,6 +17,8 @@ + #include + + #include ++/* for struct sock_extended_err*/ ++#include + + #include + #include +diff -Naur a/folly/io/async/AsyncUDPSocket.cpp b/folly/io/async/AsyncUDPSocket.cpp +--- a/folly/io/async/AsyncUDPSocket.cpp 2024-08-12 09:28:44.000000000 +0000 ++++ b/folly/io/async/AsyncUDPSocket.cpp 2025-10-22 16:13:34.440374956 +0000 +@@ -17,6 +17,8 @@ + #include + #include + ++/* for struct sock_extended_err*/ ++#include + #include + + #include +diff -Naur a/folly/net/NetOps.h b/folly/net/NetOps.h +--- a/folly/net/NetOps.h 2024-08-12 09:28:44.000000000 +0000 ++++ b/folly/net/NetOps.h 2025-10-22 16:14:31.665479235 +0000 +@@ -114,7 +114,7 @@ + #endif + #endif + /* for struct sock_extended_err*/ +-#include ++#include + #endif + #endif + diff --git a/conanfile.py b/conanfile.py index 6f171704..afa1d164 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class SISLConan(ConanFile): name = "sisl" - version = "12.4.8" + version = "13.0.2" homepage = "https://github.com/eBay/sisl" description = "Library for fast data structures, utilities" @@ -24,7 +24,6 @@ class SISLConan(ConanFile): "fPIC": ['True', 'False'], "coverage": ['True', 'False'], "sanitize": ['True', 'False'], - 'prerelease' : ['True', 'False'], 'metrics': ['False', 'True'], 'grpc': ['False', 'True'], 'malloc_impl' : ['libc', 'tcmalloc', 'jemalloc'], @@ -34,7 +33,6 @@ class SISLConan(ConanFile): 'fPIC': True, 'coverage': False, 'sanitize': False, - 'prerelease': False, 'metrics': True, 'grpc': True, 'malloc_impl': 'libc', @@ -64,7 +62,6 @@ def configure(self): if self.options.shared: self.options.rm_safe("fPIC") if self.settings.build_type == "Debug": - self.options.rm_safe("prerelease") if self.options.coverage and self.options.sanitize: raise ConanInvalidConfiguration("Sanitizer does not work with Code Coverage!") if self.conf.get("tools.build:skip_test", default=False): @@ -72,43 +69,38 @@ def configure(self): raise ConanInvalidConfiguration("Coverage/Sanitizer requires Testing!") def build_requirements(self): - self.test_requires("gtest/1.14.0") + self.test_requires("gtest/1.17.0") if self.options.metrics: - self.test_requires("benchmark/1.8.2") + self.test_requires("benchmark/1.9.4") def requirements(self): # Required self.requires("boost/1.85.0", transitive_headers=True) - self.requires("cxxopts/3.1.1", transitive_headers=True) - self.requires("nlohmann_json/3.11.2", transitive_headers=True) - self.requires("spdlog/1.12.0", transitive_headers=True) + self.requires("cxxopts/3.3.1", transitive_headers=True) + self.requires("nlohmann_json/3.12.0", transitive_headers=True) + self.requires("spdlog/1.14.1", transitive_headers=True) self.requires("zmarok-semver/1.1.0", transitive_headers=True) + self.requires("lz4/1.10.0", override=True) if self.settings.os in ["Linux"]: self.requires("breakpad/cci.20210521") - self.requires("fmt/10.0.0", override=True) - self.requires("lz4/1.10.0", override=True) # ARM needs unreleased versionof libunwind if not self.settings.arch in ['x86', 'x86_64']: self.requires("libunwind/1.8.2@baydb/develop", override=True) - else: - self.requires("libunwind/1.8.0", override=True) if self.options.metrics: - self.requires("flatbuffers/23.5.26", transitive_headers=True) + self.requires("flatbuffers/24.12.23", transitive_headers=True) self.requires("folly/nu2.2023.12.18.00", transitive_headers=True) self.requires("prometheus-cpp/1.1.0", transitive_headers=True) - self.requires("snappy/[^1.1]", transitive_headers=True) + self.requires("snappy/[^1.2]", transitive_headers=True) self.requires("userspace-rcu/nu2.0.14.0", transitive_headers=True) - self.requires("libcurl/8.4.0", override=True) - self.requires("xz_utils/5.4.5", override=True) if self.options.grpc: self.requires("grpc/1.54.3", transitive_headers=True) # Memory allocation if self.options.malloc_impl == "tcmalloc": - self.requires("gperftools/2.15", transitive_headers=True) + self.requires("gperftools/2.17.2", transitive_headers=True) elif self.options.malloc_impl == "jemalloc": self.requires("jemalloc/5.3.0", transitive_headers=True) @@ -165,11 +157,7 @@ def generate(self): tc.variables['MALLOC_IMPL'] = self.options.malloc_impl tc.preprocessor_definitions["PACKAGE_VERSION"] = self.version tc.preprocessor_definitions["PACKAGE_NAME"] = self.name - if self.options.get_safe("prerelease") or (self.settings.build_type == "Debug"): - tc.preprocessor_definitions["_PRERELEASE"] = "1" - tc.variables["_PRERELEASE"] = "ON" if self.settings.build_type == "Debug": - tc.preprocessor_definitions["_PRERELEASE"] = "1" if self.options.get_safe("coverage"): tc.variables['BUILD_COVERAGE'] = 'ON' elif self.options.get_safe("sanitize"): @@ -284,8 +272,6 @@ def package_info(self): component.defines.append("_LARGEFILE64") component.system_libs.extend(["dl", "pthread"]) component.exelinkflags.extend(["-export-dynamic"]) - if self.options.get_safe("prerelease") or (self.settings.build_type == "Debug"): - component.defines.append("_PRERELEASE=1") if self.options.get_safe("sanitize"): component.sharedlinkflags.append("-fsanitize=address") component.exelinkflags.append("-fsanitize=address") diff --git a/include/sisl/auth_manager/token_verifier.hpp b/include/sisl/auth_manager/token_verifier.hpp index 82ee0a68..3ee306f2 100644 --- a/include/sisl/auth_manager/token_verifier.hpp +++ b/include/sisl/auth_manager/token_verifier.hpp @@ -43,7 +43,7 @@ class GrpcTokenVerifier : public TokenVerifier { explicit GrpcTokenVerifier(std::string const& auth_header_key) : m_auth_header_key(auth_header_key) {} virtual ~GrpcTokenVerifier() = default; - virtual grpc::Status verify(grpc::ServerContext const* srv_ctx) const = 0; + virtual grpc::Status verify_ctx(grpc::ServerContext const* srv_ctx) const = 0; protected: std::string m_auth_header_key; diff --git a/include/sisl/cache/range_hashmap.hpp b/include/sisl/cache/range_hashmap.hpp index 0c2c3d5b..d7384025 100644 --- a/include/sisl/cache/range_hashmap.hpp +++ b/include/sisl/cache/range_hashmap.hpp @@ -250,7 +250,7 @@ class MultiEntryHashNode : public boost::intrusive::slist_base_hook<> { K m_base_key; big_offset_t m_base_nth; - folly::small_vector< ValueEntryRange, 8, folly::small_vector_policy::policy_size_type > m_values; + folly::small_vector< ValueEntryRange, 8, folly::small_vector_policy::policy_size_type< small_count_t > > m_values; public: MultiEntryHashNode(const K& base_key, big_offset_t nth) : m_base_key{base_key}, m_base_nth{nth} {} @@ -464,7 +464,7 @@ template < typename K > class HashBucket { private: #ifndef GLOBAL_HASHSET_LOCK - mutable folly::SharedMutexWritePriority m_lock; + mutable folly::SharedMutex m_lock; #endif typedef boost::intrusive::slist< MultiEntryHashNode< K > > hash_node_list_t; hash_node_list_t m_list; @@ -483,7 +483,7 @@ class HashBucket { void insert(const RangeKey< K >& input_key, sisl::byte_view&& value) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); #endif const auto input_nth_rounded = input_key.rounded_nth(); MultiEntryHashNode< K >* n = nullptr; @@ -511,7 +511,7 @@ class HashBucket { big_count_t get(const RangeKey< K >& input_key, std::vector< std::pair< RangeKey< K >, sisl::byte_view > >& out_values) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); #endif big_count_t ret{0}; const auto input_nth_rounded = input_key.rounded_nth(); @@ -533,7 +533,7 @@ class HashBucket { void erase(const RangeKey< K >& input_key) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); #endif const auto input_nth_rounded = input_key.rounded_nth(); MultiEntryHashNode< K >* n = nullptr; diff --git a/include/sisl/cache/simple_hashmap.hpp b/include/sisl/cache/simple_hashmap.hpp index 1488f7a8..8319f9ac 100644 --- a/include/sisl/cache/simple_hashmap.hpp +++ b/include/sisl/cache/simple_hashmap.hpp @@ -116,7 +116,7 @@ template < typename K, typename V > class SimpleHashBucket { private: #ifndef GLOBAL_HASHSET_LOCK - mutable folly::SharedMutexWritePriority m_lock; + mutable folly::SharedMutex m_lock; #endif typedef boost::intrusive::slist< SingleEntryHashNode< V > > hash_node_list_t; hash_node_list_t m_list; @@ -135,7 +135,7 @@ class SimpleHashBucket { bool insert(const K& input_key, const V& input_value, bool overwrite_ok) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); #endif SingleEntryHashNode< V >* n = nullptr; auto it = m_list.begin(); @@ -164,7 +164,7 @@ class SimpleHashBucket { bool get(const K& input_key, V& out_val) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); #endif bool found{false}; for (const auto& n : m_list) { @@ -183,7 +183,7 @@ class SimpleHashBucket { bool erase(const K& input_key, V& out_val) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); #endif return erase_unsafe(input_key, out_val, true /* call_access_cb */); } @@ -191,23 +191,20 @@ class SimpleHashBucket { bool try_erase(const K& input_key) { V dummy_val; #ifndef GLOBAL_HASHSET_LOCK - if(m_lock.try_lock()) { + if (m_lock.try_lock()) { bool ret = erase_unsafe(input_key, dummy_val, false /* call_access_cb */); m_lock.unlock(); return ret; } else { return false; } -#else - // We are in global hashset lock - return erase_unsafe(input_key, dummy_val, false); #endif #endif return erase_unsafe(input_key, dummy_val, false); } bool upsert_or_delete(const K& input_key, auto&& update_or_delete_cb) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); #endif SingleEntryHashNode< V >* n = nullptr; @@ -243,7 +240,7 @@ class SimpleHashBucket { bool update(const K& input_key, auto&& update_cb) { #ifndef GLOBAL_HASHSET_LOCK - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); #endif bool found{false}; for (auto& n : m_list) { @@ -343,7 +340,7 @@ template < typename K, typename V > bool SimpleHashMap< K, V >::try_erase(const K& key) { set_current_instance(this); return get_bucket(key).try_erase(key); -} +} /// This is a special atomic operation where user can insert_or_update_or_erase based on condition atomically. It /// performs differently based on certain conditions. diff --git a/include/sisl/fds/bitset.hpp b/include/sisl/fds/bitset.hpp index 616fb0e2..8a98cf4d 100644 --- a/include/sisl/fds/bitset.hpp +++ b/include/sisl/fds/bitset.hpp @@ -103,7 +103,10 @@ class BitsetImpl { void destroy(const bool destroy_words = true) { // destruct the BitWords if (destroy_words) { std::destroy(get_words(), end_words()); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" this->~bitset_serialized(); +#pragma GCC diagnostic pop } bool valid_bit(const uint64_t bit) const { return (bit + m_skip_bits) < m_nbits; } diff --git a/include/sisl/fds/bitword.hpp b/include/sisl/fds/bitword.hpp index 69c910eb..a3e25091 100644 --- a/include/sisl/fds/bitword.hpp +++ b/include/sisl/fds/bitword.hpp @@ -370,7 +370,10 @@ class Bitword { if (nbits == 1) { return (is_bit_set_reset(start, check_for_set)); } const word_t actual{extract(start, nbits)}; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" const word_t expected{static_cast< word_t >(check_for_set ? consecutive_bitmask[nbits - 1] : 0)}; +#pragma GCC diagnostic pop return (actual == expected); } @@ -544,14 +547,7 @@ class Bitword { word_t to_integer() const { return m_bits.get(); } - std::string to_string() const { - std::ostringstream oSS{}; - const word_t e = m_bits.get(); - for (uint8_t bit{0}; bit < bits(); ++bit) { - oSS << (((e & bit_mask[bit]) == bit_mask[bit]) ? '1' : '0'); - } - return oSS.str(); - } + std::string to_string() const { return fmt::format("{0:0>{1}b}", m_bits.get(), bits()); } void print() const { std::cout << to_string() << std::endl; } @@ -562,8 +558,11 @@ class Bitword { private: word_t extract(const uint8_t start, const uint8_t nbits) const { const uint8_t wanted_bits{std::min< uint8_t >(bits() - start, nbits)}; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" assert(wanted_bits > 0); const word_t mask{static_cast< word_t >(static_cast< word_t >(consecutive_bitmask[wanted_bits - 1]) << start)}; +#pragma GCC diagnostic pop return ((m_bits.get() & mask) >> start); } diff --git a/include/sisl/fds/freelist_allocator.hpp b/include/sisl/fds/freelist_allocator.hpp deleted file mode 100644 index 60879f39..00000000 --- a/include/sisl/fds/freelist_allocator.hpp +++ /dev/null @@ -1,173 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * Author/Developer(s): Harihara Kadayam - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#pragma once - -#include -#include -#include -#include -#include -#include - -#if defined __clang__ or defined __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wattributes" -#endif -#include -#if defined __clang__ or defined __GNUC__ -#pragma GCC diagnostic pop -#endif - -#include -#include "utils.hpp" - -namespace sisl { - -/** - * @brief FreeListAllocator is a high performing memory allocator for a fixed size object. It uses thread local to - * free list of same blob sizes. While jemalloc and tcmalloc maintains per thread buffer, it slightly outperforms them - * for the specific case it supports - fixed size. While others are generic size where it needs to additionally look for - * closest match for size requested, this allocator simply pops the top of the list and returns the buffer. It is as - * fast as it can get from memory allocation and deallocation perspective. - */ -struct free_list_header { - free_list_header* next; -}; - -#if defined(FREELIST_METRICS) || !defined(NDEBUG) -class FreeListAllocatorMetrics : public sisl::MetricsGroupWrapper { -public: - explicit FreeListAllocatorMetrics() : sisl::MetricsGroupWrapper("FreeListAllocator", "Singleton") { - REGISTER_COUNTER(freelist_alloc_hit, "freelist: Number of allocs from cache"); - REGISTER_COUNTER(freelist_alloc_miss, "freelist: Number of allocs from system"); - REGISTER_COUNTER(freelist_dealloc_passthru, "freelist: Number of dealloc not cached because of size mismatch"); - REGISTER_COUNTER(freelist_dealloc, "freelist: Number of deallocs to system"); - REGISTER_COUNTER(freelist_alloc_size, "freelist: size of alloc", sisl::_publish_as::publish_as_gauge); - REGISTER_COUNTER(freelist_cache_size, "freelist: cache size", sisl::_publish_as::publish_as_gauge); - - register_me_to_farm(); - } - FreeListAllocatorMetrics(const FreeListAllocatorMetrics&) = delete; - FreeListAllocatorMetrics(FreeListAllocatorMetrics&&) noexcept = delete; - FreeListAllocatorMetrics& operator=(const FreeListAllocatorMetrics&) = delete; - FreeListAllocatorMetrics& operator=(FreeListAllocatorMetrics&&) noexcept = delete; - - ~FreeListAllocatorMetrics() { deregister_me_from_farm(); } - - static FreeListAllocatorMetrics& instance() { - static FreeListAllocatorMetrics inst; - return inst; - } -}; - -#define COUNTER_INCREMENT_IF_ENABLED(p, v) COUNTER_INCREMENT(FreeListAllocatorMetrics::instance(), p, v); -#define COUNTER_DECREMENT_IF_ENABLED(p, v) COUNTER_DECREMENT(FreeListAllocatorMetrics::instance(), p, v); -#define INIT_METRICS [[maybe_unused]] auto& metrics = FreeListAllocatorMetrics::instance(); -#else -#define COUNTER_INCREMENT_IF_ENABLED(p, v) ; -#define COUNTER_DECREMENT_IF_ENABLED(p, v) ; -#define INIT_METRICS ; -#endif - -template < uint16_t MaxListCount, std::size_t Size > -class FreeListAllocatorImpl { -private: - free_list_header* m_head; - int64_t m_list_count; - -public: - FreeListAllocatorImpl() : m_head(nullptr), m_list_count(0) {} - FreeListAllocatorImpl(const FreeListAllocatorImpl&) = delete; - FreeListAllocatorImpl(FreeListAllocatorImpl&&) noexcept = delete; - FreeListAllocatorImpl& operator=(const FreeListAllocatorImpl&) = delete; - FreeListAllocatorImpl& operator=(FreeListAllocatorImpl&&) noexcept = delete; - - ~FreeListAllocatorImpl() { - free_list_header* hdr{m_head}; - while (hdr) { - free_list_header* const next{hdr->next}; - std::free(static_cast< void* >(hdr)); - hdr = next; - } - } - - uint8_t* allocate(const uint32_t size_needed) { - uint8_t* ptr; - INIT_METRICS; - - if (m_head == nullptr) { - ptr = static_cast< uint8_t* >(std::malloc(size_needed)); - COUNTER_INCREMENT_IF_ENABLED(freelist_alloc_miss, 1); - } else { - ptr = reinterpret_cast< uint8_t* >(m_head); - COUNTER_INCREMENT_IF_ENABLED(freelist_alloc_hit, 1); - m_head = m_head->next; - --m_list_count; - } - - return ptr; - } - - bool deallocate(uint8_t* const mem, const uint32_t size_alloced) { - if ((size_alloced != Size) || (m_list_count == MaxListCount)) { - std::free(static_cast< void* >(mem)); - return true; - } - auto* const hdr{reinterpret_cast< free_list_header* >(mem)}; - hdr->next = m_head; - m_head = hdr; - ++m_list_count; - - return true; - } -}; - -template < const uint16_t MaxListCount, const size_t Size > -class FreeListAllocator { -private: - folly::ThreadLocalPtr< FreeListAllocatorImpl< MaxListCount, Size > > m_impl; - -public: - static_assert((Size >= sizeof(uint8_t*)), "Size requested should be atleast a pointer size"); - - FreeListAllocator() { m_impl.reset(nullptr); } - FreeListAllocator(const FreeListAllocator&) = delete; - FreeListAllocator(FreeListAllocator&&) noexcept = delete; - FreeListAllocator& operator=(const FreeListAllocator&) = delete; - FreeListAllocator& operator=(FreeListAllocator&&) noexcept = delete; - - ~FreeListAllocator() { m_impl.reset(nullptr); } - - uint8_t* allocate(const uint32_t size_needed) { - if (sisl_unlikely(m_impl.get() == nullptr)) { m_impl.reset(new FreeListAllocatorImpl< MaxListCount, Size >()); } - return (m_impl->allocate(size_needed)); - } - - bool deallocate(uint8_t* const mem, const uint32_t size_alloced) { - if (sisl_unlikely(m_impl.get() == nullptr)) { - std::free(static_cast< void* >(mem)); - return true; - } else { - return m_impl->deallocate(mem, size_alloced); - } - } - - bool owns(uint8_t* const mem) const { return true; } - bool is_thread_safe_allocator() const { return true; } -}; -} // namespace sisl diff --git a/include/sisl/fds/obj_allocator.hpp b/include/sisl/fds/obj_allocator.hpp deleted file mode 100644 index 022256d2..00000000 --- a/include/sisl/fds/obj_allocator.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * Author/Developer(s): Harihara Kadayam - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#pragma once - -#include -#include - -#include "freelist_allocator.hpp" - -namespace sisl { - -static constexpr size_t FREELIST_CACHE_COUNT{200}; - -/** - * @brief Object Allocator is an object wrapper on top of freelist allocator. It provides convenient method to create - * a C++ object and destruct them. - */ -template < typename T, const size_t CacheCount = FREELIST_CACHE_COUNT > -class ObjectAllocator { -public: - ObjectAllocator() { - m_allocator = std::make_unique< sisl::FreeListAllocator< FREELIST_CACHE_COUNT, sizeof(T) > >(); - } - ObjectAllocator(const ObjectAllocator&) = delete; - ObjectAllocator(ObjectAllocator&&) noexcept = delete; - ObjectAllocator& operator=(const ObjectAllocator&) = delete; - ObjectAllocator& operator=(ObjectAllocator&&) noexcept = delete; - - template < class... Args > - static T* make_object(Args&&... args) { - uint8_t* const mem{get_obj_allocator()->m_allocator->allocate(sizeof(T))}; - T* const ptr{new (mem) T(std::forward< Args >(args)...)}; - return ptr; - } - - static void deallocate(T* const mem, const size_t obj_size = sizeof(T)) { - mem->~T(); - get_obj_allocator()->m_allocator->deallocate(reinterpret_cast< uint8_t* >(mem), obj_size); - } - - static std::unique_ptr< ObjectAllocator< T, CacheCount > > obj_allocator; - -private: - sisl::FreeListAllocator< FREELIST_CACHE_COUNT, sizeof(T) >* get_freelist_allocator() { return m_allocator.get(); } - -private: - std::unique_ptr< sisl::FreeListAllocator< FREELIST_CACHE_COUNT, sizeof(T) > > m_allocator; - - static ObjectAllocator< T, CacheCount >* get_obj_allocator() { - static ObjectAllocator< T, CacheCount > obj_allocator{}; - return &obj_allocator; - } -}; - -} // namespace sisl diff --git a/include/sisl/fds/stream_tracker.hpp b/include/sisl/fds/stream_tracker.hpp index 3cff7e64..9fe2402f 100644 --- a/include/sisl/fds/stream_tracker.hpp +++ b/include/sisl/fds/stream_tracker.hpp @@ -83,13 +83,13 @@ class StreamTracker { } void complete(int64_t start_idx, int64_t end_idx) { - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); auto start_bit = start_idx - m_slot_ref_idx; m_comp_slot_bits.set_bits(start_bit, end_idx - start_idx + 1); } void rollback(int64_t new_end_idx) { - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); if ((new_end_idx < m_slot_ref_idx) || (new_end_idx >= (m_slot_ref_idx + int64_cast(m_active_slot_bits.size())))) { throw std::out_of_range("Slot idx is not in range"); @@ -101,7 +101,7 @@ class StreamTracker { } T& at(int64_t idx) const { - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); if (idx < m_slot_ref_idx) { throw std::out_of_range("Slot idx is not in range"); } size_t nbit = idx - m_slot_ref_idx; @@ -123,7 +123,7 @@ class StreamTracker { bool is_completed = false; } ret; - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); if (idx < m_slot_ref_idx) { ret.is_out_of_range = true; } else { @@ -140,7 +140,7 @@ class StreamTracker { } size_t truncate(int64_t idx) { - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); auto upto_bit = idx - m_slot_ref_idx + 1; if (upto_bit <= 0) { return m_slot_ref_idx - 1; } @@ -150,7 +150,7 @@ class StreamTracker { size_t truncate() { if (AutoTruncate && (m_cmpltd_count_since_last_truncate.load(std::memory_order_acquire) == 0)) { return 0; } - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); // Find the first bit with 0 in it auto first_incomplete_bit = m_comp_slot_bits.get_next_reset_bit(0); @@ -194,12 +194,12 @@ class StreamTracker { void foreach_all_active(int64_t start_idx, const auto& cb) { _foreach_all(start_idx, false, cb); } int64_t completed_upto(int64_t search_hint_idx = 0) const { - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); return _upto(true /* completed */, search_hint_idx); } int64_t active_upto(int64_t search_hint_idx = 0) const { - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); return _upto(false /* completed */, search_hint_idx); } @@ -275,7 +275,7 @@ class StreamTracker { } void do_resize(size_t atleast_count) { - folly::SharedMutexWritePriority::WriteHolder holder(m_lock); + auto holder = std::unique_lock< folly::SharedMutex >(m_lock); // Check if we already resized enough if (atleast_count < m_alloced_slots) { return; } @@ -309,7 +309,7 @@ class StreamTracker { } void _foreach_contiguous(int64_t start_idx, bool completed_only, const auto& cb) { - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); auto upto = _upto(completed_only, start_idx); for (auto idx = start_idx; idx <= upto; ++idx) { auto proceed = cb(idx, upto, *(get_slot_data(idx - m_slot_ref_idx))); @@ -318,7 +318,7 @@ class StreamTracker { } void _foreach_all(int64_t start_idx, bool completed_only, const auto& cb) { - folly::SharedMutexWritePriority::ReadHolder holder(m_lock); + auto holder = std::shared_lock< folly::SharedMutex >(m_lock); auto search_bit = std::max(0l, (start_idx - m_slot_ref_idx)); do { search_bit = completed_only ? m_comp_slot_bits.get_next_set_bit(search_bit) @@ -333,7 +333,7 @@ class StreamTracker { private: // Mutex to protect the completion of last commit info - mutable folly::SharedMutexWritePriority m_lock; + mutable folly::SharedMutex m_lock; // A bitset that covers the completion and truncation sisl::AtomicBitset m_comp_slot_bits; diff --git a/include/sisl/grpc/rpc_call.hpp b/include/sisl/grpc/rpc_call.hpp index 5407265e..258ba4c8 100644 --- a/include/sisl/grpc/rpc_call.hpp +++ b/include/sisl/grpc/rpc_call.hpp @@ -34,10 +34,10 @@ SISL_LOGGING_DECL(grpc_server) #define RPC_SERVER_LOG(level, msg, ...) \ LOG##level##MOD_FMT(grpc_server, ([&](fmt::memory_buffer& buf, const char* __m, auto&&... args) -> bool { \ fmt::vformat_to(fmt::appender{buf}, std::string_view{"[{}:{}] [RPC={} id={}] "}, \ - fmt::make_format_args(file_name(__FILE__), __LINE__, \ - m_rpc_info->m_rpc_name, request_id())); \ + fmt::make_format_args(unmove(file_name(__FILE__)), unmove(__LINE__), \ + m_rpc_info->m_rpc_name, unmove(request_id()))); \ fmt::vformat_to(fmt::appender{buf}, fmt::string_view{__m}, \ - fmt::make_format_args(std::forward< decltype(args) >(args)...)); \ + fmt::make_format_args(args...)); \ return true; \ }), \ msg, ##__VA_ARGS__); diff --git a/include/sisl/grpc/rpc_client.hpp b/include/sisl/grpc/rpc_client.hpp index c702e5ef..1b7b3ec2 100644 --- a/include/sisl/grpc/rpc_client.hpp +++ b/include/sisl/grpc/rpc_client.hpp @@ -27,7 +27,13 @@ #include #include #include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #include +#pragma GCC diagnostic pop +#pragma GCC diagnostic pop #include #include @@ -317,8 +323,8 @@ class GrpcAsyncClient : public GrpcBaseClient { GrpcAsyncClient(const std::string& server_addr, const std::shared_ptr< sisl::GrpcTokenClient > token_client, const std::string& target_domain = "", const std::string& ssl_cert = "", int max_receive_msg_size = 0, int max_send_msg_size = 0) : - GrpcBaseClient(server_addr, token_client, target_domain, ssl_cert, - max_receive_msg_size, max_send_msg_size) {} + GrpcBaseClient(server_addr, token_client, target_domain, ssl_cert, max_receive_msg_size, + max_send_msg_size) {} GrpcAsyncClient(const std::string& server_addr, const std::string& target_domain = "", const std::string& ssl_cert = "") : diff --git a/include/sisl/logging/logging.h b/include/sisl/logging/logging.h index ebcf7a43..d0e57dc1 100644 --- a/include/sisl/logging/logging.h +++ b/include/sisl/logging/logging.h @@ -127,6 +127,11 @@ constexpr const char* file_name(const char* const str) { return str_slant(str) ? #define LOGERRORMOD(mod, msg, ...) LOGERRORMOD_USING_LOGGER(mod, sisl::logging::GetLogger(), msg, ##__VA_ARGS__) #define LOGCRITICALMOD(mod, msg, ...) LOGCRITICALMOD_USING_LOGGER(mod, sisl::logging::GetLogger(), msg, ##__VA_ARGS__) +template < typename T > +const T& unmove(T&& x) { + return x; +} + /* Extension macros to support custom formatting of messages */ #if __cplusplus > 201703L #define _LOG_WITH_CUSTOM_FORMATTER(lvl, method, mod, logger, is_flush, formatter, msg, ...) \ @@ -134,7 +139,7 @@ constexpr const char* file_name(const char* const str) { return str_slant(str) ? fmt::memory_buffer _log_buf{}; \ const auto& cb{formatter}; \ [[likely]] if (cb(_log_buf, msg __VA_OPT__(, ) __VA_ARGS__)) { \ - fmt::vformat_to(fmt::appender{_log_buf}, fmt::string_view{"{}"}, fmt::make_format_args('\0')); \ + fmt::vformat_to(fmt::appender{_log_buf}, fmt::string_view{"{}"}, fmt::make_format_args(unmove('\0'))); \ _l->method(_log_buf.data()); \ if (is_flush) { _l->flush(); } \ } \ @@ -278,9 +283,8 @@ constexpr const char* file_name(const char* const str) { return str_slant(str) ? #define RELEASE_ASSERT(cond, m, ...) \ _GENERIC_ASSERT( \ 0, cond, \ - [](fmt::memory_buffer& buf, const char* msg, auto&&... args) -> bool { \ - fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, \ - fmt::make_format_args(std::forward< decltype(args) >(args)...)); \ + [](fmt::memory_buffer& buf, const char* msg, auto... args) -> bool { \ + fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, fmt::make_format_args(args...)); \ return true; \ }, \ m, ##__VA_ARGS__) @@ -308,8 +312,7 @@ constexpr const char* file_name(const char* const str) { return str_slant(str) ? _GENERIC_ASSERT( \ 1, cond, \ [](fmt::memory_buffer& buf, const char* msg, auto&&... args) -> bool { \ - fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, \ - fmt::make_format_args(std::forward< decltype(args) >(args)...)); \ + fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, fmt::make_format_args(args...)); \ return true; \ }, \ m, ##__VA_ARGS__) @@ -486,7 +489,7 @@ extern std::filesystem::path get_base_dir(); template < typename... Args > std::string format_log_msg(const char* const msg, Args&&... args) { fmt::memory_buffer buf{}; - fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, fmt::make_format_args(std::forward< Args >(args)...)); + fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, fmt::make_format_args(args...)); return fmt::to_string(buf); } extern std::string format_log_msg(); @@ -497,8 +500,8 @@ void _cmp_assert_with_msg(fmt::memory_buffer& buf, const char* const msg, T1&& v fmt::vformat_to(fmt::appender{buf}, fmt::string_view{"******************** Assertion failure: =====> Expected '{}' to be {} to '{}' "}, - fmt::make_format_args(std::forward< T1 >(val1), std::forward< T2 >(op), std::forward< T3 >(val2))); - fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, fmt::make_format_args(std::forward< Args >(args)...)); + fmt::make_format_args(val1, op, val2)); + fmt::vformat_to(fmt::appender{buf}, fmt::string_view{msg}, fmt::make_format_args(args...)); } template < typename... Args > diff --git a/prepare.sh b/prepare.sh deleted file mode 100755 index ec92a7be..00000000 --- a/prepare.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -eu - -echo -n "Exporting custom recipes..." -echo -n "folly." -conan export 3rd_party/folly folly/nu2.2023.12.18.00@ >/dev/null -echo -n "userpace rcu." -conan export 3rd_party/userspace-rcu userspace-rcu/nu2.0.14.0@ >/dev/null -echo "done." diff --git a/prepare_v2.sh b/prepare_v2.sh index 480965bb..60dad499 100755 --- a/prepare_v2.sh +++ b/prepare_v2.sh @@ -3,6 +3,6 @@ set -eu echo -n "Exporting custom recipes..." echo -n "folly." conan export 3rd_party/folly --name folly --version nu2.2023.12.18.00 >/dev/null -echo -n "folly." +echo -n "userspace rcu." conan export 3rd_party/userspace-rcu --name userspace-rcu --version nu2.0.14.0 >/dev/null echo "done." diff --git a/src/cache/tests/test_range_hashmap.cpp b/src/cache/tests/test_range_hashmap.cpp index d6de8557..158d30d0 100644 --- a/src/cache/tests/test_range_hashmap.cpp +++ b/src/cache/tests/test_range_hashmap.cpp @@ -28,7 +28,7 @@ using namespace sisl; SISL_LOGGING_INIT(test_hashmap) -static uint32_t g_max_offset; +static uint32_t g_max_offset{UINT16_MAX}; static constexpr uint32_t per_val_size = 128; static thread_local std::random_device g_rd{}; diff --git a/src/fds/CMakeLists.txt b/src/fds/CMakeLists.txt index 04ca1753..f5f55d66 100644 --- a/src/fds/CMakeLists.txt +++ b/src/fds/CMakeLists.txt @@ -60,27 +60,6 @@ target_sources(concurrent_insert_vector_bench PRIVATE target_link_libraries(concurrent_insert_vector_bench sisl_buffer benchmark::benchmark) add_test(NAME ConcurrentVectorBench COMMAND concurrent_insert_vector_bench) -add_executable(obj_allocator_benchmark) -target_sources(obj_allocator_benchmark PRIVATE - tests/obj_allocator_benchmark.cpp - ) -target_link_libraries(obj_allocator_benchmark sisl_buffer benchmark::benchmark) -add_test(NAME ObjAllocatorBenchmark COMMAND obj_allocator_benchmark) - -add_executable(test_obj_allocator) -target_sources(test_obj_allocator PRIVATE - tests/test_obj_allocator.cpp - ) -target_link_libraries(test_obj_allocator sisl_buffer) -add_test(NAME ObjAlloc COMMAND test_obj_allocator) - -add_executable(test_cb_mutex) -target_sources(test_cb_mutex PRIVATE - tests/test_cb_mutex.cpp - ) -target_link_libraries(test_cb_mutex sisl_logging folly::folly GTest::gtest) -#add_test(NAME TestCBMutex COMMAND test_cb_mutex) - add_executable(test_sg_list) target_sources(test_sg_list PRIVATE tests/test_sg_list.cpp diff --git a/src/fds/callback_mutex.hpp b/src/fds/callback_mutex.hpp deleted file mode 100644 index 64c847b7..00000000 --- a/src/fds/callback_mutex.hpp +++ /dev/null @@ -1,174 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * Author/Developer(s): Harihara Kadayam - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#include -#include -#include -#include "sisl/fds/vector_pool.hpp" -#include -#include - -// Generate the metafunction -BOOST_TTI_HAS_MEMBER_FUNCTION(try_lock_shared) -BOOST_TTI_HAS_MEMBER_FUNCTION(unlock_shared) - -template < typename T > -static constexpr bool try_lock_shared_check = has_member_function_try_lock_shared< T, bool >::value; - -template < typename T > -static constexpr bool unlock_shared_check = has_member_function_unlock_shared< T, void >::value; - -namespace sisl { -using post_lock_cb_t = std::function< void(void) >; - -class _cb_wait_q { -public: - _cb_wait_q() = default; - ~_cb_wait_q() = default; - - void add_cb(post_lock_cb_t&& cb) { - folly::SharedMutexWritePriority::WriteHolder holder{m_waitq_lock}; - if (m_wait_q == nullptr) { m_wait_q = sisl::VectorPool< post_lock_cb_t >::alloc(); } - m_wait_q->emplace_back(std::move(cb)); - } - - bool drain_cb() { - std::vector< post_lock_cb_t >* wait_q{nullptr}; - { - folly::SharedMutexWritePriority::WriteHolder holder{m_waitq_lock}; - std::swap(wait_q, m_wait_q); - } - - if (wait_q) { - for (const auto& cb : *wait_q) { - cb(); - } - sisl::VectorPool< post_lock_cb_t >::free(wait_q); - } - return (wait_q != nullptr); - } - - bool empty() const { - folly::SharedMutexWritePriority::ReadHolder holder{m_waitq_lock}; - return ((m_wait_q == nullptr) || (m_wait_q->empty())); - } - -private: - mutable folly::SharedMutexWritePriority m_waitq_lock; - std::vector< post_lock_cb_t >* m_wait_q{nullptr}; -}; - -template < typename MutexImpl > -class CallbackMutex { -public: - explicit CallbackMutex() = default; - ~CallbackMutex() = default; - - bool try_lock(post_lock_cb_t&& cb) { - if (m_base_mutex.try_lock()) { - cb(); - return true; - } - m_q.add_cb(std::move(cb)); - return false; - } - - template < class I = MutexImpl > - typename std::enable_if< try_lock_shared_check< I >, bool >::type try_lock_shared(post_lock_cb_t&& cb) { - if (m_base_mutex.try_lock_shared()) { - cb(); - return true; - } - m_q.add_cb(std::move(cb)); - return false; - } - - bool unlock() { - auto drained = m_q.drain_cb(); - m_base_mutex.unlock(); - return drained; - } - - template < class I = MutexImpl > - typename std::enable_if< unlock_shared_check< I >, void >::type unlock_shared() { - m_base_mutex.unlock_shared(); - if (!m_q.empty()) { - // If Q is not empty, try to lock the base mutex (which callers wait on) and if successful, - // we can drain the q. If unsuccessful, ignore it, because next unlock on elements in the q - // will do the same - if (m_base_mutex.try_lock()) { - m_q.drain_cb(); - m_base_mutex.unlock(); - } - } - } - - template < class I = MutexImpl > - typename std::enable_if< !unlock_shared_check< I >, void >::type unlock_shared() { - m_base_mutex.unlock(); - } - - static constexpr bool shared_mode_supported = try_lock_shared_check< MutexImpl >; -#if 0 - template < class I = MutexImpl > - static constexpr typename std::enable_if< try_lock_shared_check< I >, bool >::type shared_mode_supported() { - return true; - } - - template < class I = MutexImpl > - static constexpr typename std::enable_if< !try_lock_shared_check< I >, bool >::type shared_mode_supported() { - return false; - } -#endif - -private: - MutexImpl m_base_mutex; - _cb_wait_q m_q; -}; - -template < typename MutexImpl > -class CBUniqueLock { -public: - CBUniqueLock(CallbackMutex< MutexImpl >& cb_mtx, post_lock_cb_t&& cb) : m_cb_mtx{cb_mtx} { - m_locked = m_cb_mtx.try_lock(std::move(cb)); - } - - ~CBUniqueLock() { - if (m_locked) { m_cb_mtx.unlock(); } - } - -private: - CallbackMutex< MutexImpl >& m_cb_mtx; - bool m_locked{false}; -}; - -template < typename MutexImpl > -class CBSharedLock { -public: - CBSharedLock(CallbackMutex< MutexImpl >& cb_mtx, post_lock_cb_t&& cb) : m_cb_mtx{cb_mtx} { - m_locked = m_cb_mtx.try_lock_shared(std::move(cb)); - } - - ~CBSharedLock() { - if (m_locked) { m_cb_mtx.unlock_shared(); } - } - -private: - CallbackMutex< MutexImpl >& m_cb_mtx; - bool m_locked{false}; -}; - -} // namespace sisl diff --git a/src/fds/memvector.hpp b/src/fds/memvector.hpp deleted file mode 100644 index f4834bb3..00000000 --- a/src/fds/memvector.hpp +++ /dev/null @@ -1,403 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * Author/Developer(s): Harihara Kadayam - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "buffer.hpp" - -namespace sisl { -#define round_off(val, rnd) ((((val)-1) / (rnd)) + 1) - -#if 0 -#define FIRST_8BITS(n) (n & 0x00ff) - -#define gen_new_tag(size, offset) ((LeftShifts< 8 >()[encode(offset)]) | encode(size)) -#define set_offset_in_tag(tag, offset) ((LeftShifts< 8 >()[encode(offset)]) | FIRST_8BITS(tag)) -#define set_size_in_tag(tag, size) (((tag)&0xff00) | encode(size)) -#define get_offset_in_tag(tag) (actual_size(((tag)&0xff00) >> 8)) -#define get_size_in_tag(tag) (actual_size((tag)&0xff)) -#endif - -struct __attribute__((__may_alias__)) __mempiece_tag { - uint16_t m_size : 8; /* Size shrinked by SizeMultipler */ - uint16_t m_offset : 8; /* Offset within the mem piece */ - - uint16_t to_integer() { return *((uint16_t*)this); } -} __attribute((packed)); - -struct MemPiece : public sisl::ObjLifeCounter< MemPiece > { - uint8_t* m_mem; - uint32_t m_size; - uint32_t m_offset; - - MemPiece(uint8_t* mem, uint32_t size, uint32_t offset) : - ObjLifeCounter(), m_mem(mem), m_size(size), m_offset(offset) {} - - MemPiece() : MemPiece(nullptr, 0, 0) {} - MemPiece(const MemPiece& other) : - ObjLifeCounter(), m_mem(other.m_mem), m_size(other.m_size), m_offset(other.m_offset) {} - - ~MemPiece() {} - - void set_ptr(uint8_t* ptr) { m_mem = ptr; } - void set_size(uint32_t size) { m_size = size; } - void set_offset(uint32_t offset) { m_offset = offset; } - void reset() { set(nullptr, 0, 0); } - - void set(uint8_t* ptr, uint32_t size, uint32_t offset) { - m_mem = ptr; - m_size = size; - m_offset = offset; - } - - uint8_t* ptr() const { return m_mem; } - uint32_t size() const { return m_size; } - uint32_t offset() const { return m_offset; } - uint32_t end_offset() const { return m_size + m_offset; } - - uint8_t* get(uint32_t* psize, uint8_t* poff) const { - *psize = size(); - *poff = offset(); - return ptr(); - } - - std::string to_string() const { - std::stringstream ss; - ss << "ptr = " << (void*)ptr() << " size = " << size() << " offset = " << offset(); - return ss.str(); - } -} __attribute__((packed)); - -struct MemVector : public sisl::ObjLifeCounter< MemVector > { -private: - std::vector< MemPiece > m_list; - /* we need recursive mutex because we can call init() function while holding it - * and which internally can again call get_blob which also takes mutex. - */ - mutable std::recursive_mutex m_mtx; - std::atomic< uint8_t > m_refcnt; - -public: - MemVector(uint8_t* ptr, uint32_t size, uint32_t offset) : ObjLifeCounter(), m_refcnt(0) { - m_list.reserve(1); - MemPiece m(ptr, size, offset); - m_list.push_back(m); - assert(size || (ptr == nullptr)); - } - - MemVector() : ObjLifeCounter(), m_refcnt(0) { m_list.reserve(1); } - ~MemVector() { m_list.erase(m_list.begin(), m_list.end()); } - - friend void intrusive_ptr_add_ref(MemVector* mvec) { mvec->m_refcnt++; } - - friend void intrusive_ptr_release(MemVector* mvec) { - if (mvec->m_refcnt.fetch_sub(1, std::memory_order_relaxed) != 1) { return; } - for (auto i = 0u; i < mvec->m_list.size(); i++) { - if (mvec->m_list[i].ptr() != nullptr) { - free(mvec->m_list[i].ptr()); - } else { - assert(0); - } - } - delete (mvec); - } - - void reserve(size_t count) { m_list.reserve(count); } - std::vector< MemPiece > get_m_list() const { return m_list; } - void copy(const MemVector& other) { - assert(other.m_refcnt > 0); - m_list = other.get_m_list(); - } - uint32_t npieces() const { return (m_list.size()); } - - void set(uint8_t* ptr, uint32_t size, uint32_t offset) { - std::unique_lock< std::recursive_mutex > mtx(m_mtx); - m_list.erase(m_list.begin(), m_list.end()); - MemPiece m(ptr, size, offset); - m_list.push_back(m); - } - - void set(const sisl::blob& b, uint32_t offset = 0) { set(b.bytes, b.size, offset); } - - void get(sisl::blob* outb, uint32_t offset = 0) const { - uint32_t ind = 0; - std::unique_lock< std::recursive_mutex > mtx(m_mtx); - if ((m_list.size() && bsearch(offset, -1, &ind))) { - auto piece_offset = m_list.at(ind).offset(); - assert(piece_offset <= offset); - uint32_t delta = offset - piece_offset; - assert(delta < m_list.at(ind).size()); - outb->bytes = m_list.at(ind).ptr() + delta; - outb->size = m_list.at(ind).size() - delta; - } else { - assert(0); - } - } - - const MemPiece& get_nth_piece(uint8_t nth) const { - if (nth < m_list.size()) { - return m_list.at(nth); - } else { - assert(0); - return m_list.at(0); - } - } - - MemPiece& get_nth_piece_mutable(uint8_t nth) { - if (nth < m_list.size()) { - return m_list[nth]; - } else { - assert(0); - return m_list[0]; - } - } - - std::string to_string() const { - auto n = npieces(); - std::stringstream ss; - - if (n > 1) ss << "Pieces = " << n << "\n"; - for (auto i = 0U; i < n; i++) { - auto& p = get_nth_piece(i); - ss << "MemPiece[" << i << "]: " << p.to_string() << ((n > 1) ? "\n" : ""); - } - return ss.str(); - } - - /* This method will try to set or append the offset to the memory piece. However, if there is already an entry - * within reach for given offset/size, it will reject the entry and return false. Else it sets or adds the entry */ - bool append(uint8_t* ptr, uint32_t offset, uint32_t size) { - std::unique_lock< std::recursive_mutex > mtx(m_mtx); - bool added = false; - MemPiece mp(ptr, size, offset); - added = add_piece_to_list(mp); - return added; - } - - void push_back(const MemPiece& piece) { m_list.push_back(piece); } - - /* - * This method actually appends the memvector by computing offset from last entry. Hence it provides a way for - * caller to build contiguous destination from scattered source - */ - void concat_back(uint8_t* ptr, uint32_t size) { - uint32_t offset = 0u; - if (m_list.size()) { - auto& p = m_list.back(); - offset = p.offset() + p.size(); - } - m_list.emplace_back(ptr, size, offset); - } - - MemPiece& insert_at(uint32_t ind, const MemPiece& piece) { - assert(ind <= m_list.size()); - auto it = m_list.emplace((m_list.begin() + ind), piece); - return *it; - } - - MemPiece& insert_at(uint32_t ind, uint8_t* ptr, uint32_t size, uint32_t offset) { - auto mp = MemPiece(ptr, size, offset); - return insert_at(ind, mp); - } - - uint32_t size(uint32_t offset, uint32_t size) const { - std::unique_lock< std::recursive_mutex > mtx(m_mtx); - uint32_t s = 0; - uint32_t offset_read = 0; - bool start = false; - for (auto it = m_list.begin(); it < m_list.end(); ++it) { - offset_read += (*it).offset(); - if ((offset_read + (*it).size()) >= offset && !start) { - if (offset_read + (*it).size() >= (offset + size)) { - s = size; - break; - } else { - s = (*it).size() - (offset - offset_read); - start = true; - continue; - } - } else if ((offset_read + (*it).size()) < offset) { - continue; - } - - assert(start); - - if ((offset_read + (*it).size()) >= (offset + size)) { - if (offset_read >= (offset + size)) { break; } - s += (*it).size() - (offset_read + (*it).size() - (offset + size)); - break; - } - s = s + (*it).size(); - } - return s; - } - - uint32_t size() const { - std::unique_lock< std::recursive_mutex > mtx(m_mtx); - uint32_t s = 0; - for (auto it = m_list.begin(); it < m_list.end(); ++it) { - s += (*it).size(); - } - return s; - } - - bool find_index(uint32_t offset, boost::optional< int > ind_hint, uint32_t* out_ind) const { - *out_ind = 0; - return (bsearch(offset, ind_hint.get_value_or(-1), out_ind)); - } - - struct cursor_t { - cursor_t() : m_ind(-1) {} - int m_ind; - }; - - uint32_t insert_missing_pieces(uint32_t offset, uint32_t size, - std::vector< std::pair< uint32_t, uint32_t > >& missing_mp) { - uint32_t new_ind; - std::unique_lock< std::recursive_mutex > mtx(m_mtx); - cursor_t c; - uint32_t inserted_size = 0; - -#ifndef NDEBUG - auto temp_offset = offset; - auto temp_size = size; -#endif - while (size != 0) { - bool found = find_index(offset, c.m_ind, &new_ind); - if (found) { - auto& mp = get_nth_piece(new_ind); - /* check for pointer */ - if (mp.ptr() == nullptr) { - /* add pair */ - std::pair ret(mp.offset(), (mp.end_offset() - mp.offset())); - missing_mp.push_back(ret); - } - if ((offset + size) <= mp.end_offset()) { - offset += size; - size = 0; - } else { - size = size - (mp.end_offset() - offset); - offset = mp.end_offset(); // Move the offset to the start of next mp. - } - - } else if (new_ind < npieces()) { - auto& mp = get_nth_piece(new_ind); - auto sz = mp.offset() - offset; - if (size < sz) { sz = size; } - insert_at(new_ind, nullptr, sz, offset); - inserted_size += sz; - - /* add pair */ - std::pair ret(offset, sz); - missing_mp.push_back(ret); - - size -= sz; - offset = offset + sz; - } else { - insert_at(new_ind, nullptr, size, offset); - inserted_size += size; - std::pair ret(offset, size); - /* add pair */ - missing_mp.push_back(ret); - - offset += size; - size = 0; - } - c.m_ind = new_ind; - } -#ifndef NDEBUG - assert(offset == (temp_offset + temp_size)); -#endif - return inserted_size; - } - - bool update_missing_piece(uint32_t offset, uint32_t size, uint8_t* ptr, const auto& init_cb) { - uint32_t new_ind; - bool inserted = false; - std::unique_lock< std::recursive_mutex > mtx(m_mtx); - [[maybe_unused]] bool found = find_index(offset, -1, &new_ind); - assert(found); - auto& mp = get_nth_piece_mutable(new_ind); - if (mp.ptr() == nullptr) { - mp.set_ptr(ptr); - inserted = true; - init_cb(); - } - assert(size == mp.size()); - return inserted; - } - -private: - bool add_piece_to_list(const MemPiece& mp) { - - uint32_t ind; - // If we found an offset in search we got to fail the add - if (bsearch(mp.offset(), -1, &ind)) { return false; } - - // If we overlap with the next entry where it is to be inserted, fail the add - if ((ind < m_list.size()) && (mp.end_offset() > m_list.at(ind).offset())) { return false; } - - m_list.emplace((m_list.begin() + ind), mp); - return true; - } - - int compare(uint32_t search_offset, const MemPiece& mp) const { - auto mp_offset = mp.offset(); - if (search_offset == mp_offset) { - return 0; - } else if (search_offset < mp_offset) { - return 1; - } else { - return ((mp_offset + mp.size()) > search_offset) ? 0 /* Overlaps */ : -1; - } - } - - /* it return the first index which is greater then or equal to offset. - * If is more then the last element in m_list.size() then it - * return m_list.size(). If it is smaller then first element, it return - * zero. - */ - bool bsearch(uint32_t offset, int start, uint32_t* out_ind) const { - int end = m_list.size(); - uint32_t mid = 0; - - while ((end - start) > 1) { - mid = start + (end - start) / 2; - int x = compare(offset, m_list.at(mid)); - if (x == 0) { - *out_ind = mid; - return true; - } else if (x > 0) { - end = mid; - } else { - start = mid; - } - } - - *out_ind = end; - return false; - } -}; -} // namespace sisl diff --git a/src/fds/tagged_ptr.hpp b/src/fds/tagged_ptr.hpp deleted file mode 100644 index aa2a871b..00000000 --- a/src/fds/tagged_ptr.hpp +++ /dev/null @@ -1,135 +0,0 @@ -// tagged pointer, for aba prevention -// -// Copyright (C) 2008, 2009 Tim Blechmann, based on code by Cory Nelson -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) -/* - * Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - -/* - * Modified by Kadayam, Hari on 03/06/17 for atomic smart pointer. - */ -#pragma once - -#include - -namespace sisl { -#if defined(__x86_64__) || defined(_M_X64) - -#define TAGGED_PTR_MASK 0x0000FFFFFFFFFFFFUL -#define TAGGED_COUNT_MASK 0xFFFF000000000000 -#define FIRST_TAG_VALUE 0x0001000000000000UL -#define TAGGED_PTR_BITS 48 - -template < typename T > -struct tagged_ptr { - typedef uint64_t compressed_ptr_t; - typedef uint16_t tag_t; - -private: - static const int tag_index = 3; - -private: - union cast_unit { - compressed_ptr_t ptr; - tag_t tag[4]; - }; - - cast_unit m_value; - -public: - tagged_ptr() { m_value.ptr = 0; } - tagged_ptr(tagged_ptr const& o) : m_value(o.m_value) {} - tagged_ptr(T* ptr, tag_t t) : m_value(pack_value(ptr, t)) {} - tagged_ptr(compressed_ptr_t cp) : m_value(cp) {} - - tagged_ptr& operator=(tagged_ptr const& p) = default; - - void set(T* p, tag_t t) { m_value = pack_value(p, t); } - - bool operator==(volatile tagged_ptr const& o) const { return (m_value.ptr == o.m_value.ptr); } - bool operator!=(volatile tagged_ptr const& o) const { return !operator==(o); } - - T* get_ptr() const { - auto p = m_value.ptr; - return (extract_ptr(p)); - } - - void set_ptr(T* p) { - tag_t t = get_tag(); - m_value = pack_value(p, t); - } - - tag_t get_tag() const { - auto p = m_value.ptr; - return extract_tag(p); - } - - void set_tag(tag_t t) { - T* p = get_ptr(); - m_value = pack_value(p, t); - } - - void inc_tag() { m_value.ptr += FIRST_TAG_VALUE; } - - void inc_tag(int count) { m_value.ptr += (FIRST_TAG_VALUE * count); } - - void dec_tag() { m_value.ptr -= FIRST_TAG_VALUE; } - - void dec_tag(int count) { m_value.ptr -= (FIRST_TAG_VALUE * count); } - - T& operator*() const { return *get_ptr(); } - - T* operator->() const { return get_ptr(); } - - operator bool(void) const { return get_ptr() != 0; } - - compressed_ptr_t get_packed() const { return m_value.ptr; } - -private: - static cast_unit pack_value(T* ptr, uint16_t tag) { - cast_unit ret; - ret.ptr = compressed_ptr_t(ptr); - ret.tag[tag_index] = tag; - return ret; - } - - static T* extract_ptr(volatile compressed_ptr_t const& i) { return (T*)(i & TAGGED_PTR_MASK); } - - static tag_t extract_tag(volatile compressed_ptr_t const& i) { - cast_unit cu; - cu.ptr = i; - return cu.tag[tag_index]; - } - -} __attribute__((packed)); - -#else -#error unsupported platform for tagged pointer -#endif -} // namespace sisl diff --git a/src/fds/tests/obj_allocator_benchmark.cpp b/src/fds/tests/obj_allocator_benchmark.cpp deleted file mode 100644 index 570f6d30..00000000 --- a/src/fds/tests/obj_allocator_benchmark.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * Author/Developer(s): Harihara Kadayam - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#include -#include -#include -#include -#include -#include - -#include -#include "sisl/logging/logging.h" -#include "sisl/options/options.h" - -#include "sisl/metrics/metrics.hpp" -#include "sisl/fds/obj_allocator.hpp" - -SISL_LOGGING_INIT(HOMESTORE_LOG_MODS) -RCU_REGISTER_INIT - -namespace { -std::mutex s_print_mutex; -constexpr size_t ITERATIONS{10000000}; -constexpr size_t THREADS{8}; - -struct my_request { - int m_a; - int m_b[10]; - std::string m_c; - uint64_t m_d; -}; - -void setup() {} -void test_malloc(benchmark::State& state) { - uint64_t counter{0}; - static thread_local std::random_device rd{}; - static thread_local std::default_random_engine engine{rd()}; - - for ([[maybe_unused]] auto si : state) { // Loops up to iteration count - my_request* req; - benchmark::DoNotOptimize(req = new my_request()); - req->m_a = 10; - req->m_b[0] = 100; - std::uniform_int_distribution< uint64_t > dist{0, RAND_MAX}; - req->m_d = req->m_a * dist(engine); - counter += req->m_d; - delete (req); - } - { - std::scoped_lock< std::mutex > lock{s_print_mutex}; - std::cout << "Counter = " << counter << std::endl; - } -} - -void test_obj_alloc(benchmark::State& state) { - uint64_t counter{0}; - static thread_local std::random_device rd{}; - static thread_local std::default_random_engine engine{rd()}; - for ([[maybe_unused]] auto si : state) { // Loops up to iteration count - my_request* req; - benchmark::DoNotOptimize(req = sisl::ObjectAllocator< my_request >::make_object()); - req->m_a = 10; - req->m_b[0] = 100; - std::uniform_int_distribution< uint64_t > dist{0, RAND_MAX}; - req->m_d = req->m_a * dist(engine); - counter += req->m_d; - sisl::ObjectAllocator< my_request >::deallocate(req); - } - - { - std::scoped_lock< std::mutex > lock{s_print_mutex}; - std::cout << "Counter = " << counter << std::endl; - } -} -} // namespace - -BENCHMARK(test_malloc)->Iterations(ITERATIONS)->Threads(THREADS); -BENCHMARK(test_obj_alloc)->Iterations(ITERATIONS)->Threads(THREADS); - -SISL_OPTIONS_ENABLE(logging) -int main(int argc, char** argv) { - SISL_OPTIONS_LOAD(argc, argv, logging) - setup(); - ::benchmark::Initialize(&argc, argv); - ::benchmark::RunSpecifiedBenchmarks(); - // std::cout << "Metrics: " << sisl::MetricsFarm::getInstance().get_result_in_json().dump(4) << "\n"; -} diff --git a/src/fds/tests/test_cb_mutex.cpp b/src/fds/tests/test_cb_mutex.cpp deleted file mode 100644 index b1e10b35..00000000 --- a/src/fds/tests/test_cb_mutex.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * Author/Developer(s): Harihara Kadayam - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#include -#include -#include -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#include -#pragma GCC diagnostic pop - -#include "callback_mutex.hpp" -#include "sisl/fds/utils.hpp" - -SISL_LOGGING_INIT(test_cb_mutex) - -static uint64_t g_prev_val{0}; -static uint64_t g_cur_val{1}; - -template < typename MutexImpl > -class CBMutexTest : public testing::Test { -protected: - sisl::CallbackMutex< MutexImpl > m_cb_mtx; - -protected: - void thread_unique_fn(uint64_t count_per_thread) { - uint64_t count{0}; - for (uint64_t i{0}; i < count_per_thread; ++i) { - sisl::CBUniqueLock< MutexImpl > h(m_cb_mtx, [&count]() { - assert((g_prev_val + 1) == g_cur_val); - g_prev_val = g_cur_val++; - ++count; - }); - } - - LOGINFO("Thread attempted {} of exclusive callbacks", count); - } - - template < typename I = MutexImpl > - typename std::enable_if< sisl::CallbackMutex< I >::shared_mode_supported, void >::type - thread_shared_fn(uint64_t count_per_thread) { - uint64_t count{0}; - for (uint64_t i{0}; i < count_per_thread; ++i) { - sisl::CBSharedLock< MutexImpl > h(m_cb_mtx, [&count]() { - assert((g_prev_val + 1) == g_cur_val); - ++count; - }); - } - - LOGINFO("Thread attempted {} of shared callbacks", count); - } - - template < typename I = MutexImpl > - typename std::enable_if< !sisl::CallbackMutex< I >::shared_mode_supported, void >::type thread_shared_fn(uint64_t) { - assert(0); - } - - void run_lock_unlock() { - auto num_threads = SISL_OPTIONS["num_threads"].as< uint32_t >(); - auto num_iters = sisl::round_up(SISL_OPTIONS["num_iters"].as< uint64_t >(), num_threads); - - uint32_t unique_threads{num_threads}; - uint32_t shared_threads{0}; - if (sisl::CallbackMutex< MutexImpl >::shared_mode_supported) { - unique_threads = std::max(1u, num_threads / 4); - shared_threads = std::max(1u, num_threads - unique_threads); - } - - std::vector< std::thread > threads; - for (uint32_t i{0}; i < unique_threads; ++i) { - threads.emplace_back( - std::move(std::thread(bind_this(CBMutexTest::thread_unique_fn, 1), num_iters / num_threads))); - } - - for (uint32_t i{0}; i < shared_threads; ++i) { - threads.emplace_back( - std::move(std::thread(bind_this(CBMutexTest::thread_shared_fn<>, 1), num_iters / num_threads))); - } - for (auto& t : threads) { - t.join(); - // delete (t); - } - } -}; - -using testing::Types; -typedef Types< std::mutex, std::shared_mutex, folly::SharedMutex > Implementations; -// typedef Types< std::mutex > Implementations; -// typedef Types< std::mutex, folly::SharedMutex > Implementations; -TYPED_TEST_SUITE(CBMutexTest, Implementations); - -TYPED_TEST(CBMutexTest, LockUnlockTest) { this->run_lock_unlock(); } - -SISL_OPTIONS_ENABLE(logging, test_cb_mutex) -SISL_OPTION_GROUP(test_cb_mutex, - (num_threads, "", "num_threads", "number of threads", - ::cxxopts::value< uint32_t >()->default_value("8"), "number"), - (num_iters, "", "num_iters", "number of iterations", - ::cxxopts::value< uint64_t >()->default_value("10000"), "number")) - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - SISL_OPTIONS_LOAD(argc, argv, logging, test_cb_mutex) - sisl::logging::SetLogger("test_cb_mutex"); - spdlog::set_pattern("[%D %T%z] [%^%L%$] [%t] %v"); - - auto ret = RUN_ALL_TESTS(); - return ret; -} diff --git a/src/fds/tests/test_obj_allocator.cpp b/src/fds/tests/test_obj_allocator.cpp deleted file mode 100644 index 54086aff..00000000 --- a/src/fds/tests/test_obj_allocator.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/********************************************************************************* - * Modifications Copyright 2017-2019 eBay Inc. - * - * Author/Developer(s): Harihara Kadayam - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - *********************************************************************************/ -#include -#include - -#include "sisl/logging/logging.h" -#include "sisl/options/options.h" - -#include "sisl/fds/obj_allocator.hpp" - -SISL_LOGGING_INIT(HOMESTORE_LOG_MODS) - -using namespace sisl; -using namespace std; - -namespace { -template < typename T > -class Node { -public: - Node(const T& id) { m_id = id; } - - const T& get_id() const { return m_id; } - - ~Node() { std::cout << "Destructor of Node " << m_id << " called\n"; } - -private: - T m_id; -}; -} // namespace - -int main() { - Node< uint64_t >* const ptr1{sisl::ObjectAllocator< Node< uint64_t > >::make_object(~static_cast< uint64_t >(0))}; - std::cout << "ptr1 = " << static_cast< const void* >(ptr1) << " Id = " << ptr1->get_id() << std::endl; - sisl::ObjectAllocator< Node< uint64_t > >::deallocate(ptr1); -} diff --git a/src/grpc/rpc_server.cpp b/src/grpc/rpc_server.cpp index 20575b21..64d27cf2 100644 --- a/src/grpc/rpc_server.cpp +++ b/src/grpc/rpc_server.cpp @@ -154,7 +154,7 @@ void GrpcServer::shutdown() { bool GrpcServer::is_auth_enabled() const { return m_auth_mgr != nullptr; } -grpc::Status GrpcServer::auth_verify(grpc::ServerContext const* srv_ctx) const { return m_auth_mgr->verify(srv_ctx); } +grpc::Status GrpcServer::auth_verify(grpc::ServerContext const* srv_ctx) const { return m_auth_mgr->verify_ctx(srv_ctx); } bool GrpcServer::run_generic_handler_cb(const std::string& rpc_name, boost::intrusive_ptr< GenericRpcData >& rpc_data) { generic_rpc_handler_cb_t cb; diff --git a/src/grpc/tests/unit/auth_test.cpp b/src/grpc/tests/unit/auth_test.cpp index 62e0acd6..d11b6e69 100644 --- a/src/grpc/tests/unit/auth_test.cpp +++ b/src/grpc/tests/unit/auth_test.cpp @@ -96,7 +96,7 @@ class EchoServiceImpl final { class MockTokenVerifier : public GrpcTokenVerifier { public: using GrpcTokenVerifier::GrpcTokenVerifier; - ::grpc::Status verify(::grpc::ServerContext const* srv_ctx) const override { + ::grpc::Status verify_ctx(::grpc::ServerContext const* srv_ctx) const override { auto& client_headers = srv_ctx->client_metadata(); if (auto it = client_headers.find(g_auth_header); it != client_headers.end()) { if (it->second == g_test_token) { return ::grpc::Status(); } diff --git a/src/logging/logging.cpp b/src/logging/logging.cpp index 635f8bf7..d3ebc052 100644 --- a/src/logging/logging.cpp +++ b/src/logging/logging.cpp @@ -296,7 +296,7 @@ static std::string setup_modules() { const std::string& mod_name{glob_enabled_mods[mod_num]}; fmt::vformat_to( std::back_inserter(out_str), fmt::string_view{"{}={}, "}, - fmt::make_format_args(mod_name, spdlog::level::to_string_view(GetModuleLogLevel(mod_name)).data())); + fmt::make_format_args(mod_name, spdlog::level::to_string_view(GetModuleLogLevel(mod_name)))); } return out_str;