From a4228b0359d9fa5c2ff77bde5864179847afc03c Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Tue, 27 Jun 2023 15:22:46 +0000 Subject: [PATCH 01/10] Cobrazz improvements --- CMakeLists.txt | 37 ++ clickhouse/CMakeLists.txt | 12 +- cmake/GrpcTargets.cmake | 16 +- cmake/Sanitizers.cmake | 1 + cmake/SetupEnvironment.cmake | 36 +- cmake/SetupGoogleProtoApis.cmake | 22 +- cmake/UServerAddLibrary.cmake | 64 +++ cmake/UServerInstall.cmake | 35 ++ cmake/userverConfig.cmake.in | 42 ++ core/CMakeLists.txt | 128 ++++- .../userver/components/run_embedded.hpp | 23 + core/src/components/run_embedded.cpp | 154 ++++++ core/src/dump/helpers.cpp | 2 +- core/src/fs/temp_file.cpp | 10 + core/src/utils/userver_experiment.cpp | 54 ++ core/src/utils/userver_experiment.hpp | 26 + grpc/CMakeLists.txt | 25 +- grpc/handlers/CMakeLists.txt | 24 +- grpc/src/ugrpc/impl/logging.cpp | 8 +- mongo/CMakeLists.txt | 11 +- packages/rpm/build.sh | 48 ++ packages/rpm/deps/build-amqp-cpp.sh | 109 ++++ packages/rpm/deps/build-c-ares.sh | 119 +++++ packages/rpm/deps/build-cctz.sh | 101 ++++ packages/rpm/deps/build-clickhouse-cpp.sh | 103 ++++ packages/rpm/deps/build-fmt.sh | 112 +++++ packages/rpm/deps/build-gtest.sh | 116 +++++ packages/rpm/deps/build-libcurl.sh | 472 ++++++++++++++++++ packages/rpm/deps/build-spdlog.sh | 121 +++++ .../deps/clickhouse-cpp/installCMake.cmake | 36 ++ packages/rpm/userver.spec | 131 +++++ postgresql/CMakeLists.txt | 9 +- postgresql/pq-extra/CMakeLists.txt | 2 +- rabbitmq/CMakeLists.txt | 8 +- redis/CMakeLists.txt | 11 +- scripts/grpc/requirements.txt | 2 +- testsuite/requirements-grpc.txt | 4 +- third_party/boost_stacktrace/CMakeLists.txt | 9 +- third_party/compiler-rt/CMakeLists.txt | 4 + .../CMakeLists.txt | 2 + tools/dns_resolver/CMakeLists.txt | 2 + tools/engine/CMakeLists.txt | 2 + tools/httpclient/CMakeLists.txt | 2 + tools/json2yaml/CMakeLists.txt | 2 + tools/netcat/CMakeLists.txt | 2 + uboost_coro/CMakeLists.txt | 7 +- universal/CMakeLists.txt | 11 +- 47 files changed, 2204 insertions(+), 73 deletions(-) create mode 100644 cmake/UServerAddLibrary.cmake create mode 100644 cmake/UServerInstall.cmake create mode 100644 cmake/userverConfig.cmake.in create mode 100644 core/include/userver/components/run_embedded.hpp create mode 100644 core/src/components/run_embedded.cpp create mode 100644 core/src/utils/userver_experiment.cpp create mode 100644 core/src/utils/userver_experiment.hpp create mode 100755 packages/rpm/build.sh create mode 100644 packages/rpm/deps/build-amqp-cpp.sh create mode 100644 packages/rpm/deps/build-c-ares.sh create mode 100644 packages/rpm/deps/build-cctz.sh create mode 100644 packages/rpm/deps/build-clickhouse-cpp.sh create mode 100644 packages/rpm/deps/build-fmt.sh create mode 100644 packages/rpm/deps/build-gtest.sh create mode 100644 packages/rpm/deps/build-libcurl.sh create mode 100644 packages/rpm/deps/build-spdlog.sh create mode 100644 packages/rpm/deps/clickhouse-cpp/installCMake.cmake create mode 100644 packages/rpm/userver.spec diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a2645676c18..6edbe90aa05d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,18 @@ endif() project(userver) +option(USERVER_GEN_GDB_DEBUGINFO "Compile with generation of gdb3 debug info format" OFF) + +if (USERVER_BUILD_SHARED_LIBS) + set(BUILD_SHARED_LIBS ${USERVER_BUILD_SHARED_LIBS}) +endif() + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/) +set(CMAKE_INSTALL_BINDIR bin/userver) + +#set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/userver) + set(USERVER_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}") set(USERVER_NOT_INCLUDED_AS_SUBDIR OFF) @@ -64,6 +76,8 @@ option(USERVER_DISABLE_PHDR_CACHE "Disable caching of dl_phdr_info items, which option(USERVER_CHECK_PACKAGE_VERSIONS "Check package versions" ON) +include(GNUInstallDirs) + include(cmake/SetupEnvironment.cmake) include(AddGoogleTests) include(CheckSubmodule) @@ -71,6 +85,8 @@ include(Sanitizers) include(FindPackageRequired) include(IncludeWhatYouUse) include(FindPython) +include(UServerAddLibrary) +include(UServerInstall) message(STATUS "Generating cmake files ...") execute_process( @@ -170,3 +186,24 @@ endif() if (USERVER_IS_THE_ROOT_PROJECT) add_subdirectory(samples) endif() + +set(CMAKE_INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/userver) +set(CMAKE_CONFIG_TARGETS_FILE "userverConfigTargets.cmake") +INSTALL (EXPORT userver-targets + FILE "${CMAKE_CONFIG_TARGETS_FILE}" DESTINATION ${CMAKE_INSTALL_CONFIGDIR}) +EXPORT (EXPORT userver-targets + FILE "${CMAKE_INSTALL_CONFIGDIR}/${CMAKE_CONFIG_TARGETS_FILE}") + +include(CMakePackageConfigHelpers) +set(CMAKE_VERSION_CONFIG_OUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/userverConfigVersion.cmake") +set(CMAKE_CONFIG_IN_FILE "${CMAKE_CURRENT_LIST_DIR}/cmake/userverConfig.cmake.in") +set(CMAKE_CONFIG_OUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/userverConfig.cmake") +set(CMAKE_CONFIG_TARGETS_FILE "userverConfigTargets.cmake") +configure_package_config_file( + "${CMAKE_CONFIG_IN_FILE}" + "${CMAKE_CONFIG_OUT_FILE}" + INSTALL_DESTINATION "${CMAKE_INSTALL_CONFIGDIR}/") +write_basic_package_version_file("${CMAKE_VERSION_CONFIG_OUT_FILE}" + VERSION "1.0.0" + COMPATIBILITY SameMajorVersion) +install(FILES "${CMAKE_CONFIG_OUT_FILE}" "${CMAKE_VERSION_CONFIG_OUT_FILE}" DESTINATION "${CMAKE_INSTALL_CONFIGDIR}") diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 6f2d61e6c7f6..071c9c413485 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -22,7 +22,7 @@ file(GLOB_RECURSE CH_FUNCTIONAL_TEST_SOURCES ) list(REMOVE_ITEM SOURCES ${CH_FUNCTIONAL_TEST_SOURCES}) -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) if (USERVER_CONAN) find_package(clickhouse-cpp REQUIRED) @@ -32,15 +32,19 @@ else() target_link_libraries(${PROJECT_NAME} PRIVATE clickhouse-cpp) endif() -target_link_libraries(${PROJECT_NAME} +userver_target_link_libraries(${PROJECT_NAME} PUBLIC userver-core PRIVATE + clickhouse-cpp + STATIC_PRIVATE userver-compiler-rt-parts ) target_include_directories( ${PROJECT_NAME} - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + PUBLIC + $ + $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) @@ -79,3 +83,5 @@ if (USERVER_IS_THE_ROOT_PROJECT) add_subdirectory(functional_tests) endif() + +userver_install_library(TARGETS ${PROJECT_NAME}) diff --git a/cmake/GrpcTargets.cmake b/cmake/GrpcTargets.cmake index b644d46fbfec..fd3ca3a01960 100644 --- a/cmake/GrpcTargets.cmake +++ b/cmake/GrpcTargets.cmake @@ -147,6 +147,9 @@ function(generate_grpc_files) set(did_generate_proto_sources FALSE) if("${newest_proto_dependency}" IS_NEWER_THAN "${GENERATED_PROTO_DIR}/${path_base}.pb.cc") + # resolve root_path, proto_file to real path's - protoc check that root_path is prefix of proto_file (this can be non true if project inside folder sym linked to other dir) + get_filename_component(real_root_path ${root_path} REALPATH) + get_filename_component(real_proto_file ${proto_file} REALPATH) execute_process( COMMAND mkdir -p proto COMMAND ${PROTOBUF_PROTOC} ${include_options} @@ -156,12 +159,12 @@ function(generate_grpc_files) --python_out=${GENERATED_PROTO_DIR} --grpc_python_out=${GENERATED_PROTO_DIR} ${pyi_out_param} - -I ${root_path} + -I ${real_root_path} -I ${USERVER_PROTOBUF_IMPORT_DIR} --plugin=protoc-gen-grpc=${PROTO_GRPC_CPP_PLUGIN} --plugin=protoc-gen-usrv=${PROTO_GRPC_USRV_PLUGIN} --plugin=protoc-gen-grpc_python=${PROTO_GRPC_PYTHON_PLUGIN} - ${proto_file} + ${real_proto_file} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} RESULT_VARIABLE execute_process_result ) @@ -229,9 +232,12 @@ function(add_grpc_library NAME) CPP_FILES generated_sources CPP_USRV_FILES generated_usrv_sources ) - add_library(${NAME} STATIC ${generated_sources} ${generated_usrv_sources}) + userver_add_library(${NAME} SOURCES ${generated_sources} ${generated_usrv_sources}) target_compile_options(${NAME} PUBLIC -Wno-unused-parameter) - target_include_directories(${NAME} SYSTEM PUBLIC ${include_paths}) - target_link_libraries(${NAME} PUBLIC userver-grpc) + target_include_directories(${NAME} SYSTEM PUBLIC + $ + ) + + target_link_libraries(${NAME} PUBLIC userver-grpc-internal) endfunction() diff --git a/cmake/Sanitizers.cmake b/cmake/Sanitizers.cmake index 3bed9d3cf078..316679a276a6 100644 --- a/cmake/Sanitizers.cmake +++ b/cmake/Sanitizers.cmake @@ -7,6 +7,7 @@ set(USERVER_SANITIZE_ENUM "mem, addr, thread, ub") set(USERVER_SANITIZE "" CACHE STRING "Sanitizer, possible values: ${USERVER_SANITIZE_ENUM}") add_library(sanitize-target INTERFACE) +install(TARGETS sanitize-target EXPORT userver-targets) # formal export for correct export dependent targets if (USERVER_SANITIZE) if (CLANG) diff --git a/cmake/SetupEnvironment.cmake b/cmake/SetupEnvironment.cmake index 70286a6d9b44..28ea02b65a28 100644 --- a/cmake/SetupEnvironment.cmake +++ b/cmake/SetupEnvironment.cmake @@ -19,7 +19,37 @@ set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_CXX_EXTENSIONS OFF) set (CMAKE_VISIBILITY_INLINES_HIDDEN ON) -add_compile_options ("-pipe" "-g" "-gz" "-fPIC") +function(remove_option OUT_LIST_NAME IN_LIST_NAME REMOVE_ITEM) + string(REPLACE " " ";" LOCAL_LIST "${IN_LIST_NAME}") + list (REMOVE_ITEM LOCAL_LIST "${REMOVE_ITEM}") + string(REPLACE ";" " " LOCAL_LIST "${LOCAL_LIST}") + set(${OUT_LIST_NAME} "${LOCAL_LIST}" PARENT_SCOPE) +endfunction() + +# remove default -g debug option (we will add it if required) +remove_option(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}" "-g") +set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}" CACHE STRING "Flags used by the C compiler during DEBUG builds" FORCE) +remove_option(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" "-g") +set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING "Flags used by the CXX compiler during DEBUG builds" FORCE) + +remove_option(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}" "-g") +set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}" CACHE STRING "Flags used by the C compiler during RELWITHDEBINFO builds" FORCE) +remove_option(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" "-g") +set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" CACHE STRING "Flags used by the CXX compiler during RELEASE builds" FORCE) + +if (USERVER_GEN_GDB_DEBUGINFO) + set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -ggdb3") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -ggdb3") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3") +else () + set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -g -gz") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -gz") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -g -gz") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -gz") +endif() + +add_compile_options ("-pipe" "-fPIC") add_definitions ("-DPIC") add_definitions(-DUSERVER) @@ -140,4 +170,8 @@ else () add_definitions(-D_FORTIFY_SOURCE=2) endif () +if (UNIX AND BUILD_SHARED_LIBS) + link_libraries(dl) +endif () + enable_testing () diff --git a/cmake/SetupGoogleProtoApis.cmake b/cmake/SetupGoogleProtoApis.cmake index 45b1cca8e87f..a252388b1e4b 100644 --- a/cmake/SetupGoogleProtoApis.cmake +++ b/cmake/SetupGoogleProtoApis.cmake @@ -49,10 +49,28 @@ generate_grpc_files( CPP_USRV_FILES generated_usrv_sources ) -add_library(userver-api-common-protos STATIC ${generated_sources}) +userver_add_library(userver-api-common-protos SOURCES ${generated_sources}) + +set_target_properties(userver-api-common-protos PROPERTIES LINKER_LANGUAGE CXX) + target_compile_options(userver-api-common-protos PUBLIC -Wno-unused-parameter) -target_include_directories(userver-api-common-protos SYSTEM PUBLIC ${include_paths}) +target_include_directories(userver-api-common-protos SYSTEM PUBLIC + $ + $) + target_link_libraries(userver-api-common-protos PUBLIC userver-core userver-grpc-deps) set(api-common-proto_LIBRARY userver-api-common-protos) set(api-common-proto_USRV_SOURCES ${generated_usrv_sources}) + +# workaround for INSTALL dependent libraries without install this static library +set_target_properties(userver-api-common-protos PROPERTIES LINKER_LANGUAGE CXX) + +install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/proto" + DESTINATION "include/userver/grpc/" + FILES_MATCHING + PATTERN "*.pb.hpp" + PATTERN "*.pb.h" + ) + +userver_export(TARGETS userver-api-common-protos) diff --git a/cmake/UServerAddLibrary.cmake b/cmake/UServerAddLibrary.cmake new file mode 100644 index 000000000000..be05bab41d74 --- /dev/null +++ b/cmake/UServerAddLibrary.cmake @@ -0,0 +1,64 @@ +function(userver_add_library NAME) + + set(options) + set(one_value_args) + set(multi_value_args + SOURCES + PUBLIC_LINK_LIBRARIES + PRIVATE_LINK_LIBRARIES + ) + cmake_parse_arguments(USERVER_ADD_LIBRARY "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + + if (BUILD_SHARED_LIBS) + add_library(${NAME} SHARED ${USERVER_ADD_LIBRARY_SOURCES}) + + set_property(TARGET "${NAME}" PROPERTY POSITION_INDEPENDENT_CODE 1) + + target_link_libraries(${NAME} + PUBLIC + ${USERVER_ADD_LIBRARY_PUBLIC_LINK_LIBRARIES} + PRIVATE + ${USERVER_ADD_LIBRARY_PRIVATE_LINK_LIBRARIES} + ) + else() + add_library(${NAME} STATIC ${USERVER_ADD_LIBRARY_SOURCES}) + + target_link_libraries(${NAME} + PUBLIC + ${USERVER_ADD_LIBRARY_PUBLIC_LINK_LIBRARIES} + PRIVATE + ${USERVER_ADD_LIBRARY_PRIVATE_LINK_LIBRARIES} + ) + endif() + +endfunction() + +function(userver_target_link_libraries NAME) + set(options) + set(one_value_args) + set(multi_value_args + PUBLIC + PRIVATE + STATIC_PRIVATE + ) + cmake_parse_arguments(USERVER_ADD_LIBRARY "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + + if(BUILD_SHARED_LIBS) + target_link_libraries(${NAME} + PUBLIC + ${USERVER_ADD_LIBRARY_PUBLIC} + ${USERVER_ADD_LIBRARY_PRIVATE} + PRIVATE + ${USERVER_ADD_LIBRARY_STATIC_PRIVATE} + ) + else() + target_link_libraries(${NAME} + PUBLIC + ${USERVER_ADD_LIBRARY_PUBLIC} + PRIVATE + ${USERVER_ADD_LIBRARY_PRIVATE} + ${USERVER_ADD_LIBRARY_STATIC_PRIVATE} + ) + endif() + +endfunction() diff --git a/cmake/UServerInstall.cmake b/cmake/UServerInstall.cmake new file mode 100644 index 000000000000..6286bb23fed0 --- /dev/null +++ b/cmake/UServerInstall.cmake @@ -0,0 +1,35 @@ +function(userver_install_library) + + set(options) + set(one_value_args) + set(multi_value_args TARGETS INCLUDE_DIR) + cmake_parse_arguments(USERVER_INSTALL "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + + install(TARGETS ${USERVER_INSTALL_TARGETS} EXPORT userver-targets + PUBLIC_HEADER DESTINATION include) + + if (NOT DEFINED USERVER_INSTALL_INCLUDE_DIR) + set(USERVER_INSTALL_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/") + endif() + + install(DIRECTORY "${USERVER_INSTALL_INCLUDE_DIR}/" + DESTINATION "include" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.ipp" + PATTERN "*.h" + PATTERN "*.inc" + ) + +endfunction() + +function(userver_export) + set(options) + set(one_value_args) + set(multi_value_args TARGETS) + cmake_parse_arguments(USERVER_EXPORT "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + + install(TARGETS ${USERVER_EXPORT_TARGETS} EXPORT userver-targets + PUBLIC_HEADER DESTINATION include) + +endfunction() diff --git a/cmake/userverConfig.cmake.in b/cmake/userverConfig.cmake.in new file mode 100644 index 000000000000..57b21fa3ff7d --- /dev/null +++ b/cmake/userverConfig.cmake.in @@ -0,0 +1,42 @@ +@PACKAGE_INIT@ + +find_package(PkgConfig REQUIRED) + +find_package(Threads REQUIRED) +find_package(Boost COMPONENTS filesystem program_options iostreams regex locale coroutine REQUIRED) +find_package(CURL REQUIRED) +find_package(OpenSSL REQUIRED) +find_package(Iconv REQUIRED) +find_package(ZLIB REQUIRED) + +find_package(yaml-cpp REQUIRED) +add_library(libyamlcpp INTERFACE) +target_link_libraries(libyamlcpp INTERFACE ${YAML_CPP_LIBRARIES}) + +pkg_check_modules (JEMALLOC jemalloc) +pkg_search_module(JEMALLOC REQUIRED jemalloc) +add_library(Jemalloc INTERFACE) +target_link_libraries(Jemalloc INTERFACE ${JEMALLOC_LIBRARIES}) + +pkg_check_modules (LIBCARES libcares) +pkg_search_module(LIBCARES REQUIRED libcares) +add_library(c-ares INTERFACE) +target_link_libraries(c-ares INTERFACE ${LIBCARES_LIBRARIES}) + +pkg_check_modules (LIBCRYPTOPP cryptopp) +pkg_search_module(LIBCRYPTOPP REQUIRED cryptopp) +add_library(CryptoPP INTERFACE) +target_link_libraries(CryptoPP INTERFACE ${LIBCRYPTOPP_LIBRARIES}) + +pkg_check_modules (LIBEV libevent) +pkg_search_module(LIBEV REQUIRED libevent) +add_library(LibEv INTERFACE) +target_link_libraries(LibEv INTERFACE ${LIBEV_LIBRARIES}) + +add_library(Http_Parser INTERFACE) +target_link_libraries(Http_Parser INTERFACE http_parser) + +set(CMAKE_CONFIG_TARGETS_FILE @CMAKE_CONFIG_TARGETS_FILE@) +include("${CMAKE_CURRENT_LIST_DIR}/${CMAKE_CONFIG_TARGETS_FILE}") + +check_required_components(userver) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 8e188d10531e..148b2606b9ff 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -116,7 +116,7 @@ else() find_package(LibEv REQUIRED) endif() -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) target_compile_definitions(${PROJECT_NAME} PRIVATE @@ -150,7 +150,7 @@ endif() option(USERVER_FEATURE_UBOOST_CORO "Use vendored boost context instead of a system one" ON) -target_link_libraries(${PROJECT_NAME} +userver_target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads Boost::locale @@ -173,7 +173,15 @@ if (USERVER_FEATURE_UBOOST_CORO) userver-uboost-coro ) target_include_directories(${PROJECT_NAME} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/uboost_coro/include + $ + $ + ) + + install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/uboost_coro/include/coroutines" + DESTINATION "include/userver/" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.h" ) else() target_link_libraries(${PROJECT_NAME} @@ -181,12 +189,20 @@ else() Boost::context ) target_include_directories(${PROJECT_NAME} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/sys_coro/include + $ + $ + ) + + install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/sys_coro/include/coroutines" + DESTINATION "include/userver/" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.h" ) endif() if (USERVER_CONAN) - target_link_libraries(${PROJECT_NAME} + userver_target_link_libraries(${PROJECT_NAME} PUBLIC yaml-cpp fmt::fmt @@ -203,7 +219,7 @@ if (USERVER_CONAN) rapidjson ) else() - target_link_libraries(${PROJECT_NAME} + userver_target_link_libraries(${PROJECT_NAME} PUBLIC libyamlcpp fmt @@ -222,12 +238,15 @@ else() ${USERVER_THIRD_PARTY_DIRS}/rapidjson/include ) target_include_directories(${PROJECT_NAME} SYSTEM PUBLIC - ${USERVER_THIRD_PARTY_DIRS}/moodycamel/include + $ + $ + $ ) endif() target_include_directories(${PROJECT_NAME} PUBLIC - ${USERVER_THIRD_PARTY_DIRS}/date/include + $ + $ ) if (NOT MACOS AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "BSD") @@ -235,17 +254,21 @@ if (NOT MACOS AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "BSD") endif() set(USERVER_PUBLIC_INCLUDE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${CMAKE_CURRENT_SOURCE_DIR}/../shared/include - ${USERVER_THIRD_PARTY_DIRS}/pfr/include + $ + $ + $ + $ + $ ) set(USERVER_GLIBC_INCLUDE_FIXES_DIR - ${CMAKE_CURRENT_SOURCE_DIR}/libc_include_fixes + $ + $ ) target_include_directories(${PROJECT_NAME} PUBLIC - ${USERVER_PUBLIC_INCLUDE_DIRS} -) + $ + ) + target_include_directories(${PROJECT_NAME} SYSTEM BEFORE PUBLIC ${USERVER_GLIBC_INCLUDE_FIXES_DIR} ) @@ -256,6 +279,63 @@ set_property( APPEND PROPERTY COMPILE_FLAGS -O2 ) +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../shared/include/userver" + DESTINATION "include/" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.h" +) + +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/utils/signal_catcher.hpp" + DESTINATION "include/userver/utils/") + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/components/manager_config.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/components/manager.hpp" + DESTINATION "include/userver/components/") + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/coro/pool_config.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/coro/pool_stats.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/coro/pool.hpp" + DESTINATION "include/userver/engine/coro/") + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/ev/async_payload_base.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/ev/thread_pool_config.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/ev/thread_pool.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/ev/thread_control.hpp" + DESTINATION "include/userver/engine/ev/") + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/concurrent/impl/intrusive_hooks.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/concurrent/impl/interference_shield.hpp" + DESTINATION "include/userver/concurrent/impl/") + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/task_processor_config.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/task_processor.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/task_processor_pools.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/counted_coroutine_ptr.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/task_counter.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/counted_coroutine_ptr.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/task_queue.hpp" + DESTINATION "include/userver/engine/task/") + +install(DIRECTORY "${USERVER_THIRD_PARTY_DIRS}/moodycamel/include/moodycamel" + DESTINATION "include/userver" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.h" +) + +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libc_include_fixes" + DESTINATION "include/userver" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.h" +) + target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../shared/src/ @@ -269,9 +349,9 @@ file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/" SRC_LOG_PATH_BASE) file(TO_NATIVE_PATH "${CMAKE_BINARY_DIR}/" BIN_LOG_PATH_BASE) target_compile_definitions(${PROJECT_NAME} PUBLIC - USERVER_LOG_SOURCE_PATH_BASE=${SRC_LOG_PATH_BASE} - USERVER_LOG_BUILD_PATH_BASE=${BIN_LOG_PATH_BASE} - USERVER_LOG_PREFIX_PATH_BASE=${BASE_PREFIX} + USERVER_LOG_SOURCE_PATH_BASE=$ + USERVER_LOG_BUILD_PATH_BASE=$ + USERVER_LOG_PREFIX_PATH_BASE=$ ) message(STATUS "Putting userver into namespace '${USERVER_NAMESPACE}': ${USERVER_NAMESPACE_BEGIN} ${USERVER_NAMESPACE_END}") @@ -284,7 +364,7 @@ target_compile_definitions(${PROJECT_NAME} PUBLIC # Suppress OpenSSL 3 warnings: we still primarily support OpenSSL 1.1.x target_compile_definitions(${PROJECT_NAME} PRIVATE OPENSSL_SUPPRESS_DEPRECATED=) -add_library(userver-core-internal STATIC ${INTERNAL_SOURCES}) +userver_add_library(userver-core-internal SOURCES ${INTERNAL_SOURCES}) target_compile_definitions(userver-core-internal PUBLIC $) target_include_directories(userver-core-internal PUBLIC $ @@ -297,10 +377,10 @@ target_link_libraries(userver-core-internal ) if (USERVER_FEATURE_UTEST) - add_library(userver-utest STATIC ${LIBUTEST_SOURCES}) + userver_add_library(userver-utest SOURCES ${LIBUTEST_SOURCES}) target_compile_definitions(userver-utest PUBLIC $) - target_link_libraries(userver-utest + userver_target_link_libraries(userver-utest PUBLIC ${PROJECT_NAME} PRIVATE @@ -343,10 +423,10 @@ if (USERVER_IS_THE_ROOT_PROJECT) endif() if (USERVER_FEATURE_UTEST) - add_library(userver-ubench ${LIBUBENCH_SOURCES}) + userver_add_library(userver-ubench SOURCES ${LIBUBENCH_SOURCES}) target_include_directories(userver-ubench PUBLIC $) target_compile_definitions(userver-ubench PUBLIC $) - target_link_libraries(userver-ubench + userver_target_link_libraries(userver-ubench PUBLIC ${PROJECT_NAME} PRIVATE @@ -363,7 +443,7 @@ if (USERVER_IS_THE_ROOT_PROJECT) add_executable(${PROJECT_NAME}_benchmark ${BENCH_SOURCES}) # find_package(absl REQUIRED) - target_link_libraries(${PROJECT_NAME}_benchmark + userver_target_link_libraries(${PROJECT_NAME}_benchmark PUBLIC userver-ubench PRIVATE @@ -372,3 +452,5 @@ if (USERVER_IS_THE_ROOT_PROJECT) ) add_google_benchmark_tests(${PROJECT_NAME}_benchmark) endif() + +userver_install_library(TARGETS ${PROJECT_NAME}) diff --git a/core/include/userver/components/run_embedded.hpp b/core/include/userver/components/run_embedded.hpp new file mode 100644 index 000000000000..1ed868555db3 --- /dev/null +++ b/core/include/userver/components/run_embedded.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace components { + +/// Starts a server with the provided component list and config. +/// Ropens the logging files on SIGUSR1. +/// +/// @see utils::DaemonMain +void RunEmbedded(std::mutex& stopper_mutex, + const InMemoryConfig& config, + const ComponentList& component_list, + const std::string& init_log_path = {}, + logging::Format format = logging::Format::kTskv); + +} // namespace components + +USERVER_NAMESPACE_END diff --git a/core/src/components/run_embedded.cpp b/core/src/components/run_embedded.cpp new file mode 100644 index 000000000000..5ac7dadc349b --- /dev/null +++ b/core/src/components/run_embedded.cpp @@ -0,0 +1,154 @@ +#include + +#include + +#include +#include +#include + +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace server::handlers::auth::apikey { +extern int auth_checker_apikey_module_activation; +} // namespace server::handlers::auth::apikey + +namespace components { + +namespace { + +void HandleJemallocSettings() { + static constexpr size_t kDefaultMaxBgThreads = 1; + + auto ec = utils::jemalloc::SetMaxBgThreads(kDefaultMaxBgThreads); + if (ec) { + LOG_WARNING() << "Failed to set max_background_threads to " + << kDefaultMaxBgThreads; + } + + if (utils::IsUserverExperimentEnabled( + utils::UserverExperiment::kJemallocBgThread)) { + utils::jemalloc::EnableBgThreads(); + } +} + +void PreheatStacktraceCollector() { + // If DEBUG logging is enabled the following line loads debug info from disk, + // hopefully preventing this to occur later, e.g. in exception constructor. + LOG_DEBUG() << utils::TracefulException{"Preheating stacktrace"}; +} + +bool IsTraced() { + static const std::string kTracerField = "TracerPid:\t"; + + try { + // /proc is only available on linux, + // on macos this will always throw and return false. + const std::string proc_status = + fs::blocking::ReadFileContents("/proc/self/status"); + auto tracer_pos = proc_status.find(kTracerField); + if (tracer_pos != std::string::npos) { + return proc_status[tracer_pos + kTracerField.size()] != '0'; + } + } catch (const std::exception&) { + // ignore + } + return false; +} + +enum class RunMode { kNormal, kOnce }; + +using PathOrConfig = std::variant; + +struct ConfigToManagerVisitor { + const std::optional& config_vars_path; + const std::optional& config_vars_override_path; + + ManagerConfig operator()(const std::string& path) const { + return ManagerConfig::FromFile(path, config_vars_path, + config_vars_override_path); + } + + ManagerConfig operator()(const components::InMemoryConfig& config) const { + return ManagerConfig::FromString(config.GetUnderlying(), config_vars_path, + config_vars_override_path); + } +}; + +void DoRun(std::mutex& stop_mutex, + const PathOrConfig& config, + const std::optional& config_vars_path, + const std::optional& config_vars_override_path, + const ComponentList& component_list, + const std::string& init_log_path, logging::Format format, + RunMode run_mode) { + + ++server::handlers::auth::apikey::auth_checker_apikey_module_activation; + crypto::impl::Openssl::Init(); + + //LogScope log_scope{init_log_path, format}; + logging::DefaultLoggerGuard log_scope(logging::MakeFileLogger("default", init_log_path, format)); + + LOG_INFO() << "Parsing configs"; + if (config_vars_path) { + LOG_INFO() << "Using config_vars from cmdline: " << *config_vars_path + << ". The config_vars filepath in config.yaml is ignored."; + } else { + LOG_INFO() << "Using config_vars from config.yaml."; + } + if (config_vars_override_path) { + LOG_INFO() << "Overriding config_vars with values from file: " + << *config_vars_override_path; + } + auto parsed_config = std::make_unique(std::visit( + ConfigToManagerVisitor{config_vars_path, config_vars_override_path}, + config)); + LOG_INFO() << "Parsed configs"; + + HandleJemallocSettings(); + PreheatStacktraceCollector(); + + std::optional manager; + try { + manager.emplace(std::move(parsed_config), component_list); + } catch (const std::exception& ex) { + LOG_ERROR() << "Loading failed: " << ex; + throw; + } + + stop_mutex.lock(); +} + +} // namespace + +void RunEmbedded(std::mutex& stop_mutex, + const InMemoryConfig& config, + const ComponentList& component_list, + const std::string& init_log_path, logging::Format format) { + DoRun(stop_mutex, config, {}, {}, component_list, init_log_path, format, + RunMode::kNormal); +} + +} // namespace components + +USERVER_NAMESPACE_END diff --git a/core/src/dump/helpers.cpp b/core/src/dump/helpers.cpp index 7e4af6ce4c28..fed6ea1f9515 100644 --- a/core/src/dump/helpers.cpp +++ b/core/src/dump/helpers.cpp @@ -8,7 +8,7 @@ USERVER_NAMESPACE_BEGIN namespace dump { -void ThrowDumpUnimplemented(const std::string& name) { +void _ThrowDumpUnimplemented(const std::string& name) { UINVARIANT(false, fmt::format("Dumps are unimplemented for {}. " "See dump::Read, dump::Write", name)); diff --git a/core/src/fs/temp_file.cpp b/core/src/fs/temp_file.cpp index 204d86976edc..751c94da977d 100644 --- a/core/src/fs/temp_file.cpp +++ b/core/src/fs/temp_file.cpp @@ -6,6 +6,16 @@ USERVER_NAMESPACE_BEGIN namespace fs { +// Please, review this TempFile::TempFile implementation +// I simple added it as workaround for linkage error (if linked as shared library) +// now project don't use TempFile +TempFile::TempFile(userver::engine::TaskProcessor& fs_task_processor, + userver::fs::blocking::TempFile temp_file) + : fs_task_processor_(fs_task_processor), + temp_file_(std::move(temp_file)) +{ +} + TempFile TempFile::Create(engine::TaskProcessor& fs_task_processor) { return { fs_task_processor, diff --git a/core/src/utils/userver_experiment.cpp b/core/src/utils/userver_experiment.cpp new file mode 100644 index 000000000000..13c94cbd3179 --- /dev/null +++ b/core/src/utils/userver_experiment.cpp @@ -0,0 +1,54 @@ +#include + +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace utils { +namespace { + +auto& GetUserverExperimentFlags() { + static std::array(UserverExperiment::kCount)> + experiments{}; + return experiments; +} + +} // namespace + +bool IsUserverExperimentEnabled(UserverExperiment exp) { + return GetUserverExperimentFlags()[static_cast(exp)]; +} + +void EnableUserverExperiment(UserverExperiment exp) { + GetUserverExperimentFlags()[static_cast(exp)] = true; +} + +void DisableUserverExperiment(UserverExperiment exp) { + GetUserverExperimentFlags()[static_cast(exp)] = false; +} + +void ParseUserverExperiments(const formats::yaml::Value& yaml) { + if (yaml.IsMissing()) return; + + static constexpr utils::TrivialBiMap kExperiments = [](auto selector) { + return selector().Case("jemalloc-bg-thread", + UserverExperiment::kJemallocBgThread); + }; + + for (const auto& exp : yaml) { + auto value = exp.As(); + auto exp_enum = kExperiments.TryFind(value); + if (exp_enum) { + LOG_WARNING() << "Enabling userver experiment " << value; + EnableUserverExperiment(*exp_enum); + } + } +} + +} // namespace utils + +USERVER_NAMESPACE_END diff --git a/core/src/utils/userver_experiment.hpp b/core/src/utils/userver_experiment.hpp new file mode 100644 index 000000000000..500c617b3b01 --- /dev/null +++ b/core/src/utils/userver_experiment.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace utils { + +enum class UserverExperiment { + kJemallocBgThread, + + kCount, +}; + +bool IsUserverExperimentEnabled(UserverExperiment); + +void EnableUserverExperiment(UserverExperiment); +void DisableUserverExperiment(UserverExperiment); + +void ParseUserverExperiments(const formats::yaml::Value& yaml); + +} // namespace utils + +USERVER_NAMESPACE_END diff --git a/grpc/CMakeLists.txt b/grpc/CMakeLists.txt index f8a6c685bb63..c19b25699492 100644 --- a/grpc/CMakeLists.txt +++ b/grpc/CMakeLists.txt @@ -3,7 +3,9 @@ project(userver-grpc CXX) include(GrpcTargets) add_library(${PROJECT_NAME}-deps INTERFACE) -target_include_directories(${PROJECT_NAME}-deps INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_include_directories(${PROJECT_NAME}-deps INTERFACE + $ + $) target_link_libraries(${PROJECT_NAME}-deps INTERFACE protobuf::libprotobuf gRPC::grpc++ ) @@ -14,7 +16,6 @@ if(USERVER_CONAN AND "${gRPC_VERSION}" VERSION_GREATER_EQUAL "1.41") ) endif() - include(SetupGoogleProtoApis) file(GLOB_RECURSE SOURCES @@ -43,13 +44,14 @@ if (api-common-proto_USRV_SOURCES) list(APPEND SOURCES ${api-common-proto_USRV_SOURCES}) endif() -add_library(${PROJECT_NAME}-internal STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME}-internal SOURCES ${SOURCES}) set_target_properties(${PROJECT_NAME}-internal PROPERTIES LINKER_LANGUAGE CXX) target_include_directories(${PROJECT_NAME}-internal PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include + $ + $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) @@ -97,7 +99,7 @@ if (USERVER_IS_THE_ROOT_PROJECT) ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src ) - target_link_libraries(${PROJECT_NAME}_unittest + userver_target_link_libraries(${PROJECT_NAME}_unittest PUBLIC ${PROJECT_NAME}-internal userver-utest @@ -116,3 +118,16 @@ add_subdirectory(handlers) add_library(${PROJECT_NAME} INTERFACE) target_link_libraries(${PROJECT_NAME} INTERFACE ${PROJECT_NAME}-handlers) target_link_libraries(${PROJECT_NAME} INTERFACE ${PROJECT_NAME}-internal) + +userver_export(TARGETS ${PROJECT_NAME}-deps) + +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/ugrpc/client/impl/client_configs.hpp" + DESTINATION "include/userver/ugrpc/client/impl/") + +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/ugrpc/server/impl/server_configs.hpp" + DESTINATION "include/userver/ugrpc/server/impl/") + +userver_install_library( + TARGETS + ${PROJECT_NAME} + ${PROJECT_NAME}-internal) diff --git a/grpc/handlers/CMakeLists.txt b/grpc/handlers/CMakeLists.txt index 4d234cce1ff9..6d2a23b22789 100644 --- a/grpc/handlers/CMakeLists.txt +++ b/grpc/handlers/CMakeLists.txt @@ -9,13 +9,14 @@ file(GLOB_RECURSE UNIT_TEST_SOURCES ) list(REMOVE_ITEM SOURCES ${UNIT_TEST_SOURCES}) -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) target_include_directories(${PROJECT_NAME} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include + $ + $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) @@ -24,6 +25,16 @@ target_link_libraries(${PROJECT_NAME} PUBLIC userver-core) target_link_libraries(${PROJECT_NAME} PUBLIC userver-grpc-internal) add_grpc_library(${PROJECT_NAME}_proto PROTOS healthchecking/healthchecking.proto) + +target_include_directories(${PROJECT_NAME} + PUBLIC + $ + ) + +set_target_properties(${PROJECT_NAME}_proto PROPERTIES LINKER_LANGUAGE CXX) + +userver_export(TARGETS ${PROJECT_NAME}_proto) + target_link_libraries(${PROJECT_NAME} PUBLIC ${PROJECT_NAME}_proto) if (USERVER_IS_THE_ROOT_PROJECT) @@ -45,3 +56,12 @@ if (USERVER_IS_THE_ROOT_PROJECT) add_subdirectory(functional_tests) endif() + +install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/proto" + DESTINATION "include/userver/grpc/handlers/" + FILES_MATCHING + PATTERN "*.pb.hpp" + PATTERN "*.pb.h" + ) + +userver_export(TARGETS ${PROJECT_NAME}) diff --git a/grpc/src/ugrpc/impl/logging.cpp b/grpc/src/ugrpc/impl/logging.cpp index b19424bc2bdc..d1939f803a06 100644 --- a/grpc/src/ugrpc/impl/logging.cpp +++ b/grpc/src/ugrpc/impl/logging.cpp @@ -64,16 +64,16 @@ auto native_log_level = logging::Level::kNone; } // namespace -void SetupNativeLogging() { ::gpr_set_log_function(&LogFunction); } +void SetupNativeLogging() { /*::gpr_set_log_function(&LogFunction);*/ } -void UpdateNativeLogLevel(logging::Level min_log_level_override) { - std::lock_guard lock(native_log_level_mutex); +void UpdateNativeLogLevel(logging::Level /*min_log_level_override*/) { + /*std::lock_guard lock(native_log_level_mutex); if (utils::UnderlyingValue(min_log_level_override) < utils::UnderlyingValue(native_log_level)) { ::gpr_set_log_verbosity(ToGprLogSeverity(min_log_level_override)); native_log_level = min_log_level_override; - } + }*/ } } // namespace ugrpc::impl diff --git a/mongo/CMakeLists.txt b/mongo/CMakeLists.txt index c1a7c7db9822..acb204404ff7 100644 --- a/mongo/CMakeLists.txt +++ b/mongo/CMakeLists.txt @@ -37,9 +37,9 @@ file(GLOB_RECURSE BENCH_SOURCES ) list (REMOVE_ITEM SOURCES ${BENCH_SOURCES}) -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) -target_link_libraries(${PROJECT_NAME} +userver_target_link_libraries(${PROJECT_NAME} PUBLIC userver-core bson @@ -49,9 +49,10 @@ target_link_libraries(${PROJECT_NAME} target_include_directories( ${PROJECT_NAME} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include + $ + $ PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_include_directories (${PROJECT_NAME} SYSTEM PRIVATE $ @@ -99,3 +100,5 @@ if (USERVER_IS_THE_ROOT_PROJECT) add_subdirectory(functional_tests) endif() + +userver_install_library(TARGETS ${PROJECT_NAME}) diff --git a/packages/rpm/build.sh b/packages/rpm/build.sh new file mode 100755 index 000000000000..b9e6720e5904 --- /dev/null +++ b/packages/rpm/build.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +VERSION=$1 +VERSIONSSV=ssv1 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build \ + epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +SPEC_FILE=$(rpm -E %_specdir)/userver-$VERSION-$VERSIONSSV.spec + +# download spec file +#curl 'https://raw.githubusercontent.com/yoori/userver/1.0.0/packages/rpm/userver.spec' >"$SPEC_FILE" +cp $SCRIPTPATH/userver.spec "$SPEC_FILE" + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +export PATH=/opt/rh/gcc-toolset-11/root/usr/bin/:$PATH + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +#cp $BIN_RPM_FOLDER/spdlog*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-amqp-cpp.sh b/packages/rpm/deps/build-amqp-cpp.sh new file mode 100644 index 000000000000..fa94d5a045f9 --- /dev/null +++ b/packages/rpm/deps/build-amqp-cpp.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +VERSION=$1 +VERSIONSSV=ssv2 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +SPEC_FILE=$(rpm -E %_specdir)/amqp-cpp-$VERSION-$VERSIONSSV.spec + +cat << 'EOF' > $SPEC_FILE +Name: amqp-cpp +Version: %{_version} +Release: ssv1%{?dist} +Summary: C++ library for communicating with a RabbitMQ message broker +Group: Development/Libraries/C and C++ +License: Apache License 2.0 +URL: https://github.com/CopernicaMarketingSoftware/AMQP-CPP +Source0: https://github.com/CopernicaMarketingSoftware/AMQP-CPP/archive/refs/tags/v%{version}.tar.gz +BuildRequires: cmake +BuildRequires: gcc +BuildRequires: gcc-c++ + +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{_version}-%{release}-XXXXXX) + +%description + +AMQP-CPP is a C++ library for communicating with a RabbitMQ message broker. +The library can be used to parse incoming data from, and generate frames to, a RabbitMQ server. + +%package -n %{name}-devel +Summary: C++ library for communicating with a RabbitMQ message broker +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +%description -n %{name}-devel + +AMQP-CPP is a C++ library for communicating with a RabbitMQ message broker. +The library can be used to parse incoming data from, and generate frames to, a RabbitMQ server. + +%prep +%setup -q -n AMQP-CPP-%{_version} + +%build +%cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX:PATH=/usr \ + -DAMQP-CPP_BUILD_SHARED:BOOL=ON \ + -DAMQP-CPP_LINUX_TCP:BOOL=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + +make -j + +%install +DESTDIR=%{buildroot}/ make install INSTALL_PATH=%{buildroot}/ +mv %{buildroot}/usr/lib %{buildroot}/usr/lib64 +mkdir -p %{buildroot}/usr/lib64/cmake +mv %{buildroot}/usr/cmake %{buildroot}/usr/lib64/cmake/amqpcpp +sed -i -r 's|get_filename_component\(_IMPORT_PREFIX "\$\{CMAKE_CURRENT_LIST_FILE\}" PATH\)|set(_IMPORT_PREFIX "/usr/x")|' \ + %{buildroot}/usr/lib64/cmake/amqpcpp/amqpcppConfig.cmake +sed -i -r 's|\{_IMPORT_PREFIX\}/lib/|{_IMPORT_PREFIX}/lib64/|' \ + %{buildroot}/usr/lib64/cmake/amqpcpp/amqpcppConfig-relwithdebinfo.cmake + +%clean +rm -rf %{buildroot} + +%files -n %{name} +%license LICENSE +%doc README.md + +%{_libdir}/libamqpcpp.so.* + +%files -n %{name}-devel + +%{_includedir}/ +%{_libdir}/pkgconfig +%{_libdir}/cmake/amqpcpp +%{_libdir}/libamqpcpp.so + +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/amqp-cpp-*$VERSION-*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-c-ares.sh b/packages/rpm/deps/build-c-ares.sh new file mode 100644 index 000000000000..ba998fa195f5 --- /dev/null +++ b/packages/rpm/deps/build-c-ares.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +VERSION=1.18.1 +VERSIONSSV=ssv1 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +sudo yum -y install libzstd-devel zlib-devel || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +# download source RPM +curl 'http://c-ares.haxx.se/license.html' >$(rpm -E '%_sourcedir')/LICENSE + +SPEC_FILE=$(rpm -E %_specdir)/c-ares-$VERSION-$VERSIONSSV.spec + +cat << 'EOF' > $SPEC_FILE +Summary: A library that performs asynchronous DNS operations +Name: c-ares +Version: %{_version} +Release: 4%{?dist} +License: MIT +Group: System Environment/Libraries +URL: http://c-ares.haxx.se/ +Source0: http://c-ares.haxx.se/download/%{name}-%{version}.tar.gz +# The license can be obtained at http://c-ares.haxx.se/license.html +Source1: LICENSE +#Patch0: 0001-Use-RPM-compiler-options.patch +#Patch1: c-ares-1.10.0-multilib.patch + +BuildRequires: autoconf +BuildRequires: automake +BuildRequires: libtool + +%description +c-ares is a C library that performs DNS requests and name resolves +asynchronously. c-ares is a fork of the library named 'ares', written +by Greg Hudson at MIT. + +%package devel +Summary: Development files for c-ares +Group: Development/Libraries +Requires: %{name} = %{version}-%{release} +Requires: pkgconfig + +%description devel +This package contains the header files and libraries needed to +compile applications or shared objects that use c-ares. + +%prep +%setup -q +#%patch0 -p1 -b .optflags +#%patch1 -p1 -b .multilib + +cp %{SOURCE1} . +f=CHANGES ; iconv -f iso-8859-1 -t utf-8 $f -o $f.utf8 ; mv $f.utf8 $f + +%build +autoreconf -if +%configure --enable-shared --disable-static --disable-dependency-tracking --enable-debug CFLAGS="-fPIC" +%{__make} %{?_smp_mflags} VERBOSE=1 + +%install +rm -rf $RPM_BUILD_ROOT +make DESTDIR=$RPM_BUILD_ROOT install +rm -f $RPM_BUILD_ROOT/%{_libdir}/libcares.la + +%post -p /sbin/ldconfig +%postun -p /sbin/ldconfig + +%files +%defattr(-, root, root) +%doc README.cares CHANGES NEWS LICENSE +%{_libdir}/*.so.* + +%files devel +%defattr(-, root, root, 0755) +%{_includedir}/ares_nameser.h +%{_includedir}/ares.h +%{_includedir}/ares_build.h +%{_includedir}/ares_dns.h +%{_includedir}/ares_rules.h +%{_includedir}/ares_version.h +%{_libdir}/*.so +%{_libdir}/pkgconfig/libcares.pc +%{_mandir}/man3/ares_* +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/c-ares*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-cctz.sh b/packages/rpm/deps/build-cctz.sh new file mode 100644 index 000000000000..135cfe21d8af --- /dev/null +++ b/packages/rpm/deps/build-cctz.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +VERSION=2.3 +VERSIONSSV=ssv1 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +sudo yum -y install libzstd-devel zlib-devel || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +#cp /home/ykuznetsov/rocksdb-6.5.2.tar.gz "`rpm -E %_sourcedir`"/ + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +# download librdkafka source RPM +SPEC_FILE=$(rpm -E %_specdir)/cctz-$VERSION-$VERSIONSSV.spec + +cat << 'EOF' > $SPEC_FILE +Name: cctz +Version: %{_version} +Release: ssv1%{?dist} +Summary: cctz give C++ programmers all the necessary tools for computing with dates, times, and time zones in a simple and correct manner. +Group: Development/Libraries/C and C++ +License: ASL2.0 +URL: https://github.com/google/cctz +Source0: https://github.com/google/cctz/archive/refs/tags/v%{version}.tar.gz +BuildRequires: cmake +BuildRequires: gcc +BuildRequires: gcc-c++ + +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{_version}-%{release}-XXXXXX) + +%description + +cctz give C++ programmers all the necessary tools for computing with dates, times, and time zones in a simple and correct manner. + +%package -n %{name}-devel +Summary: cctz give C++ programmers all the necessary tools for computing with dates, times, and time zones in a simple and correct manner. +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +%description -n %{name}-devel + +cctz give C++ programmers all the necessary tools for computing with dates, times, and time zones in a simple and correct manner. + +%prep +%autosetup -p1 + +%build +%cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + +%cmake_build + +%install +%cmake_install + +%clean +rm -rf %{buildroot} +%post -n %{name} -p /sbin/ldconfig +%postun -n %{name} -p /sbin/ldconfig + +%files -n %{name} +%{_bindir}/time_tool +%{_libdir}/lib%{name}.so + +%files -n %{name}-devel + +%{_includedir}/%{name} +%{_libdir}/cmake/%{name} + +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/cctz-*$VERSION-*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-clickhouse-cpp.sh b/packages/rpm/deps/build-clickhouse-cpp.sh new file mode 100644 index 000000000000..72028403a0de --- /dev/null +++ b/packages/rpm/deps/build-clickhouse-cpp.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +VERSION=$1 +VERSIONSSV=ssv2 +SCRIPTDIR=$(dirname "$0") + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +SPEC_FILE=$(rpm -E %_specdir)/clickhouse-cpp-$VERSION-$VERSIONSSV.spec + +cat << 'EOF' > $SPEC_FILE +Name: clickhouse-cpp +Version: %{_version} +Release: ssv1%{?dist} +Summary: Clickhouse C++ client library +Group: Development/Libraries/C and C++ +License: Apache License 2.0 +URL: https://github.com/ClickHouse/clickhouse-cpp +Source0: https://github.com/yoori/clickhouse-cpp/archive/refs/tags/v%{version}.tar.gz +#Source0: https://github.com/ClickHouse/clickhouse-cpp/archive/refs/tags/v{version}.tar.gz +BuildRequires: cmake +BuildRequires: gcc +BuildRequires: gcc-c++ + +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{_version}-%{release}-XXXXXX) + +%description + +Clickhouse C++ client library + +%package -n %{name}-devel +Summary: Clickhouse C++ client library +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +%description -n %{name}-devel + +Clickhouse C++ client library + +%prep +%setup -q -n clickhouse-cpp-%{_version} + +%build +%cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX:PATH=/usr \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_INSTALL_LIBDIR=%{_lib} + +make -j + +%install +DESTDIR=%{buildroot}/ make install INSTALL_PATH=%{buildroot}/ +#mv %{buildroot}/usr/lib %{buildroot}/usr/lib64 + +%clean +rm -rf %{buildroot} + +%files -n %{name} +%license LICENSE +%doc README.md + +%{_libdir}/libclickhouse-cpp-lib.so + +%files -n %{name}-devel + +%{_includedir}/clickhouse +%{_includedir}/clickhouse-cpp-deps +%{_libdir}/libclickhouse-cpp-lib-static.a +%{_libdir}/clickhouse-cpp-deps +%{_libdir}/cmake +%{_libdir}/pkgconfig + +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/clickhouse-cpp-*$VERSION-*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-fmt.sh b/packages/rpm/deps/build-fmt.sh new file mode 100644 index 000000000000..0761ade41717 --- /dev/null +++ b/packages/rpm/deps/build-fmt.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +VERSION=$1 +VERSIONSSV=ssv2 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +sudo yum -y install libzstd-devel zlib-devel || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +#cp /home/ykuznetsov/rocksdb-6.5.2.tar.gz "`rpm -E %_sourcedir`"/ + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +# download librdkafka source RPM +SPEC_FILE=$(rpm -E %_specdir)/fmt-$VERSION-$VERSIONSSV.spec + +cat << 'EOF' > $SPEC_FILE +Name: fmt +Version: %{_version} +Release: ssv1%{?dist} +Summary: Small, safe and fast formatting library for C++ +Group: Development/Libraries/C and C++ +License: BSD and ASL2.0 +URL: https://github.com/fmtlib/fmt +Source0: https://github.com/fmtlib/fmt/archive/refs/tags/%{version}.tar.gz +BuildRequires: cmake +BuildRequires: gcc +BuildRequires: gcc-c++ +BuildRequires: ninja-build + +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{_version}-%{release}-XXXXXX) + +%description + +C++ Format is an open-source formatting library for C++. It can be used as a +safe alternative to printf or as a fast alternative to IOStreams. + +%package -n %{name}-devel +Summary: Small, safe and fast formatting library for C++ +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +%description -n %{name}-devel + +C++ Format is an open-source formatting library for C++. It can be used as a +safe alternative to printf or as a fast alternative to IOStreams. + +%prep +%autosetup -p1 + +%build +%cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \ + -DFMT_CMAKE_DIR:STRING=%{_libdir}/cmake/%{name} \ + -DFMT_LIB_DIR:STRING=%{_libdir} + +%cmake_build + +%install +%cmake_install + +%clean +rm -rf %{buildroot} +%post -n %{name} -p /sbin/ldconfig +%postun -n %{name} -p /sbin/ldconfig + +%files -n %{name} +%license LICENSE.rst +%doc ChangeLog.rst README.rst + +%{_libdir}/lib%{name}.so.9* + +%files -n %{name}-devel + +%{_includedir}/%{name} +%{_libdir}/lib%{name}.so +%{_libdir}/cmake/%{name} +%{_libdir}/pkgconfig/%{name}.pc + +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/fmt-*$VERSION-*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-gtest.sh b/packages/rpm/deps/build-gtest.sh new file mode 100644 index 000000000000..e20859e92554 --- /dev/null +++ b/packages/rpm/deps/build-gtest.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +VERSION=$1 +VERSIONSSV=ssv2 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +sudo yum -y install libzstd-devel zlib-devel || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +#cp /home/ykuznetsov/rocksdb-6.5.2.tar.gz "`rpm -E %_sourcedir`"/ + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +# download librdkafka source RPM +SPEC_FILE=$(rpm -E %_specdir)/gtest-$VERSION-$VERSIONSSV.spec + +cat << 'EOF' > $SPEC_FILE +Name: gtest +Version: %{_version} +Release: ssv2%{?dist} +Summary: Google C++ testing framework +Group: Development/Libraries/C and C++ +License: BSD and ASL2.0 +URL: https://github.com/google/googletest +Source0: https://github.com/google/googletest/archive/refs/tags/release-%{version}.tar.gz +BuildRequires: cmake +BuildRequires: gcc-c++ + +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{_version}-%{release}-XXXXXX) +%define __product protobuf +%define __cmake_build_dir build +%description + +This package contains development files for gtest. + +%package -n %{name}-devel +Summary: Google C++ testing framework +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +%description -n %{name}-devel + +This package contains development files for gtest. + +%prep +%setup -q -n googletest-release-%{_version} + +%build +mkdir build +pushd build +cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=ON -DCMAKE_SKIP_BUILD_RPATH=TRUE -Dgtest_build_tests=OFF .. +make -j +popd + +%install +rm -rf %{buildroot} +mkdir -p %{buildroot} +echo "Install to %{buildroot}" + +pushd build +DESTDIR=%{buildroot}/ make install INSTALL_PATH=%{buildroot}/ +popd + +%clean +rm -rf %{buildroot} +%post -n %{name} -p /sbin/ldconfig +%postun -n %{name} -p /sbin/ldconfig + +%files -n %{name} +%defattr(444,root,root) +%{_libdir}/libgmock.so.* +%{_libdir}/libgtest.so.* +%{_libdir}/libgmock_main.so.* +%{_libdir}/libgtest_main.so.* + +%files -n %{name}-devel +%{_libdir}/libgtest.so +%{_libdir}/libgtest_main.so +%{_libdir}/libgmock.so +%{_libdir}/libgmock_main.so +%{_includedir}/gtest +%{_includedir}/gmock +%{_libdir}/cmake/GTest +%{_libdir}/pkgconfig/*.pc +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/gtest-*$VERSION-*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-libcurl.sh b/packages/rpm/deps/build-libcurl.sh new file mode 100644 index 000000000000..2216f9ec31b8 --- /dev/null +++ b/packages/rpm/deps/build-libcurl.sh @@ -0,0 +1,472 @@ +#!/bin/bash + +VERSION=7.86.0 +VERSIONSSV=ssv1 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +sudo yum -y install libzstd-devel zlib-devel || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +# download librdkafka source RPM +SPEC_FILE=$(rpm -E %_specdir)/libcurl-$VERSION-$VERSIONSSV.spec + +SOURCES_DIR=$(rpm -E %_sourcedir) +cp libcurl/0101-curl-7.32.0-multilib.patch $SOURCES_DIR/ +cp libcurl/0102-curl-7.84.0-test3026.patch $SOURCES_DIR/ + +cat << 'EOF' > $SPEC_FILE +Summary: A utility for getting files from remote servers (FTP, HTTP, and others) +Name: curl +Version: %{_version} +Release: 1%{?dist} +License: MIT +Source0: https://curl.se/download/%{name}-%{version}.tar.xz +Source1: https://curl.se/download/%{name}-%{version}.tar.xz.asc +# The curl download page ( https://curl.se/download.html ) links +# to Daniel's address page https://daniel.haxx.se/address.html for the GPG Key, +# which points to the GPG key as of April 7th 2016 of https://daniel.haxx.se/mykey.asc +Source2: https://daniel.haxx.se/mykey.asc + +# patch making libcurl multilib ready +Patch101: 0101-curl-7.32.0-multilib.patch + +# test3026: disable valgrind +Patch102: 0102-curl-7.84.0-test3026.patch + +Provides: curl-full = %{version}-%{release} +Provides: webclient +URL: https://curl.se/ +BuildRequires: automake +BuildRequires: brotli-devel +BuildRequires: coreutils +BuildRequires: gcc +BuildRequires: groff +BuildRequires: krb5-devel +BuildRequires: libidn2-devel +BuildRequires: libnghttp2-devel +BuildRequires: libpsl-devel +BuildRequires: libssh-devel +BuildRequires: libtool +BuildRequires: make +BuildRequires: openldap-devel +BuildRequires: openssh-clients +BuildRequires: openssh-server +BuildRequires: openssl-devel +BuildRequires: perl-interpreter +BuildRequires: pkgconfig +BuildRequires: python3 +BuildRequires: python3-devel +BuildRequires: sed +BuildRequires: zlib-devel + +# For gpg verification of source tarball +BuildRequires: gnupg2 + +# needed to compress content of tool_hugehelp.c after changing curl.1 man page +BuildRequires: perl(IO::Compress::Gzip) + +# needed for generation of shell completions +BuildRequires: perl(Getopt::Long) +BuildRequires: perl(Pod::Usage) +BuildRequires: perl(strict) +BuildRequires: perl(warnings) + +# gnutls-serv is used by the upstream test-suite +BuildRequires: gnutls-utils + +# hostname(1) is used by the test-suite but it is missing in armv7hl buildroot +BuildRequires: hostname + +# nghttpx (an HTTP/2 proxy) is used by the upstream test-suite +BuildRequires: libnghttp2 + +# perl modules used in the test suite +BuildRequires: perl(Cwd) +BuildRequires: perl(Digest::MD5) +BuildRequires: perl(Digest::SHA) +BuildRequires: perl(Exporter) +BuildRequires: perl(File::Basename) +BuildRequires: perl(File::Copy) +BuildRequires: perl(File::Spec) +BuildRequires: perl(IPC::Open2) +BuildRequires: perl(MIME::Base64) +BuildRequires: perl(Time::Local) +BuildRequires: perl(Time::HiRes) +BuildRequires: perl(vars) + +%if 0%{?fedora} +# needed for upstream test 1451 +BuildRequires: python3-impacket +%endif + +# The test-suite runs automatically through valgrind if valgrind is available +# on the system. By not installing valgrind into mock's chroot, we disable +# this feature for production builds on architectures where valgrind is known +# to be less reliable, in order to avoid unnecessary build failures (see RHBZ +# #810992, #816175, and #886891). Nevertheless developers are free to install +# valgrind manually to improve test coverage on any architecture. +%ifarch x86_64 +BuildRequires: valgrind +%endif + +# stunnel is used by upstream tests but it does not seem to work reliably +# on s390x and occasionally breaks some tests (mainly 1561 and 1562) +%ifnarch s390x +BuildRequires: stunnel +%endif + +# using an older version of libcurl could result in CURLE_UNKNOWN_OPTION +Requires: libcurl%{?_isa} >= %{version}-%{release} + +# require at least the version of libpsl that we were built against, +# to ensure that we have the necessary symbols available (#1631804) +%global libpsl_version %(pkg-config --modversion libpsl 2>/dev/null || echo 0) + +# require at least the version of libssh that we were built against, +# to ensure that we have the necessary symbols available (#525002, #642796) +%global libssh_version %(pkg-config --modversion libssh 2>/dev/null || echo 0) + +# require at least the version of openssl-libs that we were built against, +# to ensure that we have the necessary symbols available (#1462184, #1462211) +# (we need to translate 3.0.0-alpha16 -> 3.0.0-0.alpha16 and 3.0.0-beta1 -> 3.0.0-0.beta1 though) +%global openssl_version %({ pkg-config --modversion openssl 2>/dev/null || echo 0;} | sed 's|-|-0.|') + +%description +curl is a command line tool for transferring data with URL syntax, supporting +FTP, FTPS, HTTP, HTTPS, SCP, SFTP, TFTP, TELNET, DICT, LDAP, LDAPS, FILE, IMAP, +SMTP, POP3 and RTSP. curl supports SSL certificates, HTTP POST, HTTP PUT, FTP +uploading, HTTP form based upload, proxies, cookies, user+password +authentication (Basic, Digest, NTLM, Negotiate, kerberos...), file transfer +resume, proxy tunneling and a busload of other useful tricks. + +%package -n libcurl +Summary: A library for getting files from web servers +Requires: libpsl%{?_isa} >= %{libpsl_version} +Requires: libssh%{?_isa} >= %{libssh_version} +Requires: openssl-libs%{?_isa} >= 1:%{openssl_version} +Provides: libcurl-full = %{version}-%{release} +Provides: libcurl-full%{?_isa} = %{version}-%{release} + +%description -n libcurl +libcurl is a free and easy-to-use client-side URL transfer library, supporting +FTP, FTPS, HTTP, HTTPS, SCP, SFTP, TFTP, TELNET, DICT, LDAP, LDAPS, FILE, IMAP, +SMTP, POP3 and RTSP. libcurl supports SSL certificates, HTTP POST, HTTP PUT, +FTP uploading, HTTP form based upload, proxies, cookies, user+password +authentication (Basic, Digest, NTLM, Negotiate, Kerberos4), file transfer +resume, http proxy tunneling and more. + +%package -n libcurl-devel +Summary: Files needed for building applications with libcurl +Requires: libcurl%{?_isa} = %{version}-%{release} + +Provides: curl-devel = %{version}-%{release} +Provides: curl-devel%{?_isa} = %{version}-%{release} +Obsoletes: curl-devel < %{version}-%{release} + +%description -n libcurl-devel +The libcurl-devel package includes header files and libraries necessary for +developing programs which use the libcurl library. It contains the API +documentation of the library, too. + +%package -n curl-minimal +Summary: Conservatively configured build of curl for minimal installations +Provides: curl = %{version}-%{release} +Conflicts: curl +Suggests: libcurl-minimal +RemovePathPostfixes: .minimal + +# using an older version of libcurl could result in CURLE_UNKNOWN_OPTION +Requires: libcurl%{?_isa} >= %{version}-%{release} + +%description -n curl-minimal +This is a replacement of the 'curl' package for minimal installations. It +comes with a limited set of features compared to the 'curl' package. On the +other hand, the package is smaller and requires fewer run-time dependencies to +be installed. + +%package -n libcurl-minimal +Summary: Conservatively configured build of libcurl for minimal installations +Requires: openssl-libs%{?_isa} >= 1:%{openssl_version} +Provides: libcurl = %{version}-%{release} +Provides: libcurl%{?_isa} = %{version}-%{release} +Conflicts: libcurl%{?_isa} +RemovePathPostfixes: .minimal +# needed for RemovePathPostfixes to work with shared libraries +%undefine __brp_ldconfig + +%description -n libcurl-minimal +This is a replacement of the 'libcurl' package for minimal installations. It +comes with a limited set of features compared to the 'libcurl' package. On the +other hand, the package is smaller and requires fewer run-time dependencies to +be installed. + +%prep +%{gpgverify} --keyring='%{SOURCE2}' --signature='%{SOURCE1}' --data='%{SOURCE0}' +%setup -q + +# upstream patches + +# Fedora patches +%patch101 -p1 +%patch102 -p1 + +# disable test 1112 (#565305), test 1455 (occasionally fails with 'bind failed +# with errno 98: Address already in use' in Koji environment), and test 1801 +# +printf "1112\n1455\n1184\n1801\n" >> tests/data/DISABLED + +# disable test 1319 on ppc64 (server times out) +%ifarch ppc64 +echo "1319" >> tests/data/DISABLED +%endif + +# disable tests 320..322 on ppc64le where it started to hang/fail +%ifarch ppc64le +printf "320\n321\n322\n" >> tests/data/DISABLED +%endif + +# temporarily disable tests 582 and 1452 on s390x (client times out) +%ifarch s390x +printf "582\n1452\n" >> tests/data/DISABLED +%endif + +# temporarily disable tests 702 703 716 on armv7hl (#1829180) +%ifarch armv7hl +printf "702\n703\n716\n" >> tests/data/DISABLED +%endif + +# temporarily disable tests 300{0,1} on x86_64 (stunnel clashes with itself) +%ifarch x86_64 +printf "3000\n3001\n" >> tests/data/DISABLED +%endif + +# test3026: avoid pthread_create() failure due to resource exhaustion on i386 +%ifarch %{ix86} +sed -e 's|NUM_THREADS 1000$|NUM_THREADS 256|' \ + -i tests/libtest/lib3026.c +%endif + +# adapt test 323 for updated OpenSSL +sed -e 's|^35$|35,52|' -i tests/data/test323 + +# use localhost6 instead of ip6-localhost in the curl test-suite +( + # avoid glob expansion in the trace output of `bash -x` + { set +x; } 2>/dev/null + cmd="sed -e 's|ip6-localhost|localhost6|' -i tests/data/test[0-9]*" + printf "+ %s\n" "$cmd" >&2 + eval "$cmd" +) + +# regenerate the configure script and Makefile.in files +autoreconf -fiv + +%build +mkdir build-{full,minimal} +export common_configure_opts=" \ + --cache-file=../config.cache \ + --disable-static \ + --enable-hsts \ + --enable-ipv6 \ + --enable-symbol-hiding \ + --enable-threaded-resolver \ + --without-zstd \ + --with-gssapi \ + --with-libidn2 \ + --with-nghttp2 \ + --with-ssl --with-ca-bundle=%{_sysconfdir}/pki/tls/certs/ca-bundle.crt" + +%global _configure ../configure + +# configure minimal build +( + cd build-minimal + %configure $common_configure_opts \ + --disable-dict \ + --disable-gopher \ + --disable-imap \ + --disable-ldap \ + --disable-ldaps \ + --disable-manual \ + --disable-mqtt \ + --disable-ntlm \ + --disable-ntlm-wb \ + --disable-pop3 \ + --disable-rtsp \ + --disable-smb \ + --disable-smtp \ + --disable-telnet \ + --disable-tftp \ + --disable-tls-srp \ + --without-brotli \ + --without-libpsl \ + --without-libssh +) + +# configure full build +( + cd build-full + %configure $common_configure_opts \ + --enable-dict \ + --enable-gopher \ + --enable-imap \ + --enable-ldap \ + --enable-ldaps \ + --enable-manual \ + --enable-mqtt \ + --enable-ntlm \ + --enable-ntlm-wb \ + --enable-pop3 \ + --enable-rtsp \ + --enable-smb \ + --enable-smtp \ + --enable-telnet \ + --enable-tftp \ + --enable-tls-srp \ + --with-brotli \ + --with-libpsl \ + --with-libssh +) + +# avoid using rpath +sed -e 's/^runpath_var=.*/runpath_var=/' \ + -e 's/^hardcode_libdir_flag_spec=".*"$/hardcode_libdir_flag_spec=""/' \ + -i build-{full,minimal}/libtool + +%make_build V=1 -C build-minimal +%make_build V=1 -C build-full + +%check +# compile upstream test-cases +#%make_build V=1 -C build-minimal/tests +#%make_build V=1 -C build-full/tests + +# relax crypto policy for the test-suite to make it pass again (#1610888) +#export OPENSSL_SYSTEM_CIPHERS_OVERRIDE=XXX +#export OPENSSL_CONF= + +# make runtests.pl work for out-of-tree builds +#export srcdir=../../tests + +# prevent valgrind from being extremely slow (#1662656) +# https://fedoraproject.org/wiki/Changes/DebuginfodByDefault +#unset DEBUGINFOD_URLS + +# run the upstream test-suite for both curl-minimal and curl-full +#for size in minimal full; do ( +# cd build-${size} + + # we have to override LD_LIBRARY_PATH because we eliminated rpath +# export LD_LIBRARY_PATH="${PWD}/lib/.libs" + +# cd tests +# perl -I../../tests ../../tests/runtests.pl -a -p -v '!flaky' +#) +#done + + +%install +# install and rename the library that will be packaged as libcurl-minimal +%make_install -C build-minimal/lib +rm -f ${RPM_BUILD_ROOT}%{_libdir}/libcurl.{la,so} +for i in ${RPM_BUILD_ROOT}%{_libdir}/*; do + mv -v $i $i.minimal +done + +# install and rename the executable that will be packaged as curl-minimal +%make_install -C build-minimal/src +mv -v ${RPM_BUILD_ROOT}%{_bindir}/curl{,.minimal} + +# install libcurl.m4 +install -d $RPM_BUILD_ROOT%{_datadir}/aclocal +install -m 644 docs/libcurl/libcurl.m4 $RPM_BUILD_ROOT%{_datadir}/aclocal + +# install the executable and library that will be packaged as curl and libcurl +cd build-full +%make_install +# install zsh completion for curl +# (we have to override LD_LIBRARY_PATH because we eliminated rpath) +LD_LIBRARY_PATH="$RPM_BUILD_ROOT%{_libdir}:$LD_LIBRARY_PATH" \ + %make_install -C scripts + +# do not install /usr/share/fish/completions/curl.fish which is also installed +# by fish-3.0.2-1.module_f31+3716+57207597 and would trigger a conflict +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/fish + +rm -f ${RPM_BUILD_ROOT}%{_libdir}/libcurl.la + +%ldconfig_scriptlets -n libcurl + +%ldconfig_scriptlets -n libcurl-minimal + +%files +%doc CHANGES +%doc README +%doc docs/BUGS.md +%doc docs/FAQ +%doc docs/FEATURES.md +%doc docs/TODO +%doc docs/TheArtOfHttpScripting.md +%{_bindir}/curl +%{_mandir}/man1/curl.1* +%{_datadir}/zsh + +%files -n libcurl +%license COPYING +%{_libdir}/libcurl.so.4 +%{_libdir}/libcurl.so.4.[0-9].[0-9] + +%files -n libcurl-devel +%doc docs/examples/*.c docs/examples/Makefile.example docs/INTERNALS.md +%doc docs/CONTRIBUTE.md docs/libcurl/ABI.md +%{_bindir}/curl-config* +%{_includedir}/curl +%{_libdir}/*.so +%{_libdir}/pkgconfig/*.pc +%{_mandir}/man1/curl-config.1* +%{_mandir}/man3/* +%{_datadir}/aclocal/libcurl.m4 + +%files -n curl-minimal +%{_bindir}/curl.minimal +%{_mandir}/man1/curl.1* + +%files -n libcurl-minimal +%license COPYING +%{_libdir}/libcurl.so.4.minimal +%{_libdir}/libcurl.so.4.[0-9].[0-9].minimal +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/libcurl*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/build-spdlog.sh b/packages/rpm/deps/build-spdlog.sh new file mode 100644 index 000000000000..e7ca8ae818e6 --- /dev/null +++ b/packages/rpm/deps/build-spdlog.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +VERSION=$1 +VERSIONSSV=ssv1 + +# script require 'sudo rpm' for install RPM packages +# and access to mirror.yandex.ru repository + +# create build/RPMS folder - all built packages will be duplicated here +RES_TMP=build/TMP/ +RES_RPMS=build/RPMS/ +rm -rf "$RES_TMP" + +mkdir -p $RES_TMP +mkdir -p $RES_RPMS + +# download and install packages required for build +sudo yum -y install spectool yum-utils rpmdevtools redhat-rpm-config rpm-build autoconf automake libtool \ + glib2-devel cmake gcc-c++ epel-rpm-macros \ + || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +sudo yum -y install libzstd-devel zlib-devel || \ + { echo "can't install base packages" >&2 ; exit 1 ; } + +# create folders for RPM build environment +mkdir -vp `rpm -E '%_tmppath %_rpmdir %_builddir %_sourcedir %_specdir %_srcrpmdir %_rpmdir/%_arch'` + +#cp /home/ykuznetsov/rocksdb-6.5.2.tar.gz "`rpm -E %_sourcedir`"/ + +BIN_RPM_FOLDER=$(rpm -E '%_rpmdir/%_arch') + +# download librdkafka source RPM +SPEC_FILE=$(rpm -E %_specdir)/spdlog-$VERSION-$VERSIONSSV.spec + +cat << 'EOF' > $SPEC_FILE +Name: spdlog +Version: %{_version} +Release: ssv2%{?dist} +Summary: Super fast C++ logging library +Group: Development/Libraries/C and C++ +License: BSD-2-Clause +URL: https://github.com/gabime/spdlog +Source0: https://github.com/gabime/spdlog/archive/refs/tags/v%{_version}.tar.gz +BuildRequires: cmake +BuildRequires: gcc-c++ + +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{_version}-%{release}-XXXXXX) +%define __product protobuf +%define __cmake_build_dir build +%description + +This is a packaged version of the gabime/spdlog C++ logging library available at Github. + +%package -n %{name}-devel +Summary: Super fast C++ logging library +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +%description -n %{name}-devel + +This is a packaged version of the gabime/spdlog C++ logging library available at Github. + +%prep +%setup -q -n spdlog-%{_version} + +%build +mkdir build-shared +pushd build-shared +cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSPDLOG_INSTALL:BOOL=ON -DSPDLOG_BUILD_SHARED:BOOL=ON -DCMAKE_INSTALL_PREFIX=/usr .. +make -j +popd + +mkdir build-static +pushd build-static +cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSPDLOG_INSTALL:BOOL=ON -DSPDLOG_BUILD_SHARED:BOOL=OFF -DCMAKE_INSTALL_PREFIX=/usr .. +make -j +popd + +%install +rm -rf %{buildroot} +mkdir -p %{buildroot} +echo "Install to %{buildroot}" + +pushd build-shared +DESTDIR=%{buildroot}/ make install INSTALL_PATH=%{buildroot}/ +popd + +pushd build-static +DESTDIR=%{buildroot}/ make install INSTALL_PATH=%{buildroot}/ +popd + +#mv %{buildroot}/usr/lib %{buildroot}/usr/lib64 + +%clean +rm -rf %{buildroot} +%post -n %{name} -p /sbin/ldconfig +%postun -n %{name} -p /sbin/ldconfig + +%files -n %{name} +%{_libdir}/libspdlog.so.1* + +%files -n %{name}-devel +%{_libdir}/lib%{name}.so +%{_libdir}/lib%{name}.a +%{_includedir}/spdlog +%{_libdir}/cmake/spdlog +%{_libdir}/pkgconfig/*.pc +EOF + +$SUDO_PREFIX yum-builddep -y "$SPEC_FILE" || \ + { echo "can't install build requirements" >&2 ; exit 1 ; } + +spectool --force -g -R --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't download sources" >&2 ; exit 1 ; } + +rpmbuild --force -ba --define "_version $VERSION" "$SPEC_FILE" || \ + { echo "can't build RPM" >&2 ; exit 1 ; } + +# install +cp $BIN_RPM_FOLDER/spdlog*.rpm $RES_RPMS/ + diff --git a/packages/rpm/deps/clickhouse-cpp/installCMake.cmake b/packages/rpm/deps/clickhouse-cpp/installCMake.cmake new file mode 100644 index 000000000000..fcc34f6eb39f --- /dev/null +++ b/packages/rpm/deps/clickhouse-cpp/installCMake.cmake @@ -0,0 +1,36 @@ +include(CMakePackageConfigHelpers) + +if(NOT DEFINED CLICKHOUSE_CPP_VERSION) + set(CLICKHOUSE_CPP_VERSION "1.0.0") +endif() + +set_verbose(CLICKHOUSE_CPP_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/clickhouse-cpp CACHE STRING + "Installation directory for cmake files, a relative path that " + "will be joined with ${CMAKE_INSTALL_PREFIX} or an absolute " + "path.") + +set(version_config ${PROJECT_BINARY_DIR}/clickhouse-cpp-config-version.cmake) +set(project_config ${PROJECT_BINARY_DIR}/clickhouse-cpp-config.cmake) + +# Generate the version, config and target files into the build directory. +write_basic_package_version_file( + ${version_config} + VERSION ${CLICKHOUSE_CPP_VERSION} + COMPATIBILITY AnyNewerVersion) + +export(TARGETS clickhouse-cpp-lib NAMESPACE clickhouse-cpp:: + FILE ${PROJECT_BINARY_DIR}/clickhouse-cpp-targets.cmake) + +# Install version, config and target files. + +install( + FILES ${project_config} ${version_config} + DESTINATION ${CLICKHOUSE_CPP_CMAKE_DIR}) + +install(EXPORT clickhouse-cpp-lib DESTINATION ${CLICKHOUSE_CPP_CMAKE_DIR} + NAMESPACE clickhouse-cpp::) + +#install(FILES $ +# DESTINATION ${CLICKHOUSE_CPP_LIB_DIR} OPTIONAL) + +#install(FILES "${pkgconfig}" DESTINATION "${CLICKHOUSE_CPP_PKGCONFIG_DIR}") diff --git a/packages/rpm/userver.spec b/packages/rpm/userver.spec new file mode 100644 index 000000000000..80d6d6e2a8d2 --- /dev/null +++ b/packages/rpm/userver.spec @@ -0,0 +1,131 @@ +%define __boost_suffix 176 + +Name: userver +Version: %{_version} +Release: ssv1%{?dist} +Summary: userver framework +Group: Development/Libraries/C and C++ +License: Apache License 2.0 +URL: https://github.com/userver-framework/userver +Source0: https://github.com/yoori/userver/archive/refs/tags/v%{_version}.tar.gz +BuildRequires: cmake +BuildRequires: python36 python3-libs +BuildRequires: gcc-c++ +BuildRequires: cctz-devel gtest-devel spdlog-devel +Requires: cctz gtest spdlog +BuildRequires: libnghttp2-devel libidn2-devel brotli-devel c-ares-devel +Requires: libnghttp2 libidn2 brotli c-ares +BuildRequires: libcurl-devel libbson-devel +Requires: libcurl libbson +BuildRequires: mongo-c-driver-libs mongo-c-driver +Requires: mongo-c-driver-libs mongo-c-driver +BuildRequires: fmt-devel +Requires: fmt +BuildRequires: libssh-devel +Requires: libssh +BuildRequires: hiredis-devel +Requires: hiredis +BuildRequires: rapidjson-devel +BuildRequires: moodycamel-concurrentqueue-devel +BuildRequires: compiler-rt +BuildRequires: boost%{__boost_suffix}-devel >= 1.67.0 +BuildRequires: amqp-cpp-devel +BuildRequires: clickhouse-cpp-devel +BuildRequires: libatomic +Requires: libatomic +BuildRequires: cryptopp-devel +Requires: cryptopp +BuildRequires: libev-devel +Requires: libev +BuildRequires: re2-devel +Requires: re2 +BuildRequires: libiconv-devel +Requires: libiconv +BuildRequires: python3-jinja2 +Requires: protobuf >= 3.21.9 +BuildRequires: protobuf-devel >= 3.21.9 +Requires: grpc >= 1.48.1 +Requires: grpc-cpp >= 1.48.1 +BuildRequires: grpc-devel >= 1.48.1 +BuildRequires: grpc-plugins >= 1.48.1 + +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{_version}-%{release}-XXXXXX) + +%description + +Open source asynchronous framework with a rich set of abstractions for fast and comfortable creation of C++ microservices, services and utilities. + +%package -n %{name}-devel +Summary: userver framework +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +Requires: cctz-devel gtest-devel spdlog-devel +Requires: libnghttp2-devel libidn2-devel brotli-devel c-ares-devel +Requires: libcurl-devel libbson-devel +Requires: mongo-c-driver-libs mongo-c-driver +Requires: fmt-devel +Requires: libssh-devel +Requires: hiredis-devel +Requires: clickhouse-cpp +Requires: amqp-cpp +Requires: protobuf-devel >= 3.21.9 +Requires: grpc-devel >= 1.48.1 +Requires: grpc-plugins >= 1.48.1 + +%description -n %{name}-devel + +Open source asynchronous framework with a rich set of abstractions for fast and comfortable creation of C++ microservices, services and utilities. + +%prep +%setup -q -n userver-%{_version} + +%build +rm -rf build +mkdir build +pushd build +export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.6/site-packages + +# -DUSERVER_LTO:BOOL=OFF as workaround for dwz crash +cmake -DUSERVER_FEATURE_PATCH_LIBPQ=0 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSERVER_FEATURE_STACKTRACE=0 \ + -DUSERVER_PYTHON_PATH=/usr/bin/python3 \ + -DUSERVER_BUILD_SHARED_LIBS:BOOL=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr \ + -DCMAKE_INSTALL_LIBDIR=lib64 -DUSERVER_DOWNLOAD_PACKAGES:BOOL=OFF \ + -DUSERVER_LTO:BOOL=OFF \ + -DUSERVER_GEN_GDB_DEBUGINFO:BOOL=ON \ + -DUSERVER_IS_THE_ROOT_PROJECT:BOOL=OFF \ + .. + +cmake --build . -j 8 +popd + +echo ">>>> Third party used from GIT: " +ls -1 third_party/ | grep -v -E '^(boost_stacktrace|compiler-rt|moodycamel|pfr|rapidjson)$' +echo "<<<<" + +%install +rm -rf %{buildroot} +mkdir -p %{buildroot} +echo "Install to %{buildroot}" + +pushd build +DESTDIR=%{buildroot}/ make install INSTALL_PATH=%{buildroot}/ +# move third party installations into lib64 +#cp -r %{buildroot}/usr/lib %{buildroot}/usr/lib64/ +#rm -rf %{buildroot}/usr/lib + +# TODO cmake config install +#mv %{buildroot}/usr/cmake %{buildroot}/usr/lib64/ +#rm -rf %{buildroot}/usr/cmake +popd + +%clean +rm -rf %{buildroot} + +%files -n %{name} +#%{_bindir}/userver/ +%{_libdir}/lib*.so + +%files -n %{name}-devel +%{_includedir}/ +#{_libdir}/lib*.a +%{_libdir}/cmake/userver/*.cmake diff --git a/postgresql/CMakeLists.txt b/postgresql/CMakeLists.txt index 6dc5775a2efe..da068b410621 100644 --- a/postgresql/CMakeLists.txt +++ b/postgresql/CMakeLists.txt @@ -50,20 +50,21 @@ file(GLOB_RECURSE BENCH_SOURCES list (REMOVE_ITEM SOURCES ${BENCH_SOURCES}) -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) target_include_directories(${PROJECT_NAME} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include + $ + $ PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_include_directories (${PROJECT_NAME} PRIVATE $ ) -target_link_libraries(${PROJECT_NAME} +userver_target_link_libraries(${PROJECT_NAME} PUBLIC userver-core PRIVATE diff --git a/postgresql/pq-extra/CMakeLists.txt b/postgresql/pq-extra/CMakeLists.txt index c1232e48df20..1efc820891a4 100644 --- a/postgresql/pq-extra/CMakeLists.txt +++ b/postgresql/pq-extra/CMakeLists.txt @@ -39,7 +39,7 @@ set(PQ_EXTRA_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/pq_workaround.c ) -add_library(${PROJECT_NAME} STATIC ${PQ_EXTRA_SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${PQ_EXTRA_SOURCES}) target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/rabbitmq/CMakeLists.txt b/rabbitmq/CMakeLists.txt index c617b4f8036e..da6a9aabc0d8 100644 --- a/rabbitmq/CMakeLists.txt +++ b/rabbitmq/CMakeLists.txt @@ -14,7 +14,7 @@ file(GLOB_RECURSE RMQ_FUNCTIONAL_TEST_SOURCES ) list(REMOVE_ITEM SOURCES ${RMQ_FUNCTIONAL_TEST_SOURCES}) -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) target_link_libraries(${PROJECT_NAME} PUBLIC userver-core) if (USERVER_CONAN) find_package(amqpcpp) @@ -24,7 +24,9 @@ endif() target_link_libraries(${PROJECT_NAME} PRIVATE amqpcpp) target_include_directories( ${PROJECT_NAME} - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + PUBLIC + $ + $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) @@ -47,3 +49,5 @@ if (USERVER_IS_THE_ROOT_PROJECT) add_subdirectory(functional_tests) endif() + +userver_install_library(TARGETS ${PROJECT_NAME}) diff --git a/redis/CMakeLists.txt b/redis/CMakeLists.txt index 34599699a1dd..abd586a4ccb6 100644 --- a/redis/CMakeLists.txt +++ b/redis/CMakeLists.txt @@ -32,7 +32,7 @@ file(GLOB_RECURSE REDIS_FUNCTIONAL_TEST_SOURCES ) list(REMOVE_ITEM SOURCES ${REDIS_FUNCTIONAL_TEST_SOURCES}) -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) # for libev @@ -58,7 +58,7 @@ if(USERVER_FEATURE_REDIS_TLS) add_definitions(-DUSERVER_FEATURE_REDIS_TLS=1) endif() -target_link_libraries(${PROJECT_NAME} +userver_target_link_libraries(${PROJECT_NAME} PUBLIC userver-core PRIVATE @@ -78,9 +78,10 @@ endif() target_include_directories( ${PROJECT_NAME} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include + $ + $ PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_include_directories (${PROJECT_NAME} PRIVATE $ @@ -89,7 +90,7 @@ target_include_directories (${PROJECT_NAME} PRIVATE # Helper library for testing user code with redis. Provides mock_client_base.hpp # and other helper headers if (USERVER_FEATURE_UTEST) - add_library(${PROJECT_NAME}_utest STATIC ${UTESTLIB_SOURCES}) + userver_add_library(${PROJECT_NAME}_utest SOURCES ${UTESTLIB_SOURCES}) target_include_directories( ${PROJECT_NAME}_utest PUBLIC diff --git a/scripts/grpc/requirements.txt b/scripts/grpc/requirements.txt index 5de3c0749685..6107ffd8500e 100644 --- a/scripts/grpc/requirements.txt +++ b/scripts/grpc/requirements.txt @@ -1 +1 @@ -protobuf>=4.21.12 +protobuf>=3.21.9 diff --git a/testsuite/requirements-grpc.txt b/testsuite/requirements-grpc.txt index 3c2af4c3631a..929e679afe09 100644 --- a/testsuite/requirements-grpc.txt +++ b/testsuite/requirements-grpc.txt @@ -1,2 +1,2 @@ -grpcio>=1.50.0 -grpcio-tools>=1.50.0 +grpcio>=1.48.0 +grpcio-tools>=1.48.0 diff --git a/third_party/boost_stacktrace/CMakeLists.txt b/third_party/boost_stacktrace/CMakeLists.txt index d4d72638dd06..f65b95efc407 100644 --- a/third_party/boost_stacktrace/CMakeLists.txt +++ b/third_party/boost_stacktrace/CMakeLists.txt @@ -55,7 +55,7 @@ else() #set(BACKTRACE_USERVER_LIBRARIES dl) endif() -add_library (${PROJECT_NAME} STATIC ${BACKTRACE_USERVER_SOURCES}) +userver_add_library (${PROJECT_NAME} SOURCES ${BACKTRACE_USERVER_SOURCES}) if (BacktraceIncludeFile) target_compile_definitions( ${PROJECT_NAME} @@ -67,7 +67,12 @@ else() ) endif() target_compile_definitions(${PROJECT_NAME} INTERFACE BOOST_STACKTRACE_LINK) -target_include_directories(${PROJECT_NAME} PUBLIC include ${Boost_INCLUDE_DIRS}) +target_include_directories(${PROJECT_NAME} PUBLIC + $ + $ + ${Boost_INCLUDE_DIRS}) target_link_libraries(${PROJECT_NAME} PUBLIC ${BACKTRACE_USERVER_LIBRARIES}) set_target_properties (${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) + +userver_export(TARGETS ${PROJECT_NAME}) diff --git a/third_party/compiler-rt/CMakeLists.txt b/third_party/compiler-rt/CMakeLists.txt index 0c4dd9160e4f..77b35045c34d 100644 --- a/third_party/compiler-rt/CMakeLists.txt +++ b/third_party/compiler-rt/CMakeLists.txt @@ -3,6 +3,10 @@ project(userver-compiler-rt-parts) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(SOURCES lib/builtins/muloti4.c) add_library(${PROJECT_NAME} STATIC ${SOURCES}) + + # workaround for INSTALL dependent libraries without install this static library + set_target_properties (${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) + else() add_library(${PROJECT_NAME} INTERFACE) endif() diff --git a/tools/congestion_control_emulator/CMakeLists.txt b/tools/congestion_control_emulator/CMakeLists.txt index 373e68675919..64eb5f03fe51 100644 --- a/tools/congestion_control_emulator/CMakeLists.txt +++ b/tools/congestion_control_emulator/CMakeLists.txt @@ -9,3 +9,5 @@ target_link_libraries (${PROJECT_NAME} userver-core Boost::program_options ) + +install(TARGETS ${PROJECT_NAME}) diff --git a/tools/dns_resolver/CMakeLists.txt b/tools/dns_resolver/CMakeLists.txt index 85567d957ec0..1add7d9f4225 100644 --- a/tools/dns_resolver/CMakeLists.txt +++ b/tools/dns_resolver/CMakeLists.txt @@ -16,3 +16,5 @@ target_include_directories (${PROJECT_NAME} SYSTEM PRIVATE $ ) target_compile_definitions(${PROJECT_NAME} PRIVATE SPDLOG_FMT_EXTERNAL=1) + +install(TARGETS ${PROJECT_NAME}) diff --git a/tools/engine/CMakeLists.txt b/tools/engine/CMakeLists.txt index cb8a24fca078..69e762d1cbcc 100644 --- a/tools/engine/CMakeLists.txt +++ b/tools/engine/CMakeLists.txt @@ -9,3 +9,5 @@ target_link_libraries (${PROJECT_NAME} userver-core Boost::program_options ) + +install(TARGETS ${PROJECT_NAME}) diff --git a/tools/httpclient/CMakeLists.txt b/tools/httpclient/CMakeLists.txt index 393ba7128965..a370457f9afa 100644 --- a/tools/httpclient/CMakeLists.txt +++ b/tools/httpclient/CMakeLists.txt @@ -9,3 +9,5 @@ target_link_libraries (${PROJECT_NAME} userver-core Boost::program_options ) + +install(TARGETS ${PROJECT_NAME}) diff --git a/tools/json2yaml/CMakeLists.txt b/tools/json2yaml/CMakeLists.txt index 62eef068b3b8..81959e8fcb2b 100644 --- a/tools/json2yaml/CMakeLists.txt +++ b/tools/json2yaml/CMakeLists.txt @@ -6,3 +6,5 @@ add_executable (${PROJECT_NAME} ${SOURCES}) target_link_libraries (${PROJECT_NAME} userver-universal ) + +install(TARGETS ${PROJECT_NAME}) diff --git a/tools/netcat/CMakeLists.txt b/tools/netcat/CMakeLists.txt index a627025f7784..ab25627e5fe4 100644 --- a/tools/netcat/CMakeLists.txt +++ b/tools/netcat/CMakeLists.txt @@ -9,3 +9,5 @@ target_link_libraries (${PROJECT_NAME} userver-core Boost::program_options ) + +install(TARGETS ${PROJECT_NAME}) diff --git a/uboost_coro/CMakeLists.txt b/uboost_coro/CMakeLists.txt index 3030273e5a2e..f846ea34c425 100644 --- a/uboost_coro/CMakeLists.txt +++ b/uboost_coro/CMakeLists.txt @@ -72,11 +72,12 @@ if(UBOOST_CORO_USE_UCONTEXT) ) endif() -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) target_include_directories(${PROJECT_NAME} PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include + $ + $ ) target_compile_options(${PROJECT_NAME} PRIVATE "-w") # no warnings @@ -104,3 +105,5 @@ endif() find_package(Boost REQUIRED) target_link_libraries(${PROJECT_NAME} PUBLIC Boost::boost) + +userver_install_library(TARGETS ${PROJECT_NAME}) diff --git a/universal/CMakeLists.txt b/universal/CMakeLists.txt index d1fbbe95485f..b39aa002b124 100644 --- a/universal/CMakeLists.txt +++ b/universal/CMakeLists.txt @@ -100,7 +100,7 @@ else() include(SetupCCTZ) endif() -add_library(${PROJECT_NAME} STATIC ${SOURCES}) +userver_add_library(${PROJECT_NAME} SOURCES ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX CXX_STANDARD 17 @@ -138,12 +138,13 @@ target_compile_definitions(${PROJECT_NAME} PRIVATE OPENSSL_SUPPRESS_DEPRECATED=) # https://bugs.llvm.org/show_bug.cgi?id=16404 if (USERVER_SANITIZE AND NOT CMAKE_BUILD_TYPE MATCHES "^Rel") - target_link_libraries(${PROJECT_NAME} PUBLIC userver-compiler-rt-parts) + userver_target_link_libraries(${PROJECT_NAME} STATIC_PRIVATE userver-compiler-rt-parts) endif() -target_link_libraries(${PROJECT_NAME} +userver_target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads + STATIC_PRIVATE sanitize-target PRIVATE Boost::filesystem @@ -182,7 +183,9 @@ set(UNIVERSAL_PUBLIC_INCLUDE_DIRS ) target_include_directories(${PROJECT_NAME} PUBLIC - ${UNIVERSAL_PUBLIC_INCLUDE_DIRS} + $ + $ + $ ) target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE From b39fa2540ee7404599272975eba4ce08a2f344db Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Tue, 4 Jul 2023 12:49:44 +0000 Subject: [PATCH 02/10] adapt rpm spec for build static libraries --- packages/rpm/userver.spec | 6 +++--- third_party/compiler-rt/CMakeLists.txt | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/rpm/userver.spec b/packages/rpm/userver.spec index 80d6d6e2a8d2..f49c0fa78d80 100644 --- a/packages/rpm/userver.spec +++ b/packages/rpm/userver.spec @@ -121,11 +121,11 @@ popd %clean rm -rf %{buildroot} -%files -n %{name} +#%files -n %{name} #%{_bindir}/userver/ -%{_libdir}/lib*.so +#%{_libdir}/lib*.so %files -n %{name}-devel %{_includedir}/ -#{_libdir}/lib*.a +%{_libdir}/lib*.a %{_libdir}/cmake/userver/*.cmake diff --git a/third_party/compiler-rt/CMakeLists.txt b/third_party/compiler-rt/CMakeLists.txt index 77b35045c34d..470900b605fa 100644 --- a/third_party/compiler-rt/CMakeLists.txt +++ b/third_party/compiler-rt/CMakeLists.txt @@ -5,8 +5,14 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_library(${PROJECT_NAME} STATIC ${SOURCES}) # workaround for INSTALL dependent libraries without install this static library - set_target_properties (${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) - + if(USERVER_BUILD_SHARED_LIBS) + set_target_properties (${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) + else() + userver_export(TARGETS userver-compiler-rt-parts) + endif() else() add_library(${PROJECT_NAME} INTERFACE) + if(NOT USERVER_BUILD_SHARED_LIBS) + userver_export(TARGETS userver-compiler-rt-parts) + endif() endif() From a1f5b256908798f181f2ae3bdb06ef715d81cf0c Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Tue, 4 Jul 2023 12:59:56 +0000 Subject: [PATCH 03/10] adapt rpm spec for build static libraries --- packages/rpm/userver.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rpm/userver.spec b/packages/rpm/userver.spec index f49c0fa78d80..b6ec40fc8717 100644 --- a/packages/rpm/userver.spec +++ b/packages/rpm/userver.spec @@ -88,7 +88,7 @@ export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.6/site-packages # -DUSERVER_LTO:BOOL=OFF as workaround for dwz crash cmake -DUSERVER_FEATURE_PATCH_LIBPQ=0 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSERVER_FEATURE_STACKTRACE=0 \ -DUSERVER_PYTHON_PATH=/usr/bin/python3 \ - -DUSERVER_BUILD_SHARED_LIBS:BOOL=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr \ + -DUSERVER_BUILD_SHARED_LIBS:BOOL=OFF -DCMAKE_INSTALL_PREFIX:PATH=/usr \ -DCMAKE_INSTALL_LIBDIR=lib64 -DUSERVER_DOWNLOAD_PACKAGES:BOOL=OFF \ -DUSERVER_LTO:BOOL=OFF \ -DUSERVER_GEN_GDB_DEBUGINFO:BOOL=ON \ From c3d33087b0c674d1480d029035637043be3fc607 Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Tue, 4 Jul 2023 14:01:38 +0000 Subject: [PATCH 04/10] adapt rpm spec for build static libraries --- packages/rpm/userver.spec | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/rpm/userver.spec b/packages/rpm/userver.spec index b6ec40fc8717..b803206a9342 100644 --- a/packages/rpm/userver.spec +++ b/packages/rpm/userver.spec @@ -1,3 +1,5 @@ +%global debug_package %{nil} + %define __boost_suffix 176 Name: userver @@ -127,5 +129,7 @@ rm -rf %{buildroot} %files -n %{name}-devel %{_includedir}/ + +%files -n %{name}-static %{_libdir}/lib*.a %{_libdir}/cmake/userver/*.cmake From 2b5ae61c2022e8bc7319ebbb31e8488d0dd13776 Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Tue, 4 Jul 2023 14:03:58 +0000 Subject: [PATCH 05/10] adapt rpm spec for build static libraries --- packages/rpm/userver.spec | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/rpm/userver.spec b/packages/rpm/userver.spec index b803206a9342..398619fa711d 100644 --- a/packages/rpm/userver.spec +++ b/packages/rpm/userver.spec @@ -78,6 +78,27 @@ Requires: grpc-plugins >= 1.48.1 Open source asynchronous framework with a rich set of abstractions for fast and comfortable creation of C++ microservices, services and utilities. +%package -n %{name}-static +Summary: userver framework static libraries +Group: Development/Libraries/C and C++ +Requires: %{name} = %{version} +Requires: cctz-devel gtest-devel spdlog-devel +Requires: libnghttp2-devel libidn2-devel brotli-devel c-ares-devel +Requires: libcurl-devel libbson-devel +Requires: mongo-c-driver-libs mongo-c-driver +Requires: fmt-devel +Requires: libssh-devel +Requires: hiredis-devel +Requires: clickhouse-cpp +Requires: amqp-cpp +Requires: protobuf-devel >= 3.21.9 +Requires: grpc-devel >= 1.48.1 +Requires: grpc-plugins >= 1.48.1 + +%description -n %{name}-static + +Open source asynchronous framework with a rich set of abstractions for fast and comfortable creation of C++ microservices, services and utilities. + %prep %setup -q -n userver-%{_version} From b08f4ffb69fcc1db086b70c51cd59ddf7cc8b34e Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Tue, 4 Jul 2023 14:16:23 +0000 Subject: [PATCH 06/10] adapt rpm spec for build static libraries --- packages/rpm/userver.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rpm/userver.spec b/packages/rpm/userver.spec index 398619fa711d..9a2ad514f437 100644 --- a/packages/rpm/userver.spec +++ b/packages/rpm/userver.spec @@ -60,7 +60,7 @@ Open source asynchronous framework with a rich set of abstractions for fast and %package -n %{name}-devel Summary: userver framework Group: Development/Libraries/C and C++ -Requires: %{name} = %{version} +#Requires: %{name} = %{version} Requires: cctz-devel gtest-devel spdlog-devel Requires: libnghttp2-devel libidn2-devel brotli-devel c-ares-devel Requires: libcurl-devel libbson-devel @@ -81,7 +81,7 @@ Open source asynchronous framework with a rich set of abstractions for fast and %package -n %{name}-static Summary: userver framework static libraries Group: Development/Libraries/C and C++ -Requires: %{name} = %{version} +#Requires: %{name} = %{version} Requires: cctz-devel gtest-devel spdlog-devel Requires: libnghttp2-devel libidn2-devel brotli-devel c-ares-devel Requires: libcurl-devel libbson-devel From 24f3ed6ae86f9951464e0966d8a4749bde87396f Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 29 Dec 2023 09:12:42 +0000 Subject: [PATCH 07/10] Http integration with cobrazz --- cmake/RequireDWCAS.cmake | 6 +- .../components/loggable_component_base.hpp | 1 + .../userver/server/handlers/handler_base.hpp | 4 ++ .../server/handlers/http_handler_base.hpp | 9 +++ core/include/userver/server/server.hpp | 6 ++ .../components/loggable_component_base.cpp | 4 ++ core/src/server/handlers/handler_base.cpp | 8 +++ .../src/server/handlers/http_handler_base.cpp | 63 ++++++++++++++++++ core/src/server/http/http_request_handler.cpp | 16 +++++ core/src/server/http/http_request_handler.hpp | 6 ++ core/src/server/server.cpp | 66 +++++++++++++++++++ 11 files changed, 186 insertions(+), 3 deletions(-) diff --git a/cmake/RequireDWCAS.cmake b/cmake/RequireDWCAS.cmake index 584c3a42dc63..b00ebb39f5cc 100644 --- a/cmake/RequireDWCAS.cmake +++ b/cmake/RequireDWCAS.cmake @@ -45,9 +45,9 @@ if(USERVER_IMPL_DWCAS_CHECKED) return() endif() -if(NOT MACOS AND NOT "${CMAKE_SYSTEM}" MATCHES "BSD") - list(APPEND TEST_LIBRARIES "atomic") -endif() +#if(NOT MACOS AND NOT "${CMAKE_SYSTEM}" MATCHES "BSD") +# list(APPEND TEST_LIBRARIES "atomic") +#endif() # Make try_run honor parent CMAKE_CXX_STANDARD cmake_policy(SET CMP0067 NEW) diff --git a/core/include/userver/components/loggable_component_base.hpp b/core/include/userver/components/loggable_component_base.hpp index 9d7c332a34ed..464563b03efe 100644 --- a/core/include/userver/components/loggable_component_base.hpp +++ b/core/include/userver/components/loggable_component_base.hpp @@ -19,6 +19,7 @@ namespace components { class LoggableComponentBase : public impl::ComponentBase { public: LoggableComponentBase(const ComponentConfig&, const ComponentContext&); + LoggableComponentBase(); LoggableComponentBase(LoggableComponentBase&&) = delete; LoggableComponentBase(const LoggableComponentBase&) = delete; diff --git a/core/include/userver/server/handlers/handler_base.hpp b/core/include/userver/server/handlers/handler_base.hpp index 8c5d4f5b8d6c..e02afc572f72 100644 --- a/core/include/userver/server/handlers/handler_base.hpp +++ b/core/include/userver/server/handlers/handler_base.hpp @@ -48,6 +48,10 @@ class HandlerBase : public components::LoggableComponentBase { HandlerBase(const components::ComponentConfig& config, const components::ComponentContext& component_context, bool is_monitor = false); + + HandlerBase(HandlerConfig config, + const bool is_monitor = false); + ~HandlerBase() noexcept override = default; /// Parses request, executes processing routines, and fills response diff --git a/core/include/userver/server/handlers/http_handler_base.hpp b/core/include/userver/server/handlers/http_handler_base.hpp index a2e91e5047a6..a0d7d3097bd6 100644 --- a/core/include/userver/server/handlers/http_handler_base.hpp +++ b/core/include/userver/server/handlers/http_handler_base.hpp @@ -63,6 +63,15 @@ class HttpHandlerBase : public HandlerBase { const components::ComponentContext& component_context, bool is_monitor = false); + HttpHandlerBase(const std::string& handler_name, + const HandlerConfig& handler_config, + const dynamic_config::Source& dynamic_config_source, + const tracing::TracingManagerBase& tracing_manager, + utils::statistics::Storage& statistics_storage, + const bool is_body_streamed = false, + std::optional log_level = {}, + const bool is_monitor = false); + ~HttpHandlerBase() override; void HandleRequest(request::RequestBase& request, diff --git a/core/include/userver/server/server.hpp b/core/include/userver/server/server.hpp index e084a054414f..3fd2b3270f49 100644 --- a/core/include/userver/server/server.hpp +++ b/core/include/userver/server/server.hpp @@ -29,6 +29,12 @@ class Server final { public: Server(ServerConfig config, const components::ComponentContext& component_context); + + Server(ServerConfig config, + engine::TaskProcessor& task_processor, + const utils::statistics::MetricsStoragePtr& metrics_storage, + const dynamic_config::Source& dynamic_config_source); + ~Server(); const ServerConfig& GetConfig() const; diff --git a/core/src/components/loggable_component_base.cpp b/core/src/components/loggable_component_base.cpp index 504f6d536793..981b0ad7a12e 100644 --- a/core/src/components/loggable_component_base.cpp +++ b/core/src/components/loggable_component_base.cpp @@ -15,6 +15,10 @@ LoggableComponentBase::LoggableComponentBase( component_context.FindComponent(); } +LoggableComponentBase::LoggableComponentBase() +{ +} + yaml_config::Schema LoggableComponentBase::GetStaticConfigSchema() { auto schema = impl::ComponentBase::GetStaticConfigSchema(); schema.UpdateDescription( diff --git a/core/src/server/handlers/handler_base.cpp b/core/src/server/handlers/handler_base.cpp index f07d9566ed78..b9cd51b2da3b 100644 --- a/core/src/server/handlers/handler_base.cpp +++ b/core/src/server/handlers/handler_base.cpp @@ -20,6 +20,14 @@ HandlerBase::HandlerBase(const components::ComponentConfig& config, context.FindComponent().GetServer().GetConfig(), is_monitor_)) {} +HandlerBase::HandlerBase(HandlerConfig config, + const bool is_monitor) + : LoggableComponentBase(), + is_monitor_(is_monitor), + config_(config) +{ +} + const HandlerConfig& HandlerBase::GetConfig() const { return config_; } yaml_config::Schema HandlerBase::GetStaticConfigSchema() { diff --git a/core/src/server/handlers/http_handler_base.cpp b/core/src/server/handlers/http_handler_base.cpp index ef7e61331565..4b292315b2cb 100644 --- a/core/src/server/handlers/http_handler_base.cpp +++ b/core/src/server/handlers/http_handler_base.cpp @@ -545,6 +545,69 @@ HttpHandlerBase::HttpHandlerBase(const components::ComponentConfig& config, .set_response_server_hostname); } +HttpHandlerBase::HttpHandlerBase(const std::string& handler_name, + const HandlerConfig& handler_config, + const dynamic_config::Source& dynamic_config_source, + const tracing::TracingManagerBase& tracing_manager, + utils::statistics::Storage& statistics_storage, + const bool is_body_streamed, + std::optional log_level, + const bool is_monitor) + : HandlerBase(handler_config, is_monitor), + config_source_(dynamic_config_source), + allowed_methods_(InitAllowedMethods(GetConfig())), + handler_name_(handler_name), + tracing_manager_(tracing_manager), + handler_statistics_(std::make_unique()), + request_statistics_(std::make_unique()), + auth_checkers_(), + log_level_(log_level), + rate_limit_(utils::TokenBucket::MakeUnbounded()), + is_body_streamed_(is_body_streamed) +{ + if (allowed_methods_.empty()) { + LOG_WARNING() << "empty allowed methods list in " << handler_name; + } + + if (GetConfig().max_requests_per_second) { + const auto max_rps = *GetConfig().max_requests_per_second; + UASSERT_MSG( + max_rps > 0, + "max_requests_per_second option was not verified in config parsing"); + rate_limit_.SetMaxSize(max_rps); + rate_limit_.SetRefillPolicy( + {1, utils::TokenBucket::Duration{std::chrono::seconds(1)} / max_rps}); + } + + std::vector labels{ + {"http_handler", handler_name}, + }; + + auto prefix = std::visit( + utils::Overloaded{[&](const std::string& path) { + labels.emplace_back( + "http_path", utils::graphite::EscapeName(path)); + return std::string{"http"}; + }, + [](FallbackHandler fallback) { + return "http.by-fallback." + ToString(fallback); + }}, + GetConfig().path); + + statistics_holder_ = statistics_storage.RegisterWriter( + std::move(prefix), + [this](utils::statistics::Writer& result) { + FormatStatistics(result["handler"], *handler_statistics_); + if constexpr (kIncludeServerHttpMetrics) { + FormatStatistics(result["request"], *request_statistics_); + } + }, + std::move(labels)); + + set_response_server_hostname_ = + GetConfig().set_response_server_hostname.value_or(false); +} + HttpHandlerBase::~HttpHandlerBase() { statistics_holder_.Unregister(); } void HttpHandlerBase::HandleRequestStream( diff --git a/core/src/server/http/http_request_handler.cpp b/core/src/server/http/http_request_handler.cpp index 2324ebf7ee7d..5557bd27e006 100644 --- a/core/src/server/http/http_request_handler.cpp +++ b/core/src/server/http/http_request_handler.cpp @@ -75,6 +75,22 @@ HttpRequestHandler::HttpRequestHandler( } } +HttpRequestHandler::HttpRequestHandler( + const utils::statistics::MetricsStoragePtr& metrics, + const dynamic_config::Source& dynamic_config_source, + const bool is_monitor, + const std::string& server_name) + : add_handler_disabled_(false), + is_monitor_(is_monitor), + server_name_(std::move(server_name)), + rate_limit_(utils::TokenBucket::MakeUnbounded()), + metrics_(metrics), + config_source_(dynamic_config_source) +{ + LOG_INFO() << "Access log is disabled"; + LOG_INFO() << "Access_tskv log is disabled"; +} + namespace { struct CcCustomStatus final { diff --git a/core/src/server/http/http_request_handler.hpp b/core/src/server/http/http_request_handler.hpp index 62bbd615922b..a0a04b04b580 100644 --- a/core/src/server/http/http_request_handler.hpp +++ b/core/src/server/http/http_request_handler.hpp @@ -26,6 +26,12 @@ class HttpRequestHandler final : public RequestHandlerBase { const std::optional& logger_access_tskv_component, bool is_monitor, std::string server_name); + HttpRequestHandler( + const utils::statistics::MetricsStoragePtr& metrics_storage, + const dynamic_config::Source& dynamic_config_source, + const bool is_monitor, + const std::string& server_name); + using NewRequestHook = std::function)>; void SetNewRequestHook(NewRequestHook hook); diff --git a/core/src/server/server.cpp b/core/src/server/server.cpp index d80bb5f14344..f7242f63f1f8 100644 --- a/core/src/server/server.cpp +++ b/core/src/server/server.cpp @@ -31,6 +31,13 @@ struct PortInfo final { const components::ComponentContext& component_context, bool is_monitor); + void Init(const net::ListenerConfig& listener_config, + engine::TaskProcessor& task_processor, + const utils::statistics::MetricsStoragePtr& metrics_storage, + const dynamic_config::Source& dynamic_config_source, + const std::string& server_name, + bool is_monitor); + void Start(); void Stop(); @@ -69,6 +76,29 @@ void PortInfo::Init(const ServerConfig& config, } } +void PortInfo::Init(const net::ListenerConfig& listener_config, + engine::TaskProcessor& task_processor, + const utils::statistics::MetricsStoragePtr& metrics_storage, + const dynamic_config::Source& dynamic_config_source, + const std::string& server_name, + bool is_monitor) { + LOG_INFO() << "Creating listener" << (is_monitor ? " (monitor)" : ""); + + request_handler_.emplace(metrics_storage, dynamic_config_source, is_monitor, server_name); + + endpoint_info_ = + std::make_shared(listener_config, *request_handler_); + + const auto& event_thread_pool = task_processor.EventThreadPool(); + size_t listener_shards = listener_config.shards ? *listener_config.shards + : event_thread_pool.GetSize(); + + listeners_.reserve(listener_shards); + while (listener_shards--) { + listeners_.emplace_back(endpoint_info_, task_processor, data_accounter_); + } +} + void PortInfo::Start() { UASSERT(request_handler_); request_handler_->DisableAddHandler(); @@ -111,6 +141,12 @@ class ServerImpl final { public: ServerImpl(ServerConfig config, const components::ComponentContext& component_context); + + ServerImpl(const ServerConfig& config, + engine::TaskProcessor& task_processor, + const utils::statistics::MetricsStoragePtr& metrics_storage, + const dynamic_config::Source& dynamic_config_source); + ~ServerImpl(); void StartPortInfos(); @@ -164,6 +200,29 @@ ServerImpl::ServerImpl(ServerConfig config, LOG_INFO() << "Server is created"; } +ServerImpl::ServerImpl(const ServerConfig& config, + engine::TaskProcessor& task_processor, + const utils::statistics::MetricsStoragePtr& metrics_storage, + const dynamic_config::Source& dynamic_config_source) + : config_(std::move(config)) { + LOG_INFO() << "Creating server"; + + main_port_info_.Init(config_.listener, task_processor, + metrics_storage, dynamic_config_source, + config.server_name, false); + if (config_.max_response_size_in_flight) { + main_port_info_.data_accounter_.SetMaxLevel( + *config_.max_response_size_in_flight); + } + if (config_.monitor_listener) { + monitor_port_info_.Init(*config_.monitor_listener, + task_processor, metrics_storage, + dynamic_config_source, config.server_name, true); + } + + LOG_INFO() << "Server is created"; +} + ServerImpl::~ServerImpl() { Stop(); } void ServerImpl::StartPortInfos() { @@ -307,6 +366,13 @@ Server::Server(ServerConfig config, : pimpl( std::make_unique(std::move(config), component_context)) {} +Server::Server(ServerConfig config, + engine::TaskProcessor& task_processor, + const utils::statistics::MetricsStoragePtr& metrics_storage, + const dynamic_config::Source& dynamic_config_source) + : pimpl( + std::make_unique(std::move(config), task_processor, metrics_storage, dynamic_config_source)) {} + Server::~Server() = default; const ServerConfig& Server::GetConfig() const { From 9d0fa8639ca0b74b19771f84043b54243cccc449 Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Mon, 22 Jan 2024 14:02:09 +0000 Subject: [PATCH 08/10] added install of required files --- core/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 148b2606b9ff..8380e6b6c3bc 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -322,6 +322,15 @@ install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/engine/task/task_queue.hpp" DESTINATION "include/userver/engine/task/") +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/server/server_config.hpp" + DESTINATION "include/userver/server/") + +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/server/net/listener_config.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/server/net/connection_config.hpp" + DESTINATION "include/userver/server/net/") + install(DIRECTORY "${USERVER_THIRD_PARTY_DIRS}/moodycamel/include/moodycamel" DESTINATION "include/userver" FILES_MATCHING From 220cf6a1d84add64750d0f0c9400adee9cd7fbd6 Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Thu, 25 Jan 2024 09:06:27 +0000 Subject: [PATCH 09/10] fix double include error for cmake --- cmake/userverConfig.cmake.in | 41 +++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/cmake/userverConfig.cmake.in b/cmake/userverConfig.cmake.in index 57b21fa3ff7d..3d7aea707a91 100644 --- a/cmake/userverConfig.cmake.in +++ b/cmake/userverConfig.cmake.in @@ -10,31 +10,48 @@ find_package(Iconv REQUIRED) find_package(ZLIB REQUIRED) find_package(yaml-cpp REQUIRED) -add_library(libyamlcpp INTERFACE) -target_link_libraries(libyamlcpp INTERFACE ${YAML_CPP_LIBRARIES}) + +if(NOT TARGET libyamlcpp) + add_library(libyamlcpp INTERFACE) + target_link_libraries(libyamlcpp INTERFACE ${YAML_CPP_LIBRARIES}) +endif() pkg_check_modules (JEMALLOC jemalloc) pkg_search_module(JEMALLOC REQUIRED jemalloc) -add_library(Jemalloc INTERFACE) -target_link_libraries(Jemalloc INTERFACE ${JEMALLOC_LIBRARIES}) + +if(NOT TARGET Jemalloc) + add_library(Jemalloc INTERFACE) + target_link_libraries(Jemalloc INTERFACE ${JEMALLOC_LIBRARIES}) +endif() pkg_check_modules (LIBCARES libcares) pkg_search_module(LIBCARES REQUIRED libcares) -add_library(c-ares INTERFACE) -target_link_libraries(c-ares INTERFACE ${LIBCARES_LIBRARIES}) + +if(NOT TARGET c-ares) + add_library(c-ares INTERFACE) + target_link_libraries(c-ares INTERFACE ${LIBCARES_LIBRARIES}) +endif() pkg_check_modules (LIBCRYPTOPP cryptopp) pkg_search_module(LIBCRYPTOPP REQUIRED cryptopp) -add_library(CryptoPP INTERFACE) -target_link_libraries(CryptoPP INTERFACE ${LIBCRYPTOPP_LIBRARIES}) + +if(NOT TARGET CryptoPP) + add_library(CryptoPP INTERFACE) + target_link_libraries(CryptoPP INTERFACE ${LIBCRYPTOPP_LIBRARIES}) +endif() pkg_check_modules (LIBEV libevent) pkg_search_module(LIBEV REQUIRED libevent) -add_library(LibEv INTERFACE) -target_link_libraries(LibEv INTERFACE ${LIBEV_LIBRARIES}) -add_library(Http_Parser INTERFACE) -target_link_libraries(Http_Parser INTERFACE http_parser) +if(NOT TARGET LibEv) + add_library(LibEv INTERFACE) + target_link_libraries(LibEv INTERFACE ${LIBEV_LIBRARIES}) +endif() + +if(NOT TARGET Http_Parser) + add_library(Http_Parser INTERFACE) + target_link_libraries(Http_Parser INTERFACE http_parser) +endif() set(CMAKE_CONFIG_TARGETS_FILE @CMAKE_CONFIG_TARGETS_FILE@) include("${CMAKE_CURRENT_LIST_DIR}/${CMAKE_CONFIG_TARGETS_FILE}") From 846405e2e6865ff80341ba750ff709ec9e245a55 Mon Sep 17 00:00:00 2001 From: Yuri Kuznecov Date: Mon, 17 Jun 2024 09:01:43 +0000 Subject: [PATCH 10/10] Replace BLOCK_SIZE moodycamel constant for solve conflict with unix system BLOCK_SIZE macro --- third_party/moodycamel/README.md | 8 +- .../moodycamel/blockingconcurrentqueue.h | 4 +- .../include/moodycamel/concurrentqueue.h | 158 +++++++++--------- 3 files changed, 85 insertions(+), 85 deletions(-) diff --git a/third_party/moodycamel/README.md b/third_party/moodycamel/README.md index 7e1f6b8dd133..7b680b3bf443 100644 --- a/third_party/moodycamel/README.md +++ b/third_party/moodycamel/README.md @@ -289,15 +289,15 @@ integer division (in order for `ceil()` to work). For explicit producers (using tokens to enqueue): - (ceil(N / BLOCK_SIZE) + 1) * MAX_NUM_PRODUCERS * BLOCK_SIZE + (ceil(N / BLOCK_SIZE_QUEUE) + 1) * MAX_NUM_PRODUCERS * BLOCK_SIZE_QUEUE For implicit producers (no tokens): - (ceil(N / BLOCK_SIZE) - 1 + 2 * MAX_NUM_PRODUCERS) * BLOCK_SIZE + (ceil(N / BLOCK_SIZE_QUEUE) - 1 + 2 * MAX_NUM_PRODUCERS) * BLOCK_SIZE_QUEUE When using mixed producer types: - ((ceil(N / BLOCK_SIZE) - 1) * (MAX_EXPLICIT_PRODUCERS + 1) + 2 * (MAX_IMPLICIT_PRODUCERS + MAX_EXPLICIT_PRODUCERS)) * BLOCK_SIZE + ((ceil(N / BLOCK_SIZE_QUEUE) - 1) * (MAX_EXPLICIT_PRODUCERS + 1) + 2 * (MAX_IMPLICIT_PRODUCERS + MAX_EXPLICIT_PRODUCERS)) * BLOCK_SIZE_QUEUE If these formulas seem rather inconvenient, you can use the constructor overload that accepts the minimum number of elements (`N`) and the maximum number of explicit and implicit producers directly, and let it do the @@ -345,7 +345,7 @@ and override only the values you wish to change. Example: struct MyTraits : public moodycamel::ConcurrentQueueDefaultTraits { - static const size_t BLOCK_SIZE = 256; // Use bigger blocks + static const size_t BLOCK_SIZE_QUEUE = 256; // Use bigger blocks }; moodycamel::ConcurrentQueue q; diff --git a/third_party/moodycamel/include/moodycamel/blockingconcurrentqueue.h b/third_party/moodycamel/include/moodycamel/blockingconcurrentqueue.h index 66579b6caf67..29e9463fc7a0 100644 --- a/third_party/moodycamel/include/moodycamel/blockingconcurrentqueue.h +++ b/third_party/moodycamel/include/moodycamel/blockingconcurrentqueue.h @@ -36,7 +36,7 @@ class BlockingConcurrentQueue typedef typename ConcurrentQueue::size_t size_t; typedef typename std::make_signed::type ssize_t; - static const size_t BLOCK_SIZE = ConcurrentQueue::BLOCK_SIZE; + static const size_t BLOCK_SIZE_QUEUE = ConcurrentQueue::BLOCK_SIZE_QUEUE; static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = ConcurrentQueue::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD; static const size_t EXPLICIT_INITIAL_INDEX_SIZE = ConcurrentQueue::EXPLICIT_INITIAL_INDEX_SIZE; static const size_t IMPLICIT_INITIAL_INDEX_SIZE = ConcurrentQueue::IMPLICIT_INITIAL_INDEX_SIZE; @@ -55,7 +55,7 @@ class BlockingConcurrentQueue // queue is fully constructed before it starts being used by other threads (this // includes making the memory effects of construction visible, possibly with a // memory barrier). - explicit BlockingConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE) + explicit BlockingConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE_QUEUE) : inner(capacity), sema(create(0, (int)Traits::MAX_SEMA_SPINS), &BlockingConcurrentQueue::template destroy) { assert(reinterpret_cast((BlockingConcurrentQueue*)1) == &((BlockingConcurrentQueue*)1)->inner && "BlockingConcurrentQueue must have ConcurrentQueue as its first member"); diff --git a/third_party/moodycamel/include/moodycamel/concurrentqueue.h b/third_party/moodycamel/include/moodycamel/concurrentqueue.h index ff3156f73706..fc77ade7298b 100644 --- a/third_party/moodycamel/include/moodycamel/concurrentqueue.h +++ b/third_party/moodycamel/include/moodycamel/concurrentqueue.h @@ -339,7 +339,7 @@ struct ConcurrentQueueDefaultTraits // but many producers, a smaller block size should be favoured. For few producers // and/or many elements, a larger block size is preferred. A sane default // is provided. Must be a power of 2. - static const size_t BLOCK_SIZE = 32; + static const size_t BLOCK_SIZE_QUEUE = 32; // For explicit producers (i.e. when using a producer token), the block is // checked for being empty by iterating through a list of flags, one per element. @@ -752,7 +752,7 @@ class ConcurrentQueue typedef typename Traits::index_t index_t; typedef typename Traits::size_t size_t; - static const size_t BLOCK_SIZE = static_cast(Traits::BLOCK_SIZE); + static const size_t BLOCK_SIZE_QUEUE = static_cast(Traits::BLOCK_SIZE_QUEUE); static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = static_cast(Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD); static const size_t EXPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::EXPLICIT_INITIAL_INDEX_SIZE); static const size_t IMPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::IMPLICIT_INITIAL_INDEX_SIZE); @@ -763,7 +763,7 @@ class ConcurrentQueue #pragma warning(disable: 4307) // + integral constant overflow (that's what the ternary expression is for!) #pragma warning(disable: 4309) // static_cast: Truncation of constant value #endif - static const size_t MAX_SUBQUEUE_SIZE = (details::const_numeric_max::value - static_cast(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE) ? details::const_numeric_max::value : ((static_cast(Traits::MAX_SUBQUEUE_SIZE) + (BLOCK_SIZE - 1)) / BLOCK_SIZE * BLOCK_SIZE); + static const size_t MAX_SUBQUEUE_SIZE = (details::const_numeric_max::value - static_cast(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE_QUEUE) ? details::const_numeric_max::value : ((static_cast(Traits::MAX_SUBQUEUE_SIZE) + (BLOCK_SIZE_QUEUE - 1)) / BLOCK_SIZE_QUEUE * BLOCK_SIZE_QUEUE); #ifdef _MSC_VER #pragma warning(pop) #endif @@ -771,7 +771,7 @@ class ConcurrentQueue static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::size_t must be an unsigned integral type"); static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::index_t must be an unsigned integral type"); static_assert(sizeof(index_t) >= sizeof(size_t), "Traits::index_t must be at least as wide as Traits::size_t"); - static_assert((BLOCK_SIZE > 1) && !(BLOCK_SIZE & (BLOCK_SIZE - 1)), "Traits::BLOCK_SIZE must be a power of 2 (and at least 2)"); + static_assert((BLOCK_SIZE_QUEUE > 1) && !(BLOCK_SIZE_QUEUE & (BLOCK_SIZE_QUEUE - 1)), "Traits::BLOCK_SIZE_QUEUE must be a power of 2 (and at least 2)"); static_assert((EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD > 1) && !(EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD & (EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD - 1)), "Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD must be a power of 2 (and greater than 1)"); static_assert((EXPLICIT_INITIAL_INDEX_SIZE > 1) && !(EXPLICIT_INITIAL_INDEX_SIZE & (EXPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::EXPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); static_assert((IMPLICIT_INITIAL_INDEX_SIZE > 1) && !(IMPLICIT_INITIAL_INDEX_SIZE & (IMPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::IMPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); @@ -789,7 +789,7 @@ class ConcurrentQueue // queue is fully constructed before it starts being used by other threads (this // includes making the memory effects of construction visible, possibly with a // memory barrier). - explicit ConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE) + explicit ConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE_QUEUE) : producerListTail(nullptr), producerCount(0), initialBlockPoolIndex(0), @@ -798,7 +798,7 @@ class ConcurrentQueue { implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); populate_initial_implicit_producer_hash(); - populate_initial_block_list(capacity / BLOCK_SIZE + ((capacity & (BLOCK_SIZE - 1)) == 0 ? 0 : 1)); + populate_initial_block_list(capacity / BLOCK_SIZE_QUEUE + ((capacity & (BLOCK_SIZE_QUEUE - 1)) == 0 ? 0 : 1)); #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG // Track all the producers using a fully-resolved typed list for @@ -822,7 +822,7 @@ class ConcurrentQueue { implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); populate_initial_implicit_producer_hash(); - size_t blocks = (((minCapacity + BLOCK_SIZE - 1) / BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) + 2 * (maxExplicitProducers + maxImplicitProducers); + size_t blocks = (((minCapacity + BLOCK_SIZE_QUEUE - 1) / BLOCK_SIZE_QUEUE) - 1) * (maxExplicitProducers + 1) + 2 * (maxExplicitProducers + maxImplicitProducers); populate_initial_block_list(blocks); #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG @@ -1548,9 +1548,9 @@ class ConcurrentQueue template inline bool is_empty() const { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE_QUEUE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { // Check flags - for (size_t i = 0; i < BLOCK_SIZE; ++i) { + for (size_t i = 0; i < BLOCK_SIZE_QUEUE; ++i) { if (!emptyFlags[i].load(std::memory_order_relaxed)) { return false; } @@ -1562,11 +1562,11 @@ class ConcurrentQueue } else { // Check counter - if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE) { + if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE_QUEUE) { std::atomic_thread_fence(std::memory_order_acquire); return true; } - assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE); + assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE_QUEUE); return false; } } @@ -1575,17 +1575,17 @@ class ConcurrentQueue template inline bool set_empty(MOODYCAMEL_MAYBE_UNUSED index_t i) { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE_QUEUE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { // Set flag - assert(!emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].load(std::memory_order_relaxed)); - emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].store(true, std::memory_order_release); + assert(!emptyFlags[BLOCK_SIZE_QUEUE - 1 - static_cast(i & static_cast(BLOCK_SIZE_QUEUE - 1))].load(std::memory_order_relaxed)); + emptyFlags[BLOCK_SIZE_QUEUE - 1 - static_cast(i & static_cast(BLOCK_SIZE_QUEUE - 1))].store(true, std::memory_order_release); return false; } else { // Increment counter auto prevVal = elementsCompletelyDequeued.fetch_add(1, std::memory_order_release); - assert(prevVal < BLOCK_SIZE); - return prevVal == BLOCK_SIZE - 1; + assert(prevVal < BLOCK_SIZE_QUEUE); + return prevVal == BLOCK_SIZE_QUEUE - 1; } } @@ -1594,10 +1594,10 @@ class ConcurrentQueue template inline bool set_many_empty(MOODYCAMEL_MAYBE_UNUSED index_t i, size_t count) { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE_QUEUE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { // Set flags std::atomic_thread_fence(std::memory_order_release); - i = BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1)) - count + 1; + i = BLOCK_SIZE_QUEUE - 1 - static_cast(i & static_cast(BLOCK_SIZE_QUEUE - 1)) - count + 1; for (size_t j = 0; j != count; ++j) { assert(!emptyFlags[i + j].load(std::memory_order_relaxed)); emptyFlags[i + j].store(true, std::memory_order_relaxed); @@ -1607,32 +1607,32 @@ class ConcurrentQueue else { // Increment counter auto prevVal = elementsCompletelyDequeued.fetch_add(count, std::memory_order_release); - assert(prevVal + count <= BLOCK_SIZE); - return prevVal + count == BLOCK_SIZE; + assert(prevVal + count <= BLOCK_SIZE_QUEUE); + return prevVal + count == BLOCK_SIZE_QUEUE; } } template inline void set_all_empty() { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE_QUEUE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { // Set all flags - for (size_t i = 0; i != BLOCK_SIZE; ++i) { + for (size_t i = 0; i != BLOCK_SIZE_QUEUE; ++i) { emptyFlags[i].store(true, std::memory_order_relaxed); } } else { // Reset counter - elementsCompletelyDequeued.store(BLOCK_SIZE, std::memory_order_relaxed); + elementsCompletelyDequeued.store(BLOCK_SIZE_QUEUE, std::memory_order_relaxed); } } template inline void reset_empty() { - MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE_QUEUE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { // Reset flags - for (size_t i = 0; i != BLOCK_SIZE; ++i) { + for (size_t i = 0; i != BLOCK_SIZE_QUEUE; ++i) { emptyFlags[i].store(false, std::memory_order_relaxed); } } @@ -1642,16 +1642,16 @@ class ConcurrentQueue } } - inline T* operator[](index_t idx) MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } - inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } + inline T* operator[](index_t idx) MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE_QUEUE - 1)); } + inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE_QUEUE - 1)); } private: static_assert(std::alignment_of::value <= sizeof(T), "The queue does not support types with an alignment greater than their size at this time"); - MOODYCAMEL_ALIGNED_TYPE_LIKE(char[sizeof(T) * BLOCK_SIZE], T) elements; + MOODYCAMEL_ALIGNED_TYPE_LIKE(char[sizeof(T) * BLOCK_SIZE_QUEUE], T) elements; public: Block* next; std::atomic elementsCompletelyDequeued; - std::atomic emptyFlags[BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE : 1]; + std::atomic emptyFlags[BLOCK_SIZE_QUEUE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE_QUEUE : 1]; public: std::atomic freeListRefs; std::atomic freeListNext; @@ -1773,11 +1773,11 @@ class ConcurrentQueue if (this->tailBlock != nullptr) { // Note this means there must be a block index too // First find the block that's partially dequeued, if any Block* halfDequeuedBlock = nullptr; - if ((this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) != 0) { + if ((this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE_QUEUE - 1)) != 0) { // The head's not on a block boundary, meaning a block somewhere is partially dequeued // (or the head block is the tail block and was fully dequeued, but the head/tail are still not on a boundary) size_t i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & (pr_blockIndexSize - 1); - while (details::circular_less_than(pr_blockIndexEntries[i].base + BLOCK_SIZE, this->headIndex.load(std::memory_order_relaxed))) { + while (details::circular_less_than(pr_blockIndexEntries[i].base + BLOCK_SIZE_QUEUE, this->headIndex.load(std::memory_order_relaxed))) { i = (i + 1) & (pr_blockIndexSize - 1); } assert(details::circular_less_than(pr_blockIndexEntries[i].base, this->headIndex.load(std::memory_order_relaxed))); @@ -1794,12 +1794,12 @@ class ConcurrentQueue size_t i = 0; // Offset into block if (block == halfDequeuedBlock) { - i = static_cast(this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); + i = static_cast(this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE_QUEUE - 1)); } // Walk through all the items in the block; if this is the tail block, we need to stop when we reach the tail index - auto lastValidIndex = (this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) == 0 ? BLOCK_SIZE : static_cast(this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); - while (i != BLOCK_SIZE && (block != this->tailBlock || i != lastValidIndex)) { + auto lastValidIndex = (this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0 ? BLOCK_SIZE_QUEUE : static_cast(this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE_QUEUE - 1)); + while (i != BLOCK_SIZE_QUEUE && (block != this->tailBlock || i != lastValidIndex)) { (*block)[i++]->~T(); } } while (block != this->tailBlock); @@ -1835,7 +1835,7 @@ class ConcurrentQueue { index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); index_t newTailIndex = 1 + currentTailIndex; - if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + if ((currentTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0) { // We reached the end of a block, start a new one auto startBlock = this->tailBlock; auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; @@ -1856,8 +1856,8 @@ class ConcurrentQueue // <= to it. auto head = this->headIndex.load(std::memory_order_relaxed); assert(!details::circular_less_than(currentTailIndex, head)); - if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) - || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE_QUEUE) + || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE_QUEUE < currentTailIndex - head))) { // We can't enqueue in another block because there's not enough leeway -- the // tail could surpass the head by the time the block fills up! (Or we'll exceed // the size limit, if the second part of the condition was true.) @@ -1997,8 +1997,8 @@ class ConcurrentQueue // When an index wraps, we need to preserve the sign of the offset when dividing it by the // block size (in order to get a correct signed block count offset in all cases): auto headBase = localBlockIndex->entries[localBlockIndexHead].base; - auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / BLOCK_SIZE); + auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE_QUEUE - 1); + auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / BLOCK_SIZE_QUEUE); auto block = localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block; // Dequeue @@ -2050,13 +2050,13 @@ class ConcurrentQueue Block* firstAllocatedBlock = nullptr; // Figure out how many blocks we'll need to allocate, and do so - size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); - index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1); if (blockBaseDiff > 0) { // Allocate as many blocks as possible from ahead while (blockBaseDiff > 0 && this->tailBlock != nullptr && this->tailBlock->next != firstAllocatedBlock && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); + blockBaseDiff -= static_cast(BLOCK_SIZE_QUEUE); + currentTailIndex += static_cast(BLOCK_SIZE_QUEUE); this->tailBlock = this->tailBlock->next; firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; @@ -2069,12 +2069,12 @@ class ConcurrentQueue // Now allocate as many blocks as necessary from the block pool while (blockBaseDiff > 0) { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); + blockBaseDiff -= static_cast(BLOCK_SIZE_QUEUE); + currentTailIndex += static_cast(BLOCK_SIZE_QUEUE); auto head = this->headIndex.load(std::memory_order_relaxed); assert(!details::circular_less_than(currentTailIndex, head)); - bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE_QUEUE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE_QUEUE < currentTailIndex - head)); if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize || full) { MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { // Failed to allocate, undo changes (but keep injected blocks) @@ -2149,12 +2149,12 @@ class ConcurrentQueue currentTailIndex = startTailIndex; auto endBlock = this->tailBlock; this->tailBlock = startBlock; - assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + assert((startTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0 && firstAllocatedBlock != nullptr) { this->tailBlock = firstAllocatedBlock; } while (true) { - index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); if (details::circular_less_than(newTailIndex, stopIndex)) { stopIndex = newTailIndex; } @@ -2191,12 +2191,12 @@ class ConcurrentQueue if (!details::is_trivially_destructible::value) { auto block = startBlock; - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + if ((startTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0) { block = firstAllocatedBlock; } currentTailIndex = startTailIndex; while (true) { - stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); if (details::circular_less_than(constructedStopIndex, stopIndex)) { stopIndex = constructedStopIndex; } @@ -2258,15 +2258,15 @@ class ConcurrentQueue auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); auto headBase = localBlockIndex->entries[localBlockIndexHead].base; - auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / BLOCK_SIZE); + auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE_QUEUE - 1); + auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / BLOCK_SIZE_QUEUE); auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1); // Iterate the blocks and dequeue auto index = firstIndex; do { auto firstIndexInBlock = index; - index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + index_t endIndex = (index & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; auto block = localBlockIndex->entries[indexIndex].block; if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { @@ -2300,7 +2300,7 @@ class ConcurrentQueue indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); firstIndexInBlock = index; - endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = (index & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; } while (index != firstIndex + actualCount); @@ -2434,7 +2434,7 @@ class ConcurrentQueue assert(index == tail || details::circular_less_than(index, tail)); bool forceFreeLastBlock = index != tail; // If we enter the loop, then the last (tail) block will not be freed while (index != tail) { - if ((index & static_cast(BLOCK_SIZE - 1)) == 0 || block == nullptr) { + if ((index & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0 || block == nullptr) { if (block != nullptr) { // Free the old block this->parent->add_block_to_free_list(block); @@ -2449,7 +2449,7 @@ class ConcurrentQueue // Even if the queue is empty, there's still one block that's not on the free list // (unless the head index reached the end of it, in which case the tail will be poised // to create a new block). - if (this->tailBlock != nullptr && (forceFreeLastBlock || (tail & static_cast(BLOCK_SIZE - 1)) != 0)) { + if (this->tailBlock != nullptr && (forceFreeLastBlock || (tail & static_cast(BLOCK_SIZE_QUEUE - 1)) != 0)) { this->parent->add_block_to_free_list(this->tailBlock); } @@ -2473,11 +2473,11 @@ class ConcurrentQueue { index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); index_t newTailIndex = 1 + currentTailIndex; - if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + if ((currentTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0) { // We reached the end of a block, start a new one auto head = this->headIndex.load(std::memory_order_relaxed); assert(!details::circular_less_than(currentTailIndex, head)); - if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE_QUEUE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE_QUEUE < currentTailIndex - head))) { return false; } #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX @@ -2625,15 +2625,15 @@ class ConcurrentQueue auto endBlock = this->tailBlock; // Figure out how many blocks we'll need to allocate, and do so - size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); - index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1); if (blockBaseDiff > 0) { #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX debug::DebugLock lock(mutex); #endif do { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); + blockBaseDiff -= static_cast(BLOCK_SIZE_QUEUE); + currentTailIndex += static_cast(BLOCK_SIZE_QUEUE); // Find out where we'll be inserting this block in the block index BlockIndexEntry* idxEntry = nullptr; // initialization here unnecessary but compiler can't always tell @@ -2641,7 +2641,7 @@ class ConcurrentQueue bool indexInserted = false; auto head = this->headIndex.load(std::memory_order_relaxed); assert(!details::circular_less_than(currentTailIndex, head)); - bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE_QUEUE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE_QUEUE < currentTailIndex - head)); if (full || !(indexInserted = insert_block_index_entry(idxEntry, currentTailIndex)) || (newBlock = this->parent->ConcurrentQueue::template requisition_block()) == nullptr) { // Index allocation or block allocation failed; revert any other allocations @@ -2650,9 +2650,9 @@ class ConcurrentQueue rewind_block_index_tail(); idxEntry->value.store(nullptr, std::memory_order_relaxed); } - currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1); for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { - currentTailIndex += static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE_QUEUE); idxEntry = get_block_index_entry_for_index(currentTailIndex); idxEntry->value.store(nullptr, std::memory_order_relaxed); rewind_block_index_tail(); @@ -2674,7 +2674,7 @@ class ConcurrentQueue // Store the chain of blocks so that we can undo if later allocations fail, // and so that we can find the blocks when we do the actual enqueueing - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr) { + if ((startTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) != 0 || firstAllocatedBlock != nullptr) { assert(this->tailBlock != nullptr); this->tailBlock->next = newBlock; } @@ -2688,12 +2688,12 @@ class ConcurrentQueue index_t newTailIndex = startTailIndex + static_cast(count); currentTailIndex = startTailIndex; this->tailBlock = startBlock; - assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + assert((startTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0 && firstAllocatedBlock != nullptr) { this->tailBlock = firstAllocatedBlock; } while (true) { - index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); if (details::circular_less_than(newTailIndex, stopIndex)) { stopIndex = newTailIndex; } @@ -2716,12 +2716,12 @@ class ConcurrentQueue if (!details::is_trivially_destructible::value) { auto block = startBlock; - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + if ((startTailIndex & static_cast(BLOCK_SIZE_QUEUE - 1)) == 0) { block = firstAllocatedBlock; } currentTailIndex = startTailIndex; while (true) { - stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); if (details::circular_less_than(constructedStopIndex, stopIndex)) { stopIndex = constructedStopIndex; } @@ -2735,9 +2735,9 @@ class ConcurrentQueue } } - currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE_QUEUE - 1); for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { - currentTailIndex += static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE_QUEUE); auto idxEntry = get_block_index_entry_for_index(currentTailIndex); idxEntry->value.store(nullptr, std::memory_order_relaxed); rewind_block_index_tail(); @@ -2791,7 +2791,7 @@ class ConcurrentQueue auto indexIndex = get_block_index_index_for_index(index, localBlockIndex); do { auto blockStartIndex = index; - index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + index_t endIndex = (index & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; auto entry = localBlockIndex->index[indexIndex]; @@ -2832,7 +2832,7 @@ class ConcurrentQueue indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); blockStartIndex = index; - endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = (index & ~static_cast(BLOCK_SIZE_QUEUE - 1)) + static_cast(BLOCK_SIZE_QUEUE); endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; } while (index != firstIndex + actualCount); @@ -2933,14 +2933,14 @@ class ConcurrentQueue #ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX debug::DebugLock lock(mutex); #endif - index &= ~static_cast(BLOCK_SIZE - 1); + index &= ~static_cast(BLOCK_SIZE_QUEUE - 1); localBlockIndex = blockIndex.load(std::memory_order_acquire); auto tail = localBlockIndex->tail.load(std::memory_order_acquire); auto tailBase = localBlockIndex->index[tail]->key.load(std::memory_order_relaxed); assert(tailBase != INVALID_BLOCK_BASE); // Note: Must use division instead of shift because the index may wrap around, causing a negative // offset, whose negativity we want to preserve - auto offset = static_cast(static_cast::type>(index - tailBase) / BLOCK_SIZE); + auto offset = static_cast(static_cast::type>(index - tailBase) / BLOCK_SIZE_QUEUE); size_t idx = (tail + offset) & (localBlockIndex->capacity - 1); assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index && localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr); return idx; @@ -3147,7 +3147,7 @@ class ConcurrentQueue stats.implicitBlockIndexBytes += sizeof(typename ImplicitProducer::BlockIndexHeader) + hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry*); } } - for (; details::circular_less_than(head, tail); head += BLOCK_SIZE) { + for (; details::circular_less_than(head, tail); head += BLOCK_SIZE_QUEUE) { //auto block = prod->get_block_index_entry_for_index(head); ++stats.usedBlocks; }