diff --git a/.ai-memory-bank/.gitkeep b/.ai-memory-bank/.gitkeep new file mode 100644 index 000000000000..689752efa566 --- /dev/null +++ b/.ai-memory-bank/.gitkeep @@ -0,0 +1,2 @@ +# This file is used to keep the memory-bank directory in the repository. +# It can be safely deleted if the directory contains other files. \ No newline at end of file diff --git a/.ai-memory-bank/comprehensive-file-index.md b/.ai-memory-bank/comprehensive-file-index.md new file mode 100644 index 000000000000..c5009320bf70 --- /dev/null +++ b/.ai-memory-bank/comprehensive-file-index.md @@ -0,0 +1,350 @@ +# Comprehensive File Index + +## Root Directory +- .ai_system_prompt - AI system prompt configuration file +- .clang-format - Code formatting configuration for C++ code +- .clang-tidy - Static analysis configuration for C++ code +- .cmake-format.py - Configuration for formatting CMake files +- .editorconfig - Editor configuration for consistent coding styles across editors +- .gitattributes - Git attributes configuration for file handling +- .gitignore - Git ignore patterns for excluding files from version control +- .mapping.json - JSON mapping configuration file +- .piglet-meta.json - Metadata configuration for Piglet tooling +- 🛡️ .roorules - Project-specific rules and guidelines for AI agents +- AUTHORS - List of contributors and authors of the project +- CMakeLists.txt - Root CMake build configuration file +- CODE_OF_CONDUCT.md - Guidelines for community conduct and interactions +- CODEOWNERS - File defining code ownership for different parts of the project +- conanfile.py - Conan package manager configuration file +- CONTRIBUTING.md - Guidelines for contributing to the project +- LICENSE - License information for the project +- Makefile - Make build system configuration +- pyproject.toml - Python project configuration file +- pytest.ini - Configuration file for pytest testing framework +- README.md - Main project documentation and introduction +- SECURITY.md - Security policies and reporting procedures +- THIRD_PARTY.md - Information about third-party dependencies and licenses + +### GitHub Configuration (.github/) +- dependabot.yml - Configuration for automated dependency updates +- docker-compose.yml - Docker Compose configuration for development environment +- pull_request_template.md - Template for pull request descriptions +- ISSUE_TEMPLATE/ - Templates for different types of issues +- workflows/ - GitHub Actions workflow definitions + +## Core Framework (core/) +The core framework provides the foundation components for building high-performance C++ services with asynchronous I/O and coroutine support. + +### Core Components +- benchmarks/ - Performance benchmarking tools and tests +- build_config.hpp.in - Build configuration template file +- CMakeLists.txt - CMake build configuration for core framework +- dynamic_configs/ - Dynamic configuration management system +- functional_tests/ - Functional tests for core framework components +- include/ - Public header files for core framework components +- internal/ - Internal implementation details not exposed in public API +- library.yaml - Library configuration file +- libc_include_fixes/ - Fixes for standard library includes +- README.md - Core framework documentation +- src/ - Source code implementation of core framework components +- static_configs/ - Static configuration files +- sys_coro/ - System coroutine implementation +- uboost_coro/ - Boost coroutine integration +- utest/ - Unit testing framework and utilities + +### Core Subdirectories +#### alerts/ +Framework for handling alert notifications and monitoring +- source.hpp - Alert source interface + +#### baggage/ +Distributed tracing baggage propagation implementation +- baggage_manager.hpp - Baggage manager implementation +- baggage.hpp - Baggage data structure +- fwd.hpp - Forward declarations for baggage components + +#### cache/ +Caching framework for in-memory and external cache solutions +- cache_config.hpp - Cache configuration structures +- cache_statistics.hpp - Cache statistics tracking +- caching_component_base.hpp - Base class for caching components +- expirable_lru_cache.hpp - LRU cache with expiration support +- lru_cache_component_base.hpp - Base class for LRU cache components +- lru_cache_config.hpp - LRU cache configuration +- lru_cache_statistics.hpp - LRU cache statistics +- nway_lru_cache.hpp - N-way LRU cache implementation +- update_type.hpp - Cache update type definitions + +#### clients/ +HTTP and other client implementations for external service communication + +##### clients/dns/ +DNS client implementation +- component.hpp - DNS client component +- resolver.hpp - DNS resolver implementation + +##### clients/http/ +HTTP client implementation +- client.hpp - HTTP client interface +- component.hpp - HTTP client component +- request.hpp - HTTP request representation +- response.hpp - HTTP response representation +- plugins/ - HTTP client plugins for additional functionality + +#### components/ +Component system for modular service architecture with lifecycle management +- component_base.hpp - Base class for all components +- component_list.hpp - Component list management +- run.hpp - Service startup and shutdown functions + +#### concurrent/ +Concurrency primitives and utilities for thread-safe operations + +#### congestion_control/ +Network congestion control mechanisms +- component.hpp - Congestion control component +- controller.hpp - Congestion controller interface +- limiter.hpp - Request rate limiter +- sensor.hpp - Congestion monitoring sensor + +#### dist_lock/ +Distributed locking implementation for coordination between services + +#### dump/ +Data dump and restore functionality for caching components +- config.hpp - Dump configuration structures +- dumper.hpp - Data dumper interface +- operations.hpp - Dump operations + +#### dynamic_config/ +Dynamic configuration system for runtime configuration changes +- snapshot.hpp - Configuration snapshot +- source.hpp - Configuration source +- value.hpp - Typed configuration value wrapper + +#### engine/ +Coroutine-based engine for asynchronous I/O operations +- async.hpp - Asynchronous task execution +- task.hpp - Task management +- mutex.hpp - Coroutine-aware mutex +- condition_variable.hpp - Coroutine-aware condition variable + +#### fs/ +File system operations and utilities + +#### logging/ +Structured logging framework with multiple output options + +#### middlewares/ +Middleware components for request/response processing +- pipeline.hpp - Middleware pipeline implementation +- runner.hpp - Middleware execution runner + +#### net/ +Networking utilities and implementations + +#### os_signals/ +Operating system signal handling +- component.hpp - OS signals handling component +- processor.hpp - Signal processor implementation + +#### rcu/ +Read-Copy-Update (RCU) synchronization mechanism +- rcu.hpp - RCU implementation +- rcu_map.hpp - RCU-protected map data structure + +#### server/ +HTTP server implementation and related components + +#### storages/ +Storage abstractions for various database systems + +#### tracing/ +Distributed tracing implementation + +#### utils/ +General utility functions and classes + +## Code Generation Components + +### Chaotic (chaotic/) +Code generation framework for data structures and serialization + +#### Chaotic Directories +- bin/ - Executable scripts for code generation +- bin-dynamic-configs/ - Dynamic configuration generation scripts +- chaotic/ - Core chaotic implementation +- golden_tests/ - Reference tests for generated code +- include/ - Public header files for chaotic components +- integration_tests/ - Integration tests for chaotic components +- mypy/ - Python type checking configuration +- src/ - Source code implementation +- tests/ - Unit tests for chaotic components + +### Chaotic OpenAPI (chaotic-openapi/) +OpenAPI-based code generation for client and server implementations + +#### Chaotic OpenAPI Directories +- bin/ - Executable scripts for OpenAPI code generation +- chaotic_openapi/ - Core OpenAPI implementation +- golden_tests/ - Reference tests for generated OpenAPI code +- include/ - Public header files for OpenAPI components +- integration_tests/ - Integration tests for OpenAPI components +- mypy/ - Python type checking configuration +- src/ - Source code implementation +- tests/ - Unit tests for OpenAPI components + +## Database Components + +### ClickHouse (clickhouse/) +ClickHouse database driver and components for analytics workloads + +### MongoDB (mongo/) +MongoDB database driver and components for document-based storage + +### MySQL (mysql/) +MySQL database driver and components for relational data storage + +### PostgreSQL (postgresql/) +PostgreSQL database driver and components for relational data storage + +### Redis/Valkey (redis/) +Redis/Valkey database driver and components for in-memory data storage + +### RocksDB (rocks/) +RocksDB key-value storage driver and components +- CMakeLists.txt - CMake build configuration for RocksDB component +- library.yaml - Library configuration file +- include/ - Public header files for RocksDB components +- src/ - Source code implementation for RocksDB components + +### SQLite (sqlite/) +SQLite database driver and components for embedded relational storage + +### YDB (ydb/) +YDB (Yandex Database) driver and components for distributed storage + +## Communication Components + +### gRPC (grpc/) +gRPC communication framework for high-performance RPC calls + +### Kafka (kafka/) +Kafka messaging system integration for event streaming + +### RabbitMQ (rabbitmq/) +RabbitMQ messaging system integration for message queuing + +## Messaging Theme Rules (ai-development-settings/extreme/.roo/rules/20-themes/messaging/) +Comprehensive messaging patterns and best practices for building robust messaging systems + +### Messaging Theme Rule Files +- async-messaging.md - Asynchronous messaging patterns and processing strategies +- event-driven.md - Event-driven architecture and event sourcing patterns +- kafka.md - Kafka integration patterns and implementation guidelines +- message-patterns.md - General message queue patterns and best practices +- rabbitmq.md - RabbitMQ patterns and message queue management + +## Other Components + +### ODBC (odbc/) +ODBC database connectivity layer + +### OpenTelemetry (otlp/) +OpenTelemetry protocol implementation for distributed tracing and metrics + +## Development Support + +### CMake (cmake/) +CMake build system files and configurations for building userver applications + +### Libraries (libraries/) +Additional libraries and dependencies + +### Scripts (scripts/) +Development scripts and tools for various tasks + +### Testsuite (testsuite/) +Testing framework and utilities for functional and integration testing + +### Third Party (third_party/) +Third-party dependencies and libraries + +### Universal (universal/) +Universal components that work across different systems + +## Sample Services (samples/) +Example services demonstrating how to use various components of the userver framework + +### Sample Services List +- chaotic_openapi_service/ - Service using OpenAPI code generation +- chaotic_service/ - Service using chaotic code generation +- clickhouse_service/ - Service using ClickHouse database +- config_service/ - Service demonstrating configuration management +- digest_auth_service/ - Service with digest authentication +- dns-resolver/ - Service demonstrating DNS resolution +- embedded_files/ - Service with embedded files +- flatbuf_service/ - Service using FlatBuffers serialization +- grpc_middleware_service/ - Service with gRPC middleware +- grpc_service/ - Service using gRPC communication +- grpc-generic-proxy/ - Generic gRPC proxy service +- hello_service/ - Basic hello world service example +- http_caching/ - Service demonstrating HTTP caching +- http_middleware_service/ - Service with HTTP middleware +- http-client-perf/ - HTTP client performance testing service +- https_service/ - Service with HTTPS support +- json2yaml/ - Service for converting JSON to YAML +- kafka_service/ - Service using Kafka messaging +- mongo_service/ - Service using MongoDB database +- mongo-support/ - MongoDB support utilities +- multipart_service/ - Service handling multipart requests +- mysql_service/ - Service using MySQL database +- netcat/ - Network utility service +- otlp_service/ - Service using OpenTelemetry protocol +- postgres_auth/ - Service with PostgreSQL authentication +- postgres_cache_order_by/ - Service with PostgreSQL caching +- postgres_service/ - Service using PostgreSQL database +- postgres-support/ - PostgreSQL support utilities +- production_service/ - Example production-ready service +- rabbitmq_service/ - Service using RabbitMQ messaging +- redis_service/ - Service using Redis database +- s3api/ - Service with S3 API integration +- static_service/ - Service serving static files +- tcp_full_duplex_service/ - Full-duplex TCP service +- tcp_service/ - TCP-based service +- testsuite-support/ - Testsuite support utilities +- websocket_service/ - Service with WebSocket support +- ydb_service/ - Service using YDB database + +## Service Template (service_template/) +Template for creating new services with standard structure and configuration + +### Service Template Components +- .clang-format - Code formatting configuration +- .gitignore - Git ignore patterns +- CMakeLists.txt - CMake build configuration +- CMakePresets.json - CMake presets configuration +- Makefile - Make build system configuration +- README.md - Service documentation +- requirements.txt - Python requirements +- run_as_user.sh - Script for running service as specific user +- .devcontainer/ - Development container configuration +- .github/ - GitHub workflow configurations +- cmake/ - CMake modules and configurations +- configs/ - Configuration files +- postgresql/ - PostgreSQL schema files +- proto/ - Protocol buffer definitions +- src/ - Source code implementation +- tests/ - Test files + +## Memory Bank (memory-bank/) +Project-specific documentation and context files for AI agents + +### Memory Bank Files +- projectbrief.md - Foundation document defining core requirements and goals +- productContext.md - Product context including problems solved and user experience goals +- systemPatterns.md - System architecture and technical decisions +- techContext.md - Technology context including development setup and dependencies +- activeContext.md - Current work focus, recent changes, and next steps +- progress.md - Current status, completed work, and known issues +- comprehensive-file-index.md - Detailed file index with descriptions (this document) \ No newline at end of file diff --git a/.ai_system_prompt b/.ai_system_prompt new file mode 100644 index 000000000000..d6cdeebb25f2 --- /dev/null +++ b/.ai_system_prompt @@ -0,0 +1,84 @@ +# AI System Prompt for userver Service Development + +I am an expert C++ software engineer specializing in developing services using the userver framework. My purpose is to help create high-performance, scalable services that leverage the userver framework's component-based architecture and asynchronous capabilities. + +## Core Principles + +### Service Development Focus +- I develop services that use the userver framework, not the framework itself +- I focus on implementing business logic in handlers and components +- I use existing framework components rather than creating new ones +- I follow service development patterns and best practices + +### Component-Based Architecture +- Services are composed of components registered in main.cpp +- Components have well-defined lifecycles (Constructor → Start → Stop → Destructor) +- Components are configured through static_config.yaml +- Dependencies between components are declared explicitly + +### Asynchronous Programming Model +- All I/O operations use userver's asynchronous drivers +- Operations are non-blocking and coroutine-safe +- Deadline propagation ensures responsive service behavior +- Cancellation support handles long-running operations gracefully + +## Available Resources + +### Examples and Templates +- `samples/` directory contains working examples of various userver features +- `service_template/` provides a base template for creating new services +- `.ai-memory-bank/comprehensive-file-index.md` contains project file overview + +### Key Directories +- `samples/hello_service/` - Simple HTTP service example +- `samples/postgres_service/` - Service with PostgreSQL integration +- `samples/redis_service/` - Service with Redis integration +- `samples/grpc_service/` - gRPC service example + +## Development Workflow + +### 1. Service Creation +- Start with the service_template/ as a base +- Copy template: `cp -r service_template/ my_new_service/` +- Customize CMakeLists.txt for project-specific dependencies +- Implement handlers in src/ directory following established patterns +- Configure components in configs/static_config.yaml + +### 2. Building and Running +- Create build directory: `mkdir build && cd build` +- Configure: `cmake ..` +- Build: `make -j$(nproc)` +- Run: `./my_service --config ../configs/static_config.yaml` + +### 3. Testing +- Unit tests: `make test` +- Functional tests: `cd tests && python3 -m pytest` + +## Best Practices + +### Performance Optimization +- Use asynchronous operations to minimize context switches +- Implement connection pooling for database and HTTP clients +- Apply deadline propagation to all downstream requests +- Use caching for frequently accessed data + +### Development Practices +- Follow component lifecycle guidelines for initialization and cleanup +- Use framework-provided synchronization primitives +- Implement structured logging with contextual information +- Write tests at multiple levels (unit, functional, integration) + +## Quality Standards + +### Code Quality +- Follow userver coding guidelines and patterns +- Use RAII and exception-safe coding practices +- Implement proper memory management with smart pointers +- Maintain consistent code formatting + +### Testing Requirements +- Unit tests for core business logic +- Functional tests for service endpoints +- Integration tests for database and external service interactions + +This system prompt provides the foundation for developing high-quality services on the userver framework while maintaining consistency with established patterns and best practices. For detailed implementation guidance, refer to the .roorules file and examples in the samples/ directory. diff --git a/.clang-tidy b/.clang-tidy index 2c46a3b74dd3..bd9896841315 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -112,3 +112,67 @@ CheckOptions: value: '1' - key: modernize-loop-convert.UseCxx20ReverseRanges value: 'false' + + - key: readability-identifier-naming.ClassCase + value: CamelCase + # Allow DISABLED_CamelCase to skip gtest tests. + - key: readability-identifier-naming.ClassIgnoredRegexp + value: '^DISABLED_([A-Z][a-z]+)+$' + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.EnumCase + value: CamelCase + - key: readability-identifier-naming.UnionCase + value: CamelCase + - key: readability-identifier-naming.MemberCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberSuffix + value: '_' + - key: readability-identifier-naming.ProtectedMemberSuffix + value: '_' + - key: readability-identifier-naming.MethodCase + value: CamelCase + - key: readability-identifier-naming.PublicMethodIgnoredRegexp + value: "(begin|end|empty|size|ysize|front|back|parse|format)" + - key: readability-identifier-naming.FunctionCase + value: CamelCase + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.ParameterPackCase + value: lower_case + # - key: readability-identifier-naming.VariableCase + # value: lower_case + # - key: readability-identifier-naming.ConstexprVariableCase + # value: CamelCase + # - key: readability-identifier-naming.ConstexprVariablePrefix + # value: k + # - key: readability-identifier-naming.GlobalVariableCase + # value: CamelCase + # - key: readability-identifier-naming.GlobalVariablePrefix + # value: k + # - key: readability-identifier-naming.StaticVariableCase + # value: CamelCase + # - key: readability-identifier-naming.StaticVariablePrefix + # value: k + - key: readability-identifier-naming.MacroDefinitionCase + value: UPPER_CASE + - key: readability-identifier-naming.NamespaceCase + value: lower_case + # Allow NCamelCase for NMonitoring. + - key: readability-identifier-naming.NamespaceIgnoredRegexp + value: ^N([A-Z][a-z]+)+$ + # Allow kCamelCase. + - key: readability-identifier-naming.ValueTemplateParameterIgnoredRegexp + value: ^k([A-Z][a-z]+)+$ + - key: readability-identifier-naming.ConceptCase + value: CamelCase + - key: readability-identifier-naming.TemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.TemplateTemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.TypedefCase + value: CamelCase + - key: readability-identifier-naming.TypeTemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.ValueTemplateParameterCase + value: CamelCase diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml index 1a349ef7037b..55cf16f71a8f 100644 --- a/.github/workflows/alpine.yml +++ b/.github/workflows/alpine.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false - name: ubuntu-24.04 (build-only) + name: alpine (build-only) runs-on: ubuntu-24.04 env: @@ -46,7 +46,7 @@ jobs: -DUSERVER_FORCE_DOWNLOAD_GRPC=1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 diff --git a/.github/workflows/archlinux.yml b/.github/workflows/archlinux.yml new file mode 100644 index 000000000000..542f88087a1d --- /dev/null +++ b/.github/workflows/archlinux.yml @@ -0,0 +1,156 @@ +name: Arch + +'on': + pull_request: + push: + branches: + - master + - develop + - feature/** + +env: + UBSAN_OPTIONS: print_stacktrace=1 + ASAN_OPTIONS: detect_odr_violation=2 + CXX: clang++ + CC: clang + CCACHE_DIR: /home/runner/.cache/ccache + CCACHE_NOHASHDIR: true + CPM_SOURCE_CACHE: /home/runner/.cache/CPM + +jobs: + archlinux: + strategy: + fail-fast: false + + name: archlinux-latest + runs-on: ubuntu-latest + container: + image: archlinux:latest + + env: + CMAKE_FLAGS: >- + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_STANDARD=17 + -DUSERVER_USE_LD=lld + -DUSERVER_NO_WERROR=1 + -DUSERVER_BUILD_ALL_COMPONENTS=1 + -DUSERVER_BUILD_SAMPLES=1 + -DUSERVER_BUILD_TESTS=1 + -DUSERVER_FEATURE_GRPC=0 + -DUSERVER_FEATURE_GRPC_REFLECTION=0 + -DUSERVER_FEATURE_GRPC_PROTOVALIDATE=0 + -DUSERVER_FEATURE_MONGODB=0 + -DUSERVER_FEATURE_OTLP=0 + -DUSERVER_FEATURE_ROCKS=0 + -DUSERVER_USE_STATIC_LIBS=0 + -DUSERVER_FEATURE_PATCH_LIBPQ=0 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Restore cached directories + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: 'archlinux-cache-dir ${{github.ref}} run-${{github.run_number}}' + restore-keys: | + archlinux-cache-dir ${{github.ref}} + archlinux-cache-dir + + - name: Setup host cache dirs + run: | + mkdir -p ${{env.CCACHE_DIR}} + mkdir -p ${{env.CPM_SOURCE_CACHE}} + + - name: Update system and install base dependencies + run: | + pacman -Syu --noconfirm + pacman -S --needed --noconfirm \ + base-devel \ + git \ + cmake \ + make \ + clang \ + lld \ + ccache \ + python \ + pkg-config + + - name: Install development dependencies + run: | + useradd build + + pacman -S --needed --noconfirm $(cat scripts/docs/en/deps/arch.md | grep -v -- 'makepkg|' ) + set -x ; \ + cat scripts/docs/en/deps/arch.md | grep -oP '^makepkg\|\K.*' | while read REPLY; \ + do \ + DIR=$(mktemp -d) ;\ + git clone --branch $REPLY --single-branch https://github.com/archlinux/aur.git $DIR ;\ + pushd $DIR ;\ + + chmod a+x . + chmod a+rw -R . + + yes | su -s /usr/bin/makepkg - build ;\ + pacman -U --noconfirm *.zst + + popd ;\ + rm -rf $DIR ;\ + done + + - name: Install test dependencies + if: ${{ false }} # Not working yet + run: | + pacman -S --needed --noconfirm \ + postgresql \ + redis \ + clickhouse \ + rabbitmq \ + mongodb + + - name: Setup caches + run: | + echo "Cached CPM packages:" + du -h -d 1 ${{env.CPM_SOURCE_CACHE}} || true + for f in $(find ${{env.CPM_SOURCE_CACHE}} -name "cmake.lock" 2>/dev/null || true); + do + repo=$(ls -d $(dirname $f)*/ 2>/dev/null || true); + echo "Repository: $repo"; + git config --global --add safe.directory $repo 2>/dev/null || true; + done + + ccache -M 2.0GB + ccache -s -v + + - name: Run cmake + run: | + cmake -S . -B build_debug $CMAKE_FLAGS + + - name: Compile + run: | + cmake --build build_debug -j$(nproc) + + - name: Save cached directories + uses: actions/cache/save@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + + - name: Show cache stats + run: | + du -h -d 1 ${{env.CCACHE_DIR}} || true + du -h -d 1 ${{env.CPM_SOURCE_CACHE}} || true + ccache -s -v + + - name: Run tests + if: ${{ false }} # Not working yet + run: | + cd build_debug + ctest -V diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml index 121cb20bc950..2da1bcf62a96 100644 --- a/.github/workflows/ci-conan.yml +++ b/.github/workflows/ci-conan.yml @@ -14,7 +14,8 @@ env: jobs: build: runs-on: ${{ matrix.os }} - name: ${{ matrix.os }} + container: ${{ matrix.container }} + name: "${{ matrix.os }} (container: ${{ matrix.container || 'GithubCI' }})" strategy: fail-fast: false matrix: @@ -22,53 +23,74 @@ jobs: - os: ubuntu-22.04 conanflags: '' tests-env: 'JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64' - - os: macos-latest + - os: ubuntu-22.04 + container: ubuntu:22.04 + conanflags: '' + tests-env: 'JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64' + - os: macos-14 conanflags: '-o python_path=python3.11' tests-env: '' steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Install sudo + if: matrix.container == 'ubuntu:22.04' + run: | + DEBIAN_FRONTEND=noninteractive apt update + DEBIAN_FRONTEND=noninteractive apt install -y sudo - name: Install Ubuntu packages if: matrix.os == 'ubuntu-22.04' run: | - sudo apt update - sudo apt install -y \ - gcc g++ cmake wget git clang-format \ + sudo DEBIAN_FRONTEND=noninteractive apt update + sudo DEBIAN_FRONTEND=noninteractive apt install -y \ + libpq-dev \ + gcc g++ cmake git clang-format \ python3 python3-pip python3-venv - - name: Install test dependencies - if: matrix.os == 'ubuntu-22.04' - run: | - wget -qO- https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor | sudo tee /usr/share/keyrings/mongodb-server-7.0.gpg >/dev/null - echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \ - | sudo tee -a /etc/apt/sources.list.d/mongodb-org-7.0.list - - sudo apt update - sudo apt install -y postgresql redis mongodb-org mongodb-mongosh - sudo ./scripts/kafka/ubuntu_install_kafka.sh - - ./scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh - - name: Install MacOS packages - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-14' run: | brew update brew tap mongodb/brew brew install clang-format postgresql redis kafka rabbitmq mongodb-community brew install python@3.11 + - name: Checkout + uses: actions/checkout@v5 + + - name: Change permissions + if: matrix.container == 'ubuntu:22.04' + run: | + useradd -m -G sudo -s /bin/bash test-user + chown -R test-user . + - name: Install common packages run: | pip install "conan==2.8.0" pip install numpy - conan profile detect - conan profile show + ${{ matrix.container && 'sudo -u test-user' }} conan profile detect + ${{ matrix.container && 'sudo -u test-user' }} conan profile show - name: Run conan run: | - conan create . --build=missing -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} + ${{ matrix.container && 'sudo -u test-user' }} git config --global --add safe.directory `pwd` + ${{ matrix.container && 'sudo -u test-user' }} conan create . --build=missing -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} + + - name: Install test dependencies + if: matrix.os == 'ubuntu-22.04' + run: | + sudo DEBIAN_FRONTEND=noninteractive apt install -y curl wget lsb-release + + wget -qO- https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor | sudo tee /usr/share/keyrings/mongodb-server-7.0.gpg >/dev/null + echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \ + | sudo tee -a /etc/apt/sources.list.d/mongodb-org-7.0.list + + sudo DEBIAN_FRONTEND=noninteractive apt update + sudo DEBIAN_FRONTEND=noninteractive apt install -y postgresql redis mongodb-org mongodb-mongosh locales + ${{ matrix.container && 'sudo locale-gen en_US.UTF-8' }} + ${{ matrix.container && 'update-locale LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' }} + sudo ./scripts/kafka/ubuntu_install_kafka.sh + sudo ./scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh - name: Test userver conan package run: |- @@ -77,7 +99,7 @@ jobs: rm -rf userver/cmake/ cd samples/ - USERVER_VERSION=$(conan list -c -v quiet userver/* | tail -n 1) + USERVER_VERSION=$(${{ matrix.container && 'sudo -u test-user' }} conan list -c -v quiet userver/* | tail -n 1) for SAMPLE in \ 3_json \ chaotic_service \ @@ -88,12 +110,13 @@ jobs: https_service \ postgres_service \ redis_service \ - kafka_service \ + ${{ matrix.container && ' ' || 'kafka_service' }} \ rabbitmq_service \ mongo_service \ s3api \ ; do mv conanfile.py $SAMPLE/ - ${{matrix.tests-env}} conan test $SAMPLE/ --build=never -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} ${USERVER_VERSION} + echo "Run: ${{ matrix.container && 'sudo -u test-user' }} ${{matrix.tests-env}} conan test $SAMPLE/ --build=never -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} ${USERVER_VERSION}" + ${{ matrix.container && 'sudo -u test-user' }} ${{matrix.tests-env}} conan test $SAMPLE/ --build=never -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} ${USERVER_VERSION} mv $SAMPLE/conanfile.py ./ done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b107f2e6e457..1b105c342137 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Restore cached directories id: restore-cache @@ -155,11 +155,11 @@ jobs: sudo apt update # Instructions from https://clickhouse.com/docs/en/getting-started/install/ sudo apt install -y apt-transport-https ca-certificates dirmngr - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 - echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee /etc/apt/sources.list.d/clickhouse.list + curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | sudo gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg arch=amd64] https://packages.clickhouse.com/deb stable main" | sudo tee /etc/apt/sources.list.d/clickhouse.list # Adding mariadb repositories (from https://www.linuxcapable.com/how-to-install-mariadb-on-ubuntu-linux/ ) - curl -fsSL http://mirror.mariadb.org/PublicKey_v2 | sudo gpg --dearmor -o "/usr/share/keyrings/mariadb.gpg" + curl -fsSL https://mariadb.org/mariadb_release_signing_key.pgp | sudo gpg --dearmor -o "/usr/share/keyrings/mariadb.gpg" sudo chmod a+r "/usr/share/keyrings/mariadb.gpg" # Restore the correct URL after https://jira.mariadb.org/browse/MDBF-651 #echo "deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/mariadb.gpg] https://deb.mariadb.org/10.11/ubuntu $(lsb_release -cs) main" \ diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 77d320b9e07a..85bcb25e4f90 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml new file mode 100644 index 000000000000..c7ee25d3b4c3 --- /dev/null +++ b/.github/workflows/debian.yml @@ -0,0 +1,161 @@ +name: Debian + +'on': + pull_request: + push: + branches: + - master + - develop + - feature/** + +env: + UBSAN_OPTIONS: print_stacktrace=1 + ASAN_OPTIONS: detect_odr_violation=2 + CCACHE_DIR: /home/runner/.cache/ccache + CCACHE_NOHASHDIR: true + CPM_SOURCE_CACHE: /home/runner/.cache/CPM + +jobs: + debian: + strategy: + fail-fast: false + matrix: + include: + # TODO: debian-11 fails to compile core + # + # - cmake-flags: >- + # -DUSERVER_FORCE_DOWNLOAD_GTEST=ON + # -DCMAKE_CC_COMPILER=clang-11 + # -DCMAKE_CXX_COMPILER=clang++-11 + # image: debian:11 + # info: Debian 11 + # deps-file: debian-11.md + + - cmake-flags: >- + -DCMAKE_CC_COMPILER=clang-14 + -DCMAKE_CXX_COMPILER=clang++-14 + image: debian:12 + info: Debian 12 + deps-file: debian-12.md + + # TODO: fails with: + # 1) ld.lld: error: undefined symbol: icudt76_dat + # >>> referenced by udata.ao:(openCommonData(char const*, int, UErrorCode*)) in archive /usr/lib/x86_64-linux-gnu/libicuuc.a + # 2) userver/userver/universal/src/decimal64/decimal64_test.cpp:29:47: error: passing no argument for the '...' parameter of a variadic macro is a C++20 extension [-Werror,-Wc++20-extensions] + # TYPED_TEST_SUITE(Decimal64Round, RoundPolicies); + # - cmake-flags: >- + # -DCMAKE_CC_COMPILER=clang-19 + # -DCMAKE_CXX_COMPILER=clang++-19 + # -DUSERVER_FEATURE_GRPC=OFF + # -DUSERVER_FEATURE_OTLP=OFF + # -DUSERVER_FEATURE_GRPC_REFLECTION=OFF + # -DUSERVER_FORCE_DOWNLOAD_RE2=ON + # image: debian:13 + # info: Debian 13 + # deps-file: debian-13.md + + name: '${{ matrix.info }}' + runs-on: ubuntu-latest + container: ${{ matrix.image }} + + env: + CMAKE_FLAGS: >- + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_STANDARD=17 + -DUSERVER_USE_LD=lld + -DUSERVER_NO_WERROR=OFF + -DUSERVER_BUILD_ALL_COMPONENTS=1 + -DUSERVER_BUILD_SAMPLES=1 + -DUSERVER_BUILD_TESTS=1 + -DUSERVER_FEATURE_JEMALLOC=OFF + -DUSERVER_FEATURE_KAFKA=OFF + -DUSERVER_FEATURE_CLICKHOUSE=OFF + -DUSERVER_FEATURE_STACKTRACE=OFF + -DUSERVER_FEATURE_PATCH_LIBPQ=OFF + -DUSERVER_DISABLE_RSEQ_ACCELERATION=YES + -DUSERVER_CHAOTIC_FORMAT=OFF + -DUSERVER_CHAOTIC_GOLDEN_TESTS=OFF + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Restore cached directories + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: 'debian-cache-dir ${{matrix.image}} ${{github.ref}} run-${{github.run_number}}' + restore-keys: | + debian-cache-dir ${{github.ref}} + debian-cache-dir + + - name: Setup host cache dirs + run: | + mkdir -p ${{env.CCACHE_DIR}} + mkdir -p ${{env.CPM_SOURCE_CACHE}} + + - name: Install dependencies + run: | + apt-get update + apt-get install -y lsb-release wget gnupg + + # Install clang and lld + apt-get install -y clang lld + + # Install project dependencies + apt-get install -y $(cat scripts/docs/en/deps/${{ matrix.deps-file }}) + + - name: Install test dependencies + if: ${{ false }} + run: | + apt-get install -y postgresql-15 \ + redis-server \ + rabbitmq-server + + - name: Setup caches + run: | + echo "Cached CPM packages:" + du -h -d 1 ${{env.CPM_SOURCE_CACHE}} || true + for f in $(find ${{env.CPM_SOURCE_CACHE}} -name "cmake.lock" 2>/dev/null || true); + do + repo=$(ls -d $(dirname $f)/*/ 2>/dev/null || true); + if [ -n "$repo" ]; then + echo "Repository: $repo"; + git config --global --add safe.directory $repo; + fi + done + + ccache -M 2.0GB + ccache -s + + - name: Run cmake + run: | + cmake -S . -B build_debug $CMAKE_FLAGS ${{ matrix.cmake-flags }} + + - name: Compile + run: | + cmake --build build_debug --parallel $(nproc) + + - name: Save cached directories + uses: actions/cache/save@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + + - name: Show cache stats + run: | + du -h -d 1 ${{env.CCACHE_DIR}} || true + du -h -d 1 ${{env.CPM_SOURCE_CACHE}} || true + ccache -s -v + + - name: Run tests + if: ${{ false }} + run: | + cd build_debug + ctest -V diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 6ca9ef00018b..0af95fa6075b 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -88,7 +88,7 @@ jobs: runs-on: ubuntu-latest name: '${{ matrix.info }}' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Install packages run: | diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml new file mode 100644 index 000000000000..4ef31fbc4876 --- /dev/null +++ b/.github/workflows/fedora.yml @@ -0,0 +1,136 @@ +name: Fedora + +'on': + pull_request: + push: + branches: + - master + - develop + - feature/** + +env: + UBSAN_OPTIONS: print_stacktrace=1 + ASAN_OPTIONS: detect_odr_violation=2 + CXX: clang++ + CC: clang + CCACHE_DIR: /home/runner/.cache/ccache + CCACHE_NOHASHDIR: true + CPM_SOURCE_CACHE: /home/runner/.cache/CPM + +jobs: + fedora: + strategy: + fail-fast: false + matrix: + include: + # - version: 36 EOL + - version: 42 + + name: fedora-${{matrix.version}} + runs-on: ubuntu-latest + + container: + image: fedora:${{matrix.version}} + + env: + CMAKE_FLAGS: >- + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_STANDARD=17 + -DUSERVER_USE_LD=lld + -DUSERVER_BUILD_ALL_COMPONENTS=1 + -DUSERVER_BUILD_SAMPLES=1 + -DUSERVER_BUILD_TESTS=1 + -DUSERVER_USE_STATIC_LIBS=OFF + -DUSERVER_FEATURE_MONGODB=OFF + -DUSERVER_FEATURE_GRPC=OFF + -DUSERVER_FEATURE_GRPC_REFLECTION=OFF + -DUSERVER_FEATURE_OTLP=OFF + -DUSERVER_FEATURE_ROCKS=OFF + -DUSERVER_FEATURE_CLICKHOUSE=OFF + -DUSERVER_FEATURE_RABBITMQ=OFF + -DUSERVER_FEATURE_PATCH_LIBPQ=OFF + -DUSERVER_DOWNLOAD_PACKAGES=OFF + -DUSERVER_FEATURE_STACKTRACE=OFF + + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup cache directories + run: | + mkdir -p ${{env.CCACHE_DIR}} + mkdir -p ${{env.CPM_SOURCE_CACHE}} + + - name: Restore cached directories + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: 'fedora-cache-dir ${{github.ref}} run-${{github.run_number}}' + restore-keys: | + fedora-cache-dir ${{github.ref}} + fedora-cache-dir + + - name: Install dependencies + run: | + dnf update -y + dnf install -y git lld which clang + dnf install -y $(cat scripts/docs/en/deps/fedora-${{matrix.version}}.md) + + - name: Install test dependencies + if: ${{ false }} # Not working yet + run: | + dnf install -y postgresql-server \ + redis \ + rabbitmq-server + + - name: Setup caches + run: | + echo "Cached CPM packages:" + du -h -d 1 ${{env.CPM_SOURCE_CACHE}} || true + for f in $(find ${{env.CPM_SOURCE_CACHE}} -name "cmake.lock" 2>/dev/null || true); + do + repo=$(ls -d $(dirname $f)/*/ 2>/dev/null || true); + echo "Repository: $repo"; + git config --global --add safe.directory $repo; + done + + ccache -M 2.0GB + ccache -s -v + + - name: Run cmake + run: | + pwd + cmake -S . -B build_debug $CMAKE_FLAGS + + - name: Reconfigure cmake + run: | + pwd + cmake -S . -B build_debug $CMAKE_FLAGS + + - name: Compile + run: | + cmake --build build_debug -j$(nproc) + + - name: Save cached directories + uses: actions/cache/save@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + + - name: Show cache stats + run: | + du -h -d 1 ${{env.CCACHE_DIR}} || true + du -h -d 1 ${{env.CPM_SOURCE_CACHE}} || true + ccache -s -v + + - name: Run tests + if: ${{ false }} # Not working yet + run: | + cd build_debug + ctest -V diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1e7bab22e807..21ec07e3ad80 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false - name: macos-latest + name: macos-latest (brew) runs-on: macos-latest env: @@ -38,7 +38,7 @@ jobs: -DUSERVER_FORCE_DOWNLOAD_GRPC=1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Restore cached directories id: restore-cache @@ -56,10 +56,11 @@ jobs: run: | export SDKROOT="`xcrun --show-sdk-path`" brew update + brew uninstall cmake brew install $(cat scripts/docs/en/deps/macos.md) brew install clang-format - brew install python@3.11 brew install lld + brew install python@3.12 brew link postgresql@16 # postgresql is keg-only brew link --force zlib # keg-only + need for static linkage @@ -83,11 +84,11 @@ jobs: - name: Run cmake run: | - cmake -S./ -B./build_debug -DUSERVER_PYTHON_PATH=$(brew --prefix)/bin/python3.11 $CMAKE_FLAGS + cmake -S./ -B./build_debug -DUSERVER_PYTHON_PATH=$(brew --prefix)/bin/python3.12 $CMAKE_FLAGS - name: Reconfigure cmake run: | - cmake -S./ -B./build_debug -DUSERVER_PYTHON_PATH=$(brew --prefix)/bin/python3.11 $CMAKE_FLAGS + cmake -S./ -B./build_debug -DUSERVER_PYTHON_PATH=$(brew --prefix)/bin/python3.12 $CMAKE_FLAGS - name: Compile run: | diff --git a/.github/workflows/publish-ubuntu-22.04-base-images.yaml b/.github/workflows/publish-ubuntu-22.04-base-images.yaml index 4a7f2bd0403b..701c6e4a1fe9 100644 --- a/.github/workflows/publish-ubuntu-22.04-base-images.yaml +++ b/.github/workflows/publish-ubuntu-22.04-base-images.yaml @@ -17,7 +17,7 @@ jobs: env: USERVER_IMAGE_TAG: ${{ github.event.release.tag_name || 'latest' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/publish-ubuntu-22.04-images.yaml b/.github/workflows/publish-ubuntu-22.04-images.yaml index 73fb7ec1a056..04f58d0d8a97 100644 --- a/.github/workflows/publish-ubuntu-22.04-images.yaml +++ b/.github/workflows/publish-ubuntu-22.04-images.yaml @@ -17,7 +17,7 @@ jobs: env: USERVER_IMAGE_TAG: ${{ github.event.release.tag_name || 'latest' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/publish-ubuntu-24.04-images.yaml b/.github/workflows/publish-ubuntu-24.04-images.yaml index 8697474a042f..7dc026c0f41a 100644 --- a/.github/workflows/publish-ubuntu-24.04-images.yaml +++ b/.github/workflows/publish-ubuntu-24.04-images.yaml @@ -17,7 +17,7 @@ jobs: env: USERVER_IMAGE_TAG: ${{ github.event.release.tag_name || 'latest' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/ubuntu-minimal.yml b/.github/workflows/ubuntu-minimal.yml new file mode 100644 index 000000000000..449963127c58 --- /dev/null +++ b/.github/workflows/ubuntu-minimal.yml @@ -0,0 +1,108 @@ +name: Ubuntu + +'on': + pull_request: + push: + branches: + - master + - develop + - feature/** + +env: + JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 + UBSAN_OPTIONS: print_stacktrace=1 + ASAN_OPTIONS: detect_odr_violation=2 + CCACHE_DIR: /home/runner/.cache/ccache + CCACHE_NOHASHDIR: true + CPM_SOURCE_CACHE: /home/runner/.cache/CPM + REDIS_SLEEP_WORKAROUND_SECONDS: 60 + +jobs: + posix: + strategy: + fail-fast: false + env: + CMAKE_FLAGS: >- + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_STANDARD=17 + -DUSERVER_SANITIZE="ub addr" + -DUSERVER_BUILD_SAMPLES=1 + -DUSERVER_BUILD_TESTS=1 + + name: ubuntu (minimal installation) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Restore cached directories + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: 'ubuntu-cache-dir ${{matrix.id}} ${{github.ref}} run-${{github.run_number}}' + restore-keys: | + ubuntu-cache-dir ${{matrix.id}} ${{github.ref}} + ubuntu-cache-dir ${{matrix.id}} + + - name: Setup ramdrive for testsuites + run: | + sudo mkdir -p "/mnt/ramdisk/$USER" + sudo chmod 777 "/mnt/ramdisk/$USER" + sudo mount -t tmpfs -o size=2048M tmpfs "/mnt/ramdisk/$USER" + + - name: Free disk space + run: | + df -h + # See https://stackoverflow.com/questions/75536771/github-runner-out-of-disk-space-after-building-docker-image + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/lib/php* /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /opt/hostedtoolcache/CodeQL || true + sudo docker image prune --all --force + df -h + + - name: Install common deps + run: | + sudo apt update + sudo apt install build-essential clang cmake ccache libjemalloc-dev + sudo apt install \ + libssl-dev \ + libboost-context1.83-dev \ + libboost-coroutine1.83-dev \ + libboost-filesystem1.83-dev \ + libboost-iostreams1.83-dev \ + libboost-locale1.83-dev \ + libboost-program-options1.83-dev \ + libboost-stacktrace1.83-dev \ + libbenchmark-dev + + - name: Setup ccache + run: | + ccache -M 2.0GB + ccache -s -v + + - name: Run cmake + run: | + cmake -S . -B build_debug + + - name: Compile + run: | + pwd + cd build_debug + cmake --build . -j $(nproc) + + - name: Save cached directories + uses: actions/cache/save@v4 + with: + path: | + ${{env.CCACHE_DIR}} + ${{env.CPM_SOURCE_CACHE}} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + + - name: Show cache stats + run: | + du -h -d 1 ${{env.CCACHE_DIR}} + du -h -d 1 ${{env.CPM_SOURCE_CACHE}} + ccache -s -v diff --git a/.gitignore b/.gitignore index ac32cfefe46c..a509f672dd89 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ static-analyzer-report .clangd .vscode scripts/docs/en/dynamic_configs +scripts/docs/en/versions.md +scripts/docs/scripts/versions.js diff --git a/.mapping.json b/.mapping.json index 6ad8348b6e82..ff9450d10596 100644 --- a/.mapping.json +++ b/.mapping.json @@ -11,14 +11,18 @@ ".github/docker-compose.yml":"taxi/uservices/userver/.github/docker-compose.yml", ".github/pull_request_template.md":"taxi/uservices/userver/.github/pull_request_template.md", ".github/workflows/alpine.yml":"taxi/uservices/userver/.github/workflows/alpine.yml", + ".github/workflows/archlinux.yml":"taxi/uservices/userver/.github/workflows/archlinux.yml", ".github/workflows/ci-conan.yml":"taxi/uservices/userver/.github/workflows/ci-conan.yml", ".github/workflows/ci.yml":"taxi/uservices/userver/.github/workflows/ci.yml", ".github/workflows/codeql-analysis.yml":"taxi/uservices/userver/.github/workflows/codeql-analysis.yml", + ".github/workflows/debian.yml":"taxi/uservices/userver/.github/workflows/debian.yml", ".github/workflows/docker.yaml":"taxi/uservices/userver/.github/workflows/docker.yaml", + ".github/workflows/fedora.yml":"taxi/uservices/userver/.github/workflows/fedora.yml", ".github/workflows/macos.yml":"taxi/uservices/userver/.github/workflows/macos.yml", ".github/workflows/publish-ubuntu-22.04-base-images.yaml":"taxi/uservices/userver/.github/workflows/publish-ubuntu-22.04-base-images.yaml", ".github/workflows/publish-ubuntu-22.04-images.yaml":"taxi/uservices/userver/.github/workflows/publish-ubuntu-22.04-images.yaml", ".github/workflows/publish-ubuntu-24.04-images.yaml":"taxi/uservices/userver/.github/workflows/publish-ubuntu-24.04-images.yaml", + ".github/workflows/ubuntu-minimal.yml":"taxi/uservices/userver/.github/workflows/ubuntu-minimal.yml", ".gitignore":"taxi/uservices/userver/.gitignore", "AUTHORS":"taxi/uservices/userver/AUTHORS", "CMakeLists.txt":"taxi/uservices/userver/CMakeLists.txt", @@ -36,6 +40,7 @@ "chaotic-openapi/chaotic_openapi/__init__.py":"taxi/uservices/userver/chaotic-openapi/chaotic_openapi/__init__.py", "chaotic-openapi/chaotic_openapi/back/__init__.py":"taxi/uservices/userver/chaotic-openapi/chaotic_openapi/back/__init__.py", "chaotic-openapi/chaotic_openapi/back/cpp_client/__init__.py":"taxi/uservices/userver/chaotic-openapi/chaotic_openapi/back/cpp_client/__init__.py", + "chaotic-openapi/chaotic_openapi/back/cpp_client/middleware.py":"taxi/uservices/userver/chaotic-openapi/chaotic_openapi/back/cpp_client/middleware.py", "chaotic-openapi/chaotic_openapi/back/cpp_client/output.py":"taxi/uservices/userver/chaotic-openapi/chaotic_openapi/back/cpp_client/output.py", "chaotic-openapi/chaotic_openapi/back/cpp_client/renderer.py":"taxi/uservices/userver/chaotic-openapi/chaotic_openapi/back/cpp_client/renderer.py", "chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client.cpp.jinja":"taxi/uservices/userver/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client.cpp.jinja", @@ -105,9 +110,15 @@ "chaotic-openapi/include/userver/chaotic/openapi/parameters_read.hpp":"taxi/uservices/userver/chaotic-openapi/include/userver/chaotic/openapi/parameters_read.hpp", "chaotic-openapi/include/userver/chaotic/openapi/parameters_write.hpp":"taxi/uservices/userver/chaotic-openapi/include/userver/chaotic/openapi/parameters_write.hpp", "chaotic-openapi/integration_tests/CMakeLists.txt":"taxi/uservices/userver/chaotic-openapi/integration_tests/CMakeLists.txt", + "chaotic-openapi/integration_tests/clients/body-with-array/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/body-with-array/openapi.yaml", + "chaotic-openapi/integration_tests/clients/empty-openapi/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/empty-openapi/openapi.yaml", "chaotic-openapi/integration_tests/clients/external-refs/one.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/external-refs/one.yaml", "chaotic-openapi/integration_tests/clients/external-refs/two.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/external-refs/two.yaml", + "chaotic-openapi/integration_tests/clients/handler-tag/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/handler-tag/openapi.yaml", "chaotic-openapi/integration_tests/clients/multiple-content-types/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/multiple-content-types/openapi.yaml", + "chaotic-openapi/integration_tests/clients/non-alphanum/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/non-alphanum/openapi.yaml", + "chaotic-openapi/integration_tests/clients/operation/non_std_type_reason.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/operation/non_std_type_reason.yaml", + "chaotic-openapi/integration_tests/clients/operation/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/operation/openapi.yaml", "chaotic-openapi/integration_tests/clients/parameters/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/parameters/openapi.yaml", "chaotic-openapi/integration_tests/clients/response-headers/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/response-headers/openapi.yaml", "chaotic-openapi/integration_tests/clients/swagger/swagger.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/swagger/swagger.yaml", @@ -115,6 +126,8 @@ "chaotic-openapi/integration_tests/clients/test-middleware/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/test-middleware/openapi.yaml", "chaotic-openapi/integration_tests/clients/test-object/openapi.yaml":"taxi/uservices/userver/chaotic-openapi/integration_tests/clients/test-object/openapi.yaml", "chaotic-openapi/integration_tests/src/middleware_test.cpp":"taxi/uservices/userver/chaotic-openapi/integration_tests/src/middleware_test.cpp", + "chaotic-openapi/integration_tests/src/operation_test.cpp":"taxi/uservices/userver/chaotic-openapi/integration_tests/src/operation_test.cpp", + "chaotic-openapi/integration_tests/src/parameters_test.cpp":"taxi/uservices/userver/chaotic-openapi/integration_tests/src/parameters_test.cpp", "chaotic-openapi/integration_tests/src/requests_test.cpp":"taxi/uservices/userver/chaotic-openapi/integration_tests/src/requests_test.cpp", "chaotic-openapi/integration_tests/src/responses_test.cpp":"taxi/uservices/userver/chaotic-openapi/integration_tests/src/responses_test.cpp", "chaotic-openapi/integration_tests/src/struct_test.cpp":"taxi/uservices/userver/chaotic-openapi/integration_tests/src/struct_test.cpp", @@ -180,6 +193,7 @@ "chaotic/chaotic/error.py":"taxi/uservices/userver/chaotic/chaotic/error.py", "chaotic/chaotic/front/__init__.py":"taxi/uservices/userver/chaotic/chaotic/front/__init__.py", "chaotic/chaotic/front/parser.py":"taxi/uservices/userver/chaotic/chaotic/front/parser.py", + "chaotic/chaotic/front/ref.py":"taxi/uservices/userver/chaotic/chaotic/front/ref.py", "chaotic/chaotic/front/ref_resolver.py":"taxi/uservices/userver/chaotic/chaotic/front/ref_resolver.py", "chaotic/chaotic/front/types.py":"taxi/uservices/userver/chaotic/chaotic/front/types.py", "chaotic/chaotic/jinja_env.py":"taxi/uservices/userver/chaotic/chaotic/jinja_env.py", @@ -231,6 +245,8 @@ "chaotic/include/userver/chaotic/io/std/chrono/minutes.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/chrono/minutes.hpp", "chaotic/include/userver/chaotic/io/std/chrono/seconds.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/chrono/seconds.hpp", "chaotic/include/userver/chaotic/io/std/chrono/years.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/chrono/years.hpp", + "chaotic/include/userver/chaotic/io/std/int32_t.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/int32_t.hpp", + "chaotic/include/userver/chaotic/io/std/int64_t.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/int64_t.hpp", "chaotic/include/userver/chaotic/io/std/map.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/map.hpp", "chaotic/include/userver/chaotic/io/std/set.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/set.hpp", "chaotic/include/userver/chaotic/io/std/size_t.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/io/std/size_t.hpp", @@ -273,6 +289,7 @@ "chaotic/integration_tests/schemas/all_of.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/all_of.yaml", "chaotic/integration_tests/schemas/array.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/array.yaml", "chaotic/integration_tests/schemas/array_of_xcpptype.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/array_of_xcpptype.yaml", + "chaotic/integration_tests/schemas/container_format.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/container_format.yaml", "chaotic/integration_tests/schemas/custom_cpp_type.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/custom_cpp_type.yaml", "chaotic/integration_tests/schemas/date.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/date.yaml", "chaotic/integration_tests/schemas/external_ref.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/external_ref.yaml", @@ -486,6 +503,7 @@ "cmake/Sanitizers.cmake":"taxi/uservices/userver/cmake/Sanitizers.cmake", "cmake/SetupAbseil.cmake":"taxi/uservices/userver/cmake/SetupAbseil.cmake", "cmake/SetupAmqpCPP.cmake":"taxi/uservices/userver/cmake/SetupAmqpCPP.cmake", + "cmake/SetupBoost.cmake":"taxi/uservices/userver/cmake/SetupBoost.cmake", "cmake/SetupBrotli.cmake":"taxi/uservices/userver/cmake/SetupBrotli.cmake", "cmake/SetupCAres.cmake":"taxi/uservices/userver/cmake/SetupCAres.cmake", "cmake/SetupCCTZ.cmake":"taxi/uservices/userver/cmake/SetupCCTZ.cmake", @@ -502,6 +520,7 @@ "cmake/SetupLTO.cmake":"taxi/uservices/userver/cmake/SetupLTO.cmake", "cmake/SetupLinker.cmake":"taxi/uservices/userver/cmake/SetupLinker.cmake", "cmake/SetupMongoDeps.cmake":"taxi/uservices/userver/cmake/SetupMongoDeps.cmake", + "cmake/SetupOpenssl.cmake":"taxi/uservices/userver/cmake/SetupOpenssl.cmake", "cmake/SetupOpentelemetryProto.cmake":"taxi/uservices/userver/cmake/SetupOpentelemetryProto.cmake", "cmake/SetupPGO.cmake":"taxi/uservices/userver/cmake/SetupPGO.cmake", "cmake/SetupPostgresqlDeps.cmake":"taxi/uservices/userver/cmake/SetupPostgresqlDeps.cmake", @@ -526,8 +545,11 @@ "cmake/UserverSql.cmake":"taxi/uservices/userver/cmake/UserverSql.cmake", "cmake/UserverTestsuite.cmake":"taxi/uservices/userver/cmake/UserverTestsuite.cmake", "cmake/UserverVenv.cmake":"taxi/uservices/userver/cmake/UserverVenv.cmake", + "cmake/abseil_pr_1707.patch":"taxi/uservices/userver/cmake/abseil_pr_1707.patch", + "cmake/abseil_pr_1739.patch":"taxi/uservices/userver/cmake/abseil_pr_1739.patch", "cmake/embedded_config.cmake":"taxi/uservices/userver/cmake/embedded_config.cmake", "cmake/get_cpm.cmake":"taxi/uservices/userver/cmake/get_cpm.cmake", + "cmake/grpc_pr_36805.patch":"taxi/uservices/userver/cmake/grpc_pr_36805.patch", "cmake/install/Config.cmake.in":"taxi/uservices/userver/cmake/install/Config.cmake.in", "cmake/install/userver-chaotic-config.cmake":"taxi/uservices/userver/cmake/install/userver-chaotic-config.cmake", "cmake/install/userver-chaotic-openapi-config.cmake":"taxi/uservices/userver/cmake/install/userver-chaotic-openapi-config.cmake", @@ -588,6 +610,8 @@ "core/benchmarks/main.cpp":"taxi/uservices/userver/core/benchmarks/main.cpp", "core/build_config.hpp.in":"taxi/uservices/userver/core/build_config.hpp.in", "core/dynamic_configs/BAGGAGE_SETTINGS.yaml":"taxi/uservices/userver/core/dynamic_configs/BAGGAGE_SETTINGS.yaml", + "core/dynamic_configs/EGRESS_HTTP_PROXY_ENABLED.yaml":"taxi/uservices/userver/core/dynamic_configs/EGRESS_HTTP_PROXY_ENABLED.yaml", + "core/dynamic_configs/EGRESS_NO_PROXY_TARGETS.yaml":"taxi/uservices/userver/core/dynamic_configs/EGRESS_NO_PROXY_TARGETS.yaml", "core/dynamic_configs/HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml":"taxi/uservices/userver/core/dynamic_configs/HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml", "core/dynamic_configs/HTTP_CLIENT_CONNECT_THROTTLE.yaml":"taxi/uservices/userver/core/dynamic_configs/HTTP_CLIENT_CONNECT_THROTTLE.yaml", "core/dynamic_configs/USERVER_BAGGAGE_ENABLED.yaml":"taxi/uservices/userver/core/dynamic_configs/USERVER_BAGGAGE_ENABLED.yaml", @@ -725,6 +749,12 @@ "core/functional_tests/tracing/static_config.yaml":"taxi/uservices/userver/core/functional_tests/tracing/static_config.yaml", "core/functional_tests/tracing/tests/conftest.py":"taxi/uservices/userver/core/functional_tests/tracing/tests/conftest.py", "core/functional_tests/tracing/tests/test_trace_headers_propagation.py":"taxi/uservices/userver/core/functional_tests/tracing/tests/test_trace_headers_propagation.py", + "core/functional_tests/trx_tracker/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/trx_tracker/CMakeLists.txt", + "core/functional_tests/trx_tracker/main.cpp":"taxi/uservices/userver/core/functional_tests/trx_tracker/main.cpp", + "core/functional_tests/trx_tracker/src/handler.hpp":"taxi/uservices/userver/core/functional_tests/trx_tracker/src/handler.hpp", + "core/functional_tests/trx_tracker/static_config.yaml":"taxi/uservices/userver/core/functional_tests/trx_tracker/static_config.yaml", + "core/functional_tests/trx_tracker/tests/conftest.py":"taxi/uservices/userver/core/functional_tests/trx_tracker/tests/conftest.py", + "core/functional_tests/trx_tracker/tests/test_ignore_testpoint.py":"taxi/uservices/userver/core/functional_tests/trx_tracker/tests/test_ignore_testpoint.py", "core/functional_tests/uctl/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/uctl/CMakeLists.txt", "core/functional_tests/uctl/config_vars.yaml":"taxi/uservices/userver/core/functional_tests/uctl/config_vars.yaml", "core/functional_tests/uctl/secure_data.json":"taxi/uservices/userver/core/functional_tests/uctl/secure_data.json", @@ -1101,6 +1131,7 @@ "core/include/userver/utils/statistics/system_statistics_collector.hpp":"taxi/uservices/userver/core/include/userver/utils/statistics/system_statistics_collector.hpp", "core/include/userver/utils/statistics/writer.hpp":"taxi/uservices/userver/core/include/userver/utils/statistics/writer.hpp", "core/include/userver/utils/text.hpp":"taxi/uservices/userver/core/include/userver/utils/text.hpp", + "core/include/userver/utils/trx_tracker.hpp":"taxi/uservices/userver/core/include/userver/utils/trx_tracker.hpp", "core/include/userver/utils/userver_info.hpp":"taxi/uservices/userver/core/include/userver/utils/userver_info.hpp", "core/internal/include/userver/engine/task/task_processor_utils.hpp":"taxi/uservices/userver/core/internal/include/userver/engine/task/task_processor_utils.hpp", "core/internal/include/userver/internal/net/net_listener.hpp":"taxi/uservices/userver/core/internal/include/userver/internal/net/net_listener.hpp", @@ -1560,6 +1591,7 @@ "core/src/fs/read.cpp":"taxi/uservices/userver/core/src/fs/read.cpp", "core/src/fs/read_test.cpp":"taxi/uservices/userver/core/src/fs/read_test.cpp", "core/src/fs/temp_file.cpp":"taxi/uservices/userver/core/src/fs/temp_file.cpp", + "core/src/fs/temp_file_test.cpp":"taxi/uservices/userver/core/src/fs/temp_file_test.cpp", "core/src/fs/write.cpp":"taxi/uservices/userver/core/src/fs/write.cpp", "core/src/fs/write_test.cpp":"taxi/uservices/userver/core/src/fs/write_test.cpp", "core/src/logging/component.cpp":"taxi/uservices/userver/core/src/logging/component.cpp", @@ -1957,6 +1989,8 @@ "core/src/utils/sys_info.hpp":"taxi/uservices/userver/core/src/utils/sys_info.hpp", "core/src/utils/text.cpp":"taxi/uservices/userver/core/src/utils/text.cpp", "core/src/utils/text_test.cpp":"taxi/uservices/userver/core/src/utils/text_test.cpp", + "core/src/utils/trx_tracker.cpp":"taxi/uservices/userver/core/src/utils/trx_tracker.cpp", + "core/src/utils/trx_tracker_test.cpp":"taxi/uservices/userver/core/src/utils/trx_tracker_test.cpp", "core/src/utils/userver_info.cpp":"taxi/uservices/userver/core/src/utils/userver_info.cpp", "core/static_configs/dns_client.yaml":"taxi/uservices/userver/core/static_configs/dns_client.yaml", "core/sys_coro/include/coroutines/coroutine.hpp":"taxi/uservices/userver/core/sys_coro/include/coroutines/coroutine.hpp", @@ -1994,6 +2028,7 @@ "grpc/benchmarks/format_log_message.cpp":"taxi/uservices/userver/grpc/benchmarks/format_log_message.cpp", "grpc/benchmarks/logging.cpp":"taxi/uservices/userver/grpc/benchmarks/logging.cpp", "grpc/benchmarks/ya.make":"taxi/uservices/userver/grpc/benchmarks/ya.make", + "grpc/dynamic_configs/EGRESS_GRPC_PROXY_ENABLED.yaml":"taxi/uservices/userver/grpc/dynamic_configs/EGRESS_GRPC_PROXY_ENABLED.yaml", "grpc/dynamic_configs/USERVER_GRPC_CLIENT_ENABLE_DEADLINE_PROPAGATION.yaml":"taxi/uservices/userver/grpc/dynamic_configs/USERVER_GRPC_CLIENT_ENABLE_DEADLINE_PROPAGATION.yaml", "grpc/dynamic_configs/USERVER_GRPC_SERVER_CANCEL_TASK_BY_DEADLINE.yaml":"taxi/uservices/userver/grpc/dynamic_configs/USERVER_GRPC_SERVER_CANCEL_TASK_BY_DEADLINE.yaml", "grpc/functional_tests/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/CMakeLists.txt", @@ -2030,6 +2065,7 @@ "grpc/functional_tests/basic_chaos/tests-grpcserver/test_network_smaller_parts.py":"taxi/uservices/userver/grpc/functional_tests/basic_chaos/tests-grpcserver/test_network_smaller_parts.py", "grpc/functional_tests/basic_chaos/tests-grpcserver/test_server_client_bytes.py":"taxi/uservices/userver/grpc/functional_tests/basic_chaos/tests-grpcserver/test_server_client_bytes.py", "grpc/functional_tests/basic_chaos/tests-grpcserver/test_server_smaller_parts.py":"taxi/uservices/userver/grpc/functional_tests/basic_chaos/tests-grpcserver/test_server_smaller_parts.py", + "grpc/functional_tests/basic_chaos/tests-grpcserver/test_tracing_metadata.py":"taxi/uservices/userver/grpc/functional_tests/basic_chaos/tests-grpcserver/test_tracing_metadata.py", "grpc/functional_tests/basic_server/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/basic_server/CMakeLists.txt", "grpc/functional_tests/basic_server/grpc_service.cpp":"taxi/uservices/userver/grpc/functional_tests/basic_server/grpc_service.cpp", "grpc/functional_tests/basic_server/proto/samples/greeter.proto":"taxi/uservices/userver/grpc/functional_tests/basic_server/proto/samples/greeter.proto", @@ -2041,6 +2077,9 @@ "grpc/functional_tests/basic_server/tests-unix-socket/conftest.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py", "grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py", "grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py", + "grpc/functional_tests/basic_server/unused_proto/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/basic_server/unused_proto/CMakeLists.txt", + "grpc/functional_tests/basic_server/unused_proto/proto/samples/unused.proto":"taxi/uservices/userver/grpc/functional_tests/basic_server/unused_proto/proto/samples/unused.proto", + "grpc/functional_tests/basic_server/unused_proto/proto/ya.make":"taxi/uservices/userver/grpc/functional_tests/basic_server/unused_proto/proto/ya.make", "grpc/functional_tests/lowlevel/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/lowlevel/CMakeLists.txt", "grpc/functional_tests/lowlevel/main.cpp":"taxi/uservices/userver/grpc/functional_tests/lowlevel/main.cpp", "grpc/functional_tests/lowlevel/proto/samples/greeter.proto":"taxi/uservices/userver/grpc/functional_tests/lowlevel/proto/samples/greeter.proto", @@ -2121,26 +2160,34 @@ "grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp", "grpc/include/userver/ugrpc/client/impl/async_methods.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_methods.hpp", "grpc/include/userver/ugrpc/client/impl/async_stream_methods.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_stream_methods.hpp", + "grpc/include/userver/ugrpc/client/impl/async_unary_call_adapter.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_unary_call_adapter.hpp", "grpc/include/userver/ugrpc/client/impl/call_kind.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/call_kind.hpp", "grpc/include/userver/ugrpc/client/impl/call_params.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/call_params.hpp", "grpc/include/userver/ugrpc/client/impl/call_state.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/call_state.hpp", - "grpc/include/userver/ugrpc/client/impl/channel_arguments_builder.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/channel_arguments_builder.hpp", + "grpc/include/userver/ugrpc/client/impl/channel_argument_utils.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/channel_argument_utils.hpp", "grpc/include/userver/ugrpc/client/impl/channel_factory.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/channel_factory.hpp", "grpc/include/userver/ugrpc/client/impl/client_data.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/client_data.hpp", + "grpc/include/userver/ugrpc/client/impl/client_data_accessor.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/client_data_accessor.hpp", "grpc/include/userver/ugrpc/client/impl/client_internals.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/client_internals.hpp", "grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp", "grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp", + "grpc/include/userver/ugrpc/client/impl/compat/channel_arguments_builder.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/compat/channel_arguments_builder.hpp", "grpc/include/userver/ugrpc/client/impl/completion_queue_pool.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/completion_queue_pool.hpp", + "grpc/include/userver/ugrpc/client/impl/fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/fwd.hpp", "grpc/include/userver/ugrpc/client/impl/graceful_stream_finish.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/graceful_stream_finish.hpp", "grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp", "grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp", "grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp", "grpc/include/userver/ugrpc/client/impl/prepare_call.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/prepare_call.hpp", + "grpc/include/userver/ugrpc/client/impl/response_future_impl_base.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/response_future_impl_base.hpp", + "grpc/include/userver/ugrpc/client/impl/retry_backoff.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/retry_backoff.hpp", + "grpc/include/userver/ugrpc/client/impl/retry_policy.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/retry_policy.hpp", "grpc/include/userver/ugrpc/client/impl/rpc.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/rpc.hpp", "grpc/include/userver/ugrpc/client/impl/stub_any.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/stub_any.hpp", "grpc/include/userver/ugrpc/client/impl/stub_handle.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/stub_handle.hpp", "grpc/include/userver/ugrpc/client/impl/stub_pool.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/stub_pool.hpp", "grpc/include/userver/ugrpc/client/impl/tracing.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/tracing.hpp", + "grpc/include/userver/ugrpc/client/impl/unary_call.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/unary_call.hpp", "grpc/include/userver/ugrpc/client/middlewares/baggage/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/middlewares/baggage/component.hpp", "grpc/include/userver/ugrpc/client/middlewares/baggage/middleware.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/middlewares/baggage/middleware.hpp", "grpc/include/userver/ugrpc/client/middlewares/base.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/middlewares/base.hpp", @@ -2154,6 +2201,7 @@ "grpc/include/userver/ugrpc/client/middlewares/testsuite/middleware.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/middlewares/testsuite/middleware.hpp", "grpc/include/userver/ugrpc/client/qos.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/qos.hpp", "grpc/include/userver/ugrpc/client/response_future.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/response_future.hpp", + "grpc/include/userver/ugrpc/client/retry_config.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/retry_config.hpp", "grpc/include/userver/ugrpc/client/simple_client_component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/simple_client_component.hpp", "grpc/include/userver/ugrpc/client/stream.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/stream.hpp", "grpc/include/userver/ugrpc/client/stream_read_future.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/stream_read_future.hpp", @@ -2165,12 +2213,14 @@ "grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp", "grpc/include/userver/ugrpc/impl/protobuf_collector.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/protobuf_collector.hpp", "grpc/include/userver/ugrpc/impl/queue_runner.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/queue_runner.hpp", + "grpc/include/userver/ugrpc/impl/rpc_type.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/rpc_type.hpp", "grpc/include/userver/ugrpc/impl/static_service_metadata.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/static_service_metadata.hpp", "grpc/include/userver/ugrpc/impl/statistics.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/statistics.hpp", "grpc/include/userver/ugrpc/impl/statistics_scope.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/statistics_scope.hpp", "grpc/include/userver/ugrpc/impl/statistics_storage.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/statistics_storage.hpp", "grpc/include/userver/ugrpc/impl/to_string.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/to_string.hpp", "grpc/include/userver/ugrpc/proto_json.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/proto_json.hpp", + "grpc/include/userver/ugrpc/protobuf_logging.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/protobuf_logging.hpp", "grpc/include/userver/ugrpc/protobuf_visit.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/protobuf_visit.hpp", "grpc/include/userver/ugrpc/server/call_context.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/call_context.hpp", "grpc/include/userver/ugrpc/server/component_list.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/component_list.hpp", @@ -2195,6 +2245,7 @@ "grpc/include/userver/ugrpc/server/impl/stream_adapter.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/stream_adapter.hpp", "grpc/include/userver/ugrpc/server/metadata_utils.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/metadata_utils.hpp", "grpc/include/userver/ugrpc/server/middlewares/access_log/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/access_log/component.hpp", + "grpc/include/userver/ugrpc/server/middlewares/access_log/log_extra.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/access_log/log_extra.hpp", "grpc/include/userver/ugrpc/server/middlewares/baggage/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/baggage/component.hpp", "grpc/include/userver/ugrpc/server/middlewares/baggage/middleware.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/baggage/middleware.hpp", "grpc/include/userver/ugrpc/server/middlewares/base.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/base.hpp", @@ -2244,24 +2295,25 @@ "grpc/src/ugrpc/client/exceptions.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/exceptions.cpp", "grpc/src/ugrpc/client/generic_client.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/generic_client.cpp", "grpc/src/ugrpc/client/impl/async_method_invocation.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/async_method_invocation.cpp", - "grpc/src/ugrpc/client/impl/async_methods.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/async_methods.cpp", "grpc/src/ugrpc/client/impl/async_stream_methods.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/async_stream_methods.cpp", "grpc/src/ugrpc/client/impl/call_options_accessor.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/call_options_accessor.cpp", "grpc/src/ugrpc/client/impl/call_options_accessor.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/call_options_accessor.hpp", "grpc/src/ugrpc/client/impl/call_params.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/call_params.cpp", "grpc/src/ugrpc/client/impl/call_state.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/call_state.cpp", - "grpc/src/ugrpc/client/impl/channel_arguments_builder.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/channel_arguments_builder.cpp", + "grpc/src/ugrpc/client/impl/channel_argument_utils.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/channel_argument_utils.cpp", "grpc/src/ugrpc/client/impl/channel_factory.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/channel_factory.cpp", "grpc/src/ugrpc/client/impl/client_data.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_data.cpp", "grpc/src/ugrpc/client/impl/client_factory_config.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_factory_config.cpp", "grpc/src/ugrpc/client/impl/client_factory_config.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_factory_config.hpp", "grpc/src/ugrpc/client/impl/client_qos.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_qos.cpp", + "grpc/src/ugrpc/client/impl/compat/channel_arguments_builder.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/compat/channel_arguments_builder.cpp", + "grpc/src/ugrpc/client/impl/compat/retry_policy.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/compat/retry_policy.cpp", + "grpc/src/ugrpc/client/impl/compat/retry_policy.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/compat/retry_policy.hpp", "grpc/src/ugrpc/client/impl/completion_queue_pool.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/completion_queue_pool.cpp", "grpc/src/ugrpc/client/impl/middleware_hooks.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/middleware_hooks.cpp", "grpc/src/ugrpc/client/impl/middleware_pipeline.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/middleware_pipeline.cpp", + "grpc/src/ugrpc/client/impl/retry_backoff.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/retry_backoff.cpp", "grpc/src/ugrpc/client/impl/retry_policy.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/retry_policy.cpp", - "grpc/src/ugrpc/client/impl/retry_policy.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/retry_policy.hpp", - "grpc/src/ugrpc/client/impl/rpc.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/rpc.cpp", "grpc/src/ugrpc/client/impl/stub_handle.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/stub_handle.cpp", "grpc/src/ugrpc/client/impl/stub_pool.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/stub_pool.cpp", "grpc/src/ugrpc/client/impl/tracing.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/tracing.cpp", @@ -2278,6 +2330,7 @@ "grpc/src/ugrpc/client/middlewares/testsuite/component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/middlewares/testsuite/component.cpp", "grpc/src/ugrpc/client/middlewares/testsuite/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/middlewares/testsuite/middleware.cpp", "grpc/src/ugrpc/client/qos.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/qos.cpp", + "grpc/src/ugrpc/client/retry_config.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/retry_config.cpp", "grpc/src/ugrpc/client/secdist.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/secdist.hpp", "grpc/src/ugrpc/client/simple_client_component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/simple_client_component.cpp", "grpc/src/ugrpc/datetime_utils.cpp":"taxi/uservices/userver/grpc/src/ugrpc/datetime_utils.cpp", @@ -2290,8 +2343,6 @@ "grpc/src/ugrpc/impl/logging.cpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/logging.cpp", "grpc/src/ugrpc/impl/logging.hpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/logging.hpp", "grpc/src/ugrpc/impl/protobuf_collector.cpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/protobuf_collector.cpp", - "grpc/src/ugrpc/impl/protobuf_utils.cpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/protobuf_utils.cpp", - "grpc/src/ugrpc/impl/protobuf_utils.hpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/protobuf_utils.hpp", "grpc/src/ugrpc/impl/queue_runner.cpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/queue_runner.cpp", "grpc/src/ugrpc/impl/rpc_metadata.cpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/rpc_metadata.cpp", "grpc/src/ugrpc/impl/rpc_metadata.hpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/rpc_metadata.hpp", @@ -2300,6 +2351,7 @@ "grpc/src/ugrpc/impl/statistics_scope.cpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/statistics_scope.cpp", "grpc/src/ugrpc/impl/statistics_storage.cpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/statistics_storage.cpp", "grpc/src/ugrpc/proto_json.cpp":"taxi/uservices/userver/grpc/src/ugrpc/proto_json.cpp", + "grpc/src/ugrpc/protobuf_logging.cpp":"taxi/uservices/userver/grpc/src/ugrpc/protobuf_logging.cpp", "grpc/src/ugrpc/protobuf_visit.cpp":"taxi/uservices/userver/grpc/src/ugrpc/protobuf_visit.cpp", "grpc/src/ugrpc/server/call_context.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/call_context.cpp", "grpc/src/ugrpc/server/component_list.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/component_list.cpp", @@ -2322,6 +2374,7 @@ "grpc/src/ugrpc/server/impl/service_worker.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/service_worker.cpp", "grpc/src/ugrpc/server/impl/service_worker_impl.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/service_worker_impl.cpp", "grpc/src/ugrpc/server/middlewares/access_log/component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/access_log/component.cpp", + "grpc/src/ugrpc/server/middlewares/access_log/log_extra.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/access_log/log_extra.cpp", "grpc/src/ugrpc/server/middlewares/access_log/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/access_log/middleware.cpp", "grpc/src/ugrpc/server/middlewares/access_log/middleware.hpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/access_log/middleware.hpp", "grpc/src/ugrpc/server/middlewares/baggage/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/baggage/middleware.cpp", @@ -2350,12 +2403,12 @@ "grpc/tests/baggage_test.cpp":"taxi/uservices/userver/grpc/tests/baggage_test.cpp", "grpc/tests/base_test.cpp":"taxi/uservices/userver/grpc/tests/base_test.cpp", "grpc/tests/cancel_test.cpp":"taxi/uservices/userver/grpc/tests/cancel_test.cpp", - "grpc/tests/channel_arguments_builder_test.cpp":"taxi/uservices/userver/grpc/tests/channel_arguments_builder_test.cpp", "grpc/tests/channels_test.cpp":"taxi/uservices/userver/grpc/tests/channels_test.cpp", "grpc/tests/client_cancel_test.cpp":"taxi/uservices/userver/grpc/tests/client_cancel_test.cpp", "grpc/tests/client_factory_test.cpp":"taxi/uservices/userver/grpc/tests/client_factory_test.cpp", "grpc/tests/client_middleware_hooks_test.cpp":"taxi/uservices/userver/grpc/tests/client_middleware_hooks_test.cpp", "grpc/tests/client_qos_test.cpp":"taxi/uservices/userver/grpc/tests/client_qos_test.cpp", + "grpc/tests/compat/channel_arguments_builder_test.cpp":"taxi/uservices/userver/grpc/tests/compat/channel_arguments_builder_test.cpp", "grpc/tests/congestion_control_test.cpp":"taxi/uservices/userver/grpc/tests/congestion_control_test.cpp", "grpc/tests/datetime_utils_test.cpp":"taxi/uservices/userver/grpc/tests/datetime_utils_test.cpp", "grpc/tests/deadline_metrics_test.cpp":"taxi/uservices/userver/grpc/tests/deadline_metrics_test.cpp", @@ -2365,8 +2418,10 @@ "grpc/tests/generic_client_test.cpp":"taxi/uservices/userver/grpc/tests/generic_client_test.cpp", "grpc/tests/generic_server_test.cpp":"taxi/uservices/userver/grpc/tests/generic_server_test.cpp", "grpc/tests/logging_test.cpp":"taxi/uservices/userver/grpc/tests/logging_test.cpp", + "grpc/tests/metadata_test.cpp":"taxi/uservices/userver/grpc/tests/metadata_test.cpp", "grpc/tests/proto_json.cpp":"taxi/uservices/userver/grpc/tests/proto_json.cpp", "grpc/tests/protobuf_collector_test.cpp":"taxi/uservices/userver/grpc/tests/protobuf_collector_test.cpp", + "grpc/tests/protobuf_logging_test.cpp":"taxi/uservices/userver/grpc/tests/protobuf_logging_test.cpp", "grpc/tests/protobuf_utils_test.cpp":"taxi/uservices/userver/grpc/tests/protobuf_utils_test.cpp", "grpc/tests/protobuf_visit_test.cpp":"taxi/uservices/userver/grpc/tests/protobuf_visit_test.cpp", "grpc/tests/retry_test.cpp":"taxi/uservices/userver/grpc/tests/retry_test.cpp", @@ -2507,6 +2562,10 @@ "libraries/easy/samples/6_pg_service_template_no_http_with/testsuite/test_basic.py":"taxi/uservices/userver/libraries/easy/samples/6_pg_service_template_no_http_with/testsuite/test_basic.py", "libraries/easy/samples/CMakeLists.txt":"taxi/uservices/userver/libraries/easy/samples/CMakeLists.txt", "libraries/easy/src/easy.cpp":"taxi/uservices/userver/libraries/easy/src/easy.cpp", + "libraries/grpc-proto-structs/include/userver/grpc-proto-structs/client/impl/codegen_declarations.hpp":"taxi/uservices/userver/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/client/impl/codegen_declarations.hpp", + "libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/impl/codegen_declarations.hpp":"taxi/uservices/userver/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/impl/codegen_declarations.hpp", + "libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/stream.hpp":"taxi/uservices/userver/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/stream.hpp", + "libraries/grpc-proto-structs/src/grpc-proto-structs.cpp":"taxi/uservices/userver/libraries/grpc-proto-structs/src/grpc-proto-structs.cpp", "libraries/grpc-protovalidate/CMakeLists.txt":"taxi/uservices/userver/libraries/grpc-protovalidate/CMakeLists.txt", "libraries/grpc-protovalidate/include/userver/grpc-protovalidate/buf_validate.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/buf_validate.hpp", "libraries/grpc-protovalidate/include/userver/grpc-protovalidate/client/component.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/client/component.hpp", @@ -2545,13 +2604,128 @@ "libraries/grpc-reflection/src/grpc-reflection/proto_server_reflection.cpp":"taxi/uservices/userver/libraries/grpc-reflection/src/grpc-reflection/proto_server_reflection.cpp", "libraries/grpc-reflection/src/grpc-reflection/proto_server_reflection.hpp":"taxi/uservices/userver/libraries/grpc-reflection/src/grpc-reflection/proto_server_reflection.hpp", "libraries/grpc-reflection/src/grpc-reflection/reflection_service_component.cpp":"taxi/uservices/userver/libraries/grpc-reflection/src/grpc-reflection/reflection_service_component.cpp", + "libraries/proto-structs/codegen-tests/proto/box/autobox/cycles.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox/cycles.proto", + "libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_nested.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_nested.proto", + "libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_self.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_self.proto", + "libraries/proto-structs/codegen-tests/proto/box/autobox/unbreakable_cycle.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox/unbreakable_cycle.proto", + "libraries/proto-structs/codegen-tests/proto/box/options/cycles.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/options/cycles.proto", + "libraries/proto-structs/codegen-tests/proto/box/options/initialization.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/options/initialization.proto", + "libraries/proto-structs/codegen-tests/proto/enums/names.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/enums/names.proto", + "libraries/proto-structs/codegen-tests/proto/maps/basic.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/maps/basic.proto", + "libraries/proto-structs/codegen-tests/proto/not_recommended_field_names/basic.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/not_recommended_field_names/basic.proto", + "libraries/proto-structs/codegen-tests/proto/oneof/basic.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof/basic.proto", + "libraries/proto-structs/codegen-tests/proto/oneof/custom_oneof_type_name.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof/custom_oneof_type_name.proto", + "libraries/proto-structs/codegen-tests/proto/oneof/proto2.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof/proto2.proto", + "libraries/proto-structs/codegen-tests/proto/simple/base.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/base.proto", + "libraries/proto-structs/codegen-tests/proto/simple/subdirectory/separate_enum.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/separate_enum.proto", + "libraries/proto-structs/codegen-tests/proto/simple/subdirectory/subdirectory.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/subdirectory.proto", + "libraries/proto-structs/codegen-tests/src/box/autobox/cycles_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/box/autobox/cycles_test.cpp", + "libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_nested_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_nested_test.cpp", + "libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_self_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_self_test.cpp", + "libraries/proto-structs/codegen-tests/src/box/autobox/unbreakable_cycle_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/box/autobox/unbreakable_cycle_test.cpp", + "libraries/proto-structs/codegen-tests/src/box/options/cycles_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/box/options/cycles_test.cpp", + "libraries/proto-structs/codegen-tests/src/box/options/initialization_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/box/options/initialization_test.cpp", + "libraries/proto-structs/codegen-tests/src/enums/names_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/enums/names_test.cpp", + "libraries/proto-structs/codegen-tests/src/maps/basic_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/maps/basic_test.cpp", + "libraries/proto-structs/codegen-tests/src/not_recommended_field_names/basic_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/not_recommended_field_names/basic_test.cpp", + "libraries/proto-structs/codegen-tests/src/oneof/basic_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/oneof/basic_test.cpp", + "libraries/proto-structs/codegen-tests/src/oneof/custom_oneof_type_name_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/oneof/custom_oneof_type_name_test.cpp", + "libraries/proto-structs/codegen-tests/src/oneof/proto2_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/oneof/proto2_test.cpp", + "libraries/proto-structs/codegen-tests/src/simple/simple_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/simple/simple_test.cpp", + "libraries/proto-structs/codegen-tests/src/test_utils/type_assertions.hpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/test_utils/type_assertions.hpp", + "libraries/proto-structs/include/userver/proto-structs/any.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/any.hpp", + "libraries/proto-structs/include/userver/proto-structs/convert.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/convert.hpp", + "libraries/proto-structs/include/userver/proto-structs/exceptions.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/exceptions.hpp", + "libraries/proto-structs/include/userver/proto-structs/hash_map.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/hash_map.hpp", + "libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_cpp.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_cpp.hpp", + "libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_hpp.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_hpp.hpp", + "libraries/proto-structs/include/userver/proto-structs/impl/oneof_codegen.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/impl/oneof_codegen.hpp", + "libraries/proto-structs/include/userver/proto-structs/impl/traits_light.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/impl/traits_light.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/context.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/context.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/context_base.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/context_base.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/fwd.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/fwd.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/impl/field_accessor.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/impl/field_accessor.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/impl/read.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/impl/read.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/impl/std/any_map_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/impl/std/any_map_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/impl/write.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/impl/write.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/duration.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/duration.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/duration_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/duration_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/hh_mm_ss.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/hh_mm_ss.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/hh_mm_ss_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/hh_mm_ss_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/time_point.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/time_point.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/time_point_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/time_point_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/year_month_day.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/year_month_day.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/chrono/year_month_day_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/chrono/year_month_day_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/int32_t.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/int32_t.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/int32_t_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/int32_t_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/int64_t.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/int64_t.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/int64_t_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/int64_t_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/map.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/map.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/map_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/map_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/optional.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/optional.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/optional_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/optional_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/scalar.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/scalar.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/scalar_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/scalar_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/size_t.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/size_t.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/size_t_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/size_t_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/string.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/string.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/string_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/string_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/uint32_t.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/uint32_t.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/uint32_t_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/uint32_t_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/uint64_t.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/uint64_t.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/uint64_t_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/uint64_t_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/unordered_map.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/unordered_map.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/unordered_map_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/unordered_map_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/vector.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/vector.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/std/vector_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/std/vector_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/supported_types.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/supported_types.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/supported_types_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/supported_types_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/decimal64/decimal.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/decimal64/decimal.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/decimal64/decimal_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/decimal64/decimal_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/any.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/any.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/any_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/any_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/hash_map.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/hash_map.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/hash_map_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/hash_map_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/oneof.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/oneof.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/oneof_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/oneof_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/unbreakable_dependency_cycle.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/unbreakable_dependency_cycle.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/unbreakable_dependency_cycle_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/proto_structs/unbreakable_dependency_cycle_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/utils/box.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/utils/box.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/utils/box_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/utils/box_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/utils/datetime/time_of_day.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/utils/datetime/time_of_day.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/utils/datetime/time_of_day_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/utils/datetime/time_of_day_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/utils/strong_typedef.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/utils/strong_typedef.hpp", + "libraries/proto-structs/include/userver/proto-structs/io/userver/utils/strong_typedef_conv.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/io/userver/utils/strong_typedef_conv.hpp", + "libraries/proto-structs/include/userver/proto-structs/oneof.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/oneof.hpp", + "libraries/proto-structs/include/userver/proto-structs/type_mapping.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/type_mapping.hpp", + "libraries/proto-structs/include/userver/proto-structs/unbreakable_dependency_cycle.hpp":"taxi/uservices/userver/libraries/proto-structs/include/userver/proto-structs/unbreakable_dependency_cycle.hpp", + "libraries/proto-structs/proto/userver/structs/annotations.proto":"taxi/uservices/userver/libraries/proto-structs/proto/userver/structs/annotations.proto", + "libraries/proto-structs/src/proto-structs/exceptions.cpp":"taxi/uservices/userver/libraries/proto-structs/src/proto-structs/exceptions.cpp", + "libraries/proto-structs/src/proto-structs/io/context_base.cpp":"taxi/uservices/userver/libraries/proto-structs/src/proto-structs/io/context_base.cpp", + "libraries/proto-structs/src/proto-structs/io/impl/field_accessor.cpp":"taxi/uservices/userver/libraries/proto-structs/src/proto-structs/io/impl/field_accessor.cpp", + "libraries/proto-structs/src/proto-structs/io/std/chrono/time_point.cpp":"taxi/uservices/userver/libraries/proto-structs/src/proto-structs/io/std/chrono/time_point.cpp", + "libraries/proto-structs/src/proto-structs/io/std/chrono/year_month_day.cpp":"taxi/uservices/userver/libraries/proto-structs/src/proto-structs/io/std/chrono/year_month_day.cpp", + "libraries/proto-structs/src/proto-structs/io/userver/proto_structs/any.cpp":"taxi/uservices/userver/libraries/proto-structs/src/proto-structs/io/userver/proto_structs/any.cpp", + "libraries/proto-structs/tests/any_test.cpp":"taxi/uservices/userver/libraries/proto-structs/tests/any_test.cpp", + "libraries/proto-structs/tests/exceptions_test.cpp":"taxi/uservices/userver/libraries/proto-structs/tests/exceptions_test.cpp", + "libraries/proto-structs/tests/message_to_struct_test.cpp":"taxi/uservices/userver/libraries/proto-structs/tests/message_to_struct_test.cpp", + "libraries/proto-structs/tests/oneof_test.cpp":"taxi/uservices/userver/libraries/proto-structs/tests/oneof_test.cpp", + "libraries/proto-structs/tests/proto/messages.proto":"taxi/uservices/userver/libraries/proto-structs/tests/proto/messages.proto", + "libraries/proto-structs/tests/struct_simple.cpp":"taxi/uservices/userver/libraries/proto-structs/tests/struct_simple.cpp", + "libraries/proto-structs/tests/struct_simple.hpp":"taxi/uservices/userver/libraries/proto-structs/tests/struct_simple.hpp", + "libraries/proto-structs/tests/struct_to_message_test.cpp":"taxi/uservices/userver/libraries/proto-structs/tests/struct_to_message_test.cpp", + "libraries/proto-structs/tests/structs.cpp":"taxi/uservices/userver/libraries/proto-structs/tests/structs.cpp", + "libraries/proto-structs/tests/structs.hpp":"taxi/uservices/userver/libraries/proto-structs/tests/structs.hpp", "libraries/s3api/CMakeLists.txt":"taxi/uservices/userver/libraries/s3api/CMakeLists.txt", "libraries/s3api/include/userver/s3api/authenticators/access_key.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/authenticators/access_key.hpp", "libraries/s3api/include/userver/s3api/authenticators/interface.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/authenticators/interface.hpp", "libraries/s3api/include/userver/s3api/authenticators/utils.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/authenticators/utils.hpp", "libraries/s3api/include/userver/s3api/clients/fwd.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/clients/fwd.hpp", "libraries/s3api/include/userver/s3api/clients/s3api.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/clients/s3api.hpp", + "libraries/s3api/include/userver/s3api/models/errors.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/models/errors.hpp", "libraries/s3api/include/userver/s3api/models/fwd.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/models/fwd.hpp", + "libraries/s3api/include/userver/s3api/models/multipart_upload/requests.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/models/multipart_upload/requests.hpp", + "libraries/s3api/include/userver/s3api/models/multipart_upload/responses.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/models/multipart_upload/responses.hpp", "libraries/s3api/include/userver/s3api/models/request.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/models/request.hpp", "libraries/s3api/include/userver/s3api/models/s3api_connection_type.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/models/s3api_connection_type.hpp", "libraries/s3api/include/userver/s3api/models/secret.hpp":"taxi/uservices/userver/libraries/s3api/include/userver/s3api/models/secret.hpp", @@ -2561,6 +2735,8 @@ "libraries/s3api/src/s3api/authenticators/utils_test.cpp":"taxi/uservices/userver/libraries/s3api/src/s3api/authenticators/utils_test.cpp", "libraries/s3api/src/s3api/clients/client.cpp":"taxi/uservices/userver/libraries/s3api/src/s3api/clients/client.cpp", "libraries/s3api/src/s3api/clients/client.hpp":"taxi/uservices/userver/libraries/s3api/src/s3api/clients/client.hpp", + "libraries/s3api/src/s3api/models/multipart_upload/responses.cpp":"taxi/uservices/userver/libraries/s3api/src/s3api/models/multipart_upload/responses.cpp", + "libraries/s3api/src/s3api/models/multipart_upload/responses_test.cpp":"taxi/uservices/userver/libraries/s3api/src/s3api/models/multipart_upload/responses_test.cpp", "libraries/s3api/src/s3api/models/s3api_connection_type.cpp":"taxi/uservices/userver/libraries/s3api/src/s3api/models/s3api_connection_type.cpp", "libraries/s3api/src/s3api/models/secret.cpp":"taxi/uservices/userver/libraries/s3api/src/s3api/models/secret.cpp", "libraries/s3api/src/s3api/s3_connection.cpp":"taxi/uservices/userver/libraries/s3api/src/s3api/s3_connection.cpp", @@ -2577,8 +2753,8 @@ "mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_ENABLED.yaml":"taxi/uservices/userver/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_ENABLED.yaml", "mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_SETTINGS.yaml":"taxi/uservices/userver/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_SETTINGS.yaml", "mongo/dynamic_configs/MONGO_CONNECTION_POOL_SETTINGS.yaml":"taxi/uservices/userver/mongo/dynamic_configs/MONGO_CONNECTION_POOL_SETTINGS.yaml", - "mongo/dynamic_configs/MONGO_DEADLINE_PROPAGATION_ENABLED_V2.yaml":"taxi/uservices/userver/mongo/dynamic_configs/MONGO_DEADLINE_PROPAGATION_ENABLED_V2.yaml", "mongo/dynamic_configs/MONGO_DEFAULT_MAX_TIME_MS.yaml":"taxi/uservices/userver/mongo/dynamic_configs/MONGO_DEFAULT_MAX_TIME_MS.yaml", + "mongo/dynamic_configs/USERVER_DEADLINE_PROPAGATION_ENABLED.yaml":"taxi/uservices/userver/mongo/dynamic_configs/USERVER_DEADLINE_PROPAGATION_ENABLED.yaml", "mongo/functional_tests/CMakeLists.txt":"taxi/uservices/userver/mongo/functional_tests/CMakeLists.txt", "mongo/functional_tests/basic_chaos/CMakeLists.txt":"taxi/uservices/userver/mongo/functional_tests/basic_chaos/CMakeLists.txt", "mongo/functional_tests/basic_chaos/mongo_service.cpp":"taxi/uservices/userver/mongo/functional_tests/basic_chaos/mongo_service.cpp", @@ -3312,6 +3488,7 @@ "redis/include/userver/storages/redis/request_data_base.hpp":"taxi/uservices/userver/redis/include/userver/storages/redis/request_data_base.hpp", "redis/include/userver/storages/redis/request_eval.hpp":"taxi/uservices/userver/redis/include/userver/storages/redis/request_eval.hpp", "redis/include/userver/storages/redis/request_evalsha.hpp":"taxi/uservices/userver/redis/include/userver/storages/redis/request_evalsha.hpp", + "redis/include/userver/storages/redis/request_generic.hpp":"taxi/uservices/userver/redis/include/userver/storages/redis/request_generic.hpp", "redis/include/userver/storages/redis/scan_tag.hpp":"taxi/uservices/userver/redis/include/userver/storages/redis/scan_tag.hpp", "redis/include/userver/storages/redis/subscribe_client.hpp":"taxi/uservices/userver/redis/include/userver/storages/redis/subscribe_client.hpp", "redis/include/userver/storages/redis/subscription_token.hpp":"taxi/uservices/userver/redis/include/userver/storages/redis/subscription_token.hpp", @@ -3711,11 +3888,11 @@ "samples/postgres_cache_order_by/tests/static/test_data.sql":"taxi/uservices/userver/samples/postgres_cache_order_by/tests/static/test_data.sql", "samples/postgres_cache_order_by/tests/test_cache_order_by.py":"taxi/uservices/userver/samples/postgres_cache_order_by/tests/test_cache_order_by.py", "samples/postgres_service/CMakeLists.txt":"taxi/uservices/userver/samples/postgres_service/CMakeLists.txt", - "samples/postgres_service/delete_value.sql":"taxi/uservices/userver/samples/postgres_service/delete_value.sql", - "samples/postgres_service/insert_value.sql":"taxi/uservices/userver/samples/postgres_service/insert_value.sql", "samples/postgres_service/main.cpp":"taxi/uservices/userver/samples/postgres_service/main.cpp", + "samples/postgres_service/queries/read/select_value.sql":"taxi/uservices/userver/samples/postgres_service/queries/read/select_value.sql", + "samples/postgres_service/queries/write/delete_value.sql":"taxi/uservices/userver/samples/postgres_service/queries/write/delete_value.sql", + "samples/postgres_service/queries/write/insert_value.sql":"taxi/uservices/userver/samples/postgres_service/queries/write/insert_value.sql", "samples/postgres_service/schemas/postgresql/admin.sql":"taxi/uservices/userver/samples/postgres_service/schemas/postgresql/admin.sql", - "samples/postgres_service/select_value.sql":"taxi/uservices/userver/samples/postgres_service/select_value.sql", "samples/postgres_service/static_config.yaml":"taxi/uservices/userver/samples/postgres_service/static_config.yaml", "samples/postgres_service/tests/conftest.py":"taxi/uservices/userver/samples/postgres_service/tests/conftest.py", "samples/postgres_service/tests/test_postgres.py":"taxi/uservices/userver/samples/postgres_service/tests/test_postgres.py", @@ -3836,6 +4013,7 @@ "scripts/chaotic/requirements.txt":"taxi/uservices/userver/scripts/chaotic/requirements.txt", "scripts/clickhouse/ubuntu-install-clickhouse.sh":"taxi/uservices/userver/scripts/clickhouse/ubuntu-install-clickhouse.sh", "scripts/create-service-test-helper":"taxi/uservices/userver/scripts/create-service-test-helper", + "scripts/debian-rules":"taxi/uservices/userver/scripts/debian-rules", "scripts/docker/Readme.md":"taxi/uservices/userver/scripts/docker/Readme.md", "scripts/docker/base-debian-11.dockerfile":"taxi/uservices/userver/scripts/docker/base-debian-11.dockerfile", "scripts/docker/base-ubuntu-22.04-ci.dockerfile":"taxi/uservices/userver/scripts/docker/base-ubuntu-22.04-ci.dockerfile", @@ -3868,16 +4046,41 @@ "scripts/docs/en/deps/arch.md":"taxi/uservices/userver/scripts/docs/en/deps/arch.md", "scripts/docs/en/deps/debian-11.md":"taxi/uservices/userver/scripts/docs/en/deps/debian-11.md", "scripts/docs/en/deps/debian-12.md":"taxi/uservices/userver/scripts/docs/en/deps/debian-12.md", - "scripts/docs/en/deps/fedora-35.md":"taxi/uservices/userver/scripts/docs/en/deps/fedora-35.md", - "scripts/docs/en/deps/fedora-36.md":"taxi/uservices/userver/scripts/docs/en/deps/fedora-36.md", + "scripts/docs/en/deps/debian-13.md":"taxi/uservices/userver/scripts/docs/en/deps/debian-13.md", + "scripts/docs/en/deps/fedora-42.md":"taxi/uservices/userver/scripts/docs/en/deps/fedora-42.md", "scripts/docs/en/deps/gentoo.md":"taxi/uservices/userver/scripts/docs/en/deps/gentoo.md", "scripts/docs/en/deps/macos.md":"taxi/uservices/userver/scripts/docs/en/deps/macos.md", "scripts/docs/en/deps/ubuntu-21.10.md":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-21.10.md", "scripts/docs/en/deps/ubuntu-22.04.md":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04.md", + "scripts/docs/en/deps/ubuntu-22.04/chaotic":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/chaotic", + "scripts/docs/en/deps/ubuntu-22.04/core":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/core", + "scripts/docs/en/deps/ubuntu-22.04/grpc":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/grpc", + "scripts/docs/en/deps/ubuntu-22.04/kafka":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/kafka", + "scripts/docs/en/deps/ubuntu-22.04/mongo":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/mongo", + "scripts/docs/en/deps/ubuntu-22.04/mysql":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/mysql", + "scripts/docs/en/deps/ubuntu-22.04/odbc":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/odbc", + "scripts/docs/en/deps/ubuntu-22.04/postgresql":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/postgresql", + "scripts/docs/en/deps/ubuntu-22.04/redis":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/redis", + "scripts/docs/en/deps/ubuntu-22.04/rocks":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/rocks", + "scripts/docs/en/deps/ubuntu-22.04/sqlite":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/sqlite", + "scripts/docs/en/deps/ubuntu-22.04/universal":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/universal", + "scripts/docs/en/deps/ubuntu-22.04/ydb":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-22.04/ydb", "scripts/docs/en/deps/ubuntu-24.04.md":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04.md", + "scripts/docs/en/deps/ubuntu-24.04/chaotic":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/chaotic", + "scripts/docs/en/deps/ubuntu-24.04/core":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/core", + "scripts/docs/en/deps/ubuntu-24.04/grpc":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/grpc", + "scripts/docs/en/deps/ubuntu-24.04/kafka":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/kafka", + "scripts/docs/en/deps/ubuntu-24.04/mongo":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/mongo", + "scripts/docs/en/deps/ubuntu-24.04/mysql":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/mysql", + "scripts/docs/en/deps/ubuntu-24.04/odbc":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/odbc", + "scripts/docs/en/deps/ubuntu-24.04/postgresql":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/postgresql", + "scripts/docs/en/deps/ubuntu-24.04/redis":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/redis", + "scripts/docs/en/deps/ubuntu-24.04/rocks":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/rocks", + "scripts/docs/en/deps/ubuntu-24.04/sqlite":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/sqlite", + "scripts/docs/en/deps/ubuntu-24.04/universal":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/universal", + "scripts/docs/en/deps/ubuntu-24.04/ydb":"taxi/uservices/userver/scripts/docs/en/deps/ubuntu-24.04/ydb", "scripts/docs/en/index.md":"taxi/uservices/userver/scripts/docs/en/index.md", "scripts/docs/en/landing.md":"taxi/uservices/userver/scripts/docs/en/landing.md", - "scripts/docs/en/schemas/dynamic_configs.md":"taxi/uservices/userver/scripts/docs/en/schemas/dynamic_configs.md", "scripts/docs/en/userver/404.md":"taxi/uservices/userver/scripts/docs/en/userver/404.md", "scripts/docs/en/userver/build/build.md":"taxi/uservices/userver/scripts/docs/en/userver/build/build.md", "scripts/docs/en/userver/build/dependencies.md":"taxi/uservices/userver/scripts/docs/en/userver/build/dependencies.md", @@ -3895,6 +4098,7 @@ "scripts/docs/en/userver/deploy_env.md":"taxi/uservices/userver/scripts/docs/en/userver/deploy_env.md", "scripts/docs/en/userver/development/releases.md":"taxi/uservices/userver/scripts/docs/en/userver/development/releases.md", "scripts/docs/en/userver/development/stability.md":"taxi/uservices/userver/scripts/docs/en/userver/development/stability.md", + "scripts/docs/en/userver/distro_maintainers.md":"taxi/uservices/userver/scripts/docs/en/userver/distro_maintainers.md", "scripts/docs/en/userver/dns_control.md":"taxi/uservices/userver/scripts/docs/en/userver/dns_control.md", "scripts/docs/en/userver/driver_guide.md":"taxi/uservices/userver/scripts/docs/en/userver/driver_guide.md", "scripts/docs/en/userver/dynamic_config.md":"taxi/uservices/userver/scripts/docs/en/userver/dynamic_config.md", @@ -3911,6 +4115,7 @@ "scripts/docs/en/userver/grpc/middlewares_toggle.md":"taxi/uservices/userver/scripts/docs/en/userver/grpc/middlewares_toggle.md", "scripts/docs/en/userver/grpc/server_middleware_implementation.md":"taxi/uservices/userver/scripts/docs/en/userver/grpc/server_middleware_implementation.md", "scripts/docs/en/userver/grpc/server_middlewares.md":"taxi/uservices/userver/scripts/docs/en/userver/grpc/server_middlewares.md", + "scripts/docs/en/userver/grpc/timeouts_retries.md":"taxi/uservices/userver/scripts/docs/en/userver/grpc/timeouts_retries.md", "scripts/docs/en/userver/http_server.md":"taxi/uservices/userver/scripts/docs/en/userver/http_server.md", "scripts/docs/en/userver/http_server_middlewares.md":"taxi/uservices/userver/scripts/docs/en/userver/http_server_middlewares.md", "scripts/docs/en/userver/intro.md":"taxi/uservices/userver/scripts/docs/en/userver/intro.md", @@ -3986,6 +4191,7 @@ "scripts/docs/fontello/font/fontello.woff":"taxi/uservices/userver/scripts/docs/fontello/font/fontello.woff", "scripts/docs/fontello/font/fontello.woff2":"taxi/uservices/userver/scripts/docs/fontello/font/fontello.woff2", "scripts/docs/footer.html":"taxi/uservices/userver/scripts/docs/footer.html", + "scripts/docs/generate_versions.py":"taxi/uservices/userver/scripts/docs/generate_versions.py", "scripts/docs/header.html":"taxi/uservices/userver/scripts/docs/header.html", "scripts/docs/highlight.js/LICENSE":"taxi/uservices/userver/scripts/docs/highlight.js/LICENSE", "scripts/docs/highlight.js/README.md":"taxi/uservices/userver/scripts/docs/highlight.js/README.md", @@ -4028,6 +4234,7 @@ "scripts/docs/img/plane.svg":"taxi/uservices/userver/scripts/docs/img/plane.svg", "scripts/docs/img/sample_static_service.png":"taxi/uservices/userver/scripts/docs/img/sample_static_service.png", "scripts/docs/img/slack_logo.svg":"taxi/uservices/userver/scripts/docs/img/slack_logo.svg", + "scripts/docs/img/slugkit.svg":"taxi/uservices/userver/scripts/docs/img/slugkit.svg", "scripts/docs/img/square_logo.svg":"taxi/uservices/userver/scripts/docs/img/square_logo.svg", "scripts/docs/img/telegram_logo.svg":"taxi/uservices/userver/scripts/docs/img/telegram_logo.svg", "scripts/docs/img/title.svg":"taxi/uservices/userver/scripts/docs/img/title.svg", @@ -4070,12 +4277,14 @@ "scripts/gdb/tests/src/formats/json/gdb_test_json.cpp":"taxi/uservices/userver/scripts/gdb/tests/src/formats/json/gdb_test_json.cpp", "scripts/gdb/tests/test_printers.py":"taxi/uservices/userver/scripts/gdb/tests/test_printers.py", "scripts/gdb/update_gdbinit.py":"taxi/uservices/userver/scripts/gdb/update_gdbinit.py", + "scripts/generate-debian-directory.sh":"taxi/uservices/userver/scripts/generate-debian-directory.sh", "scripts/grpc/__init__.py":"taxi/uservices/userver/scripts/grpc/__init__.py", "scripts/grpc/generator.py":"taxi/uservices/userver/scripts/grpc/generator.py", "scripts/grpc/protoc_usrv_plugin.sh":"taxi/uservices/userver/scripts/grpc/protoc_usrv_plugin.sh", "scripts/grpc/requirements-3.txt":"taxi/uservices/userver/scripts/grpc/requirements-3.txt", "scripts/grpc/requirements-4.txt":"taxi/uservices/userver/scripts/grpc/requirements-4.txt", "scripts/grpc/requirements-5.txt":"taxi/uservices/userver/scripts/grpc/requirements-5.txt", + "scripts/grpc/requirements-6.txt":"taxi/uservices/userver/scripts/grpc/requirements-6.txt", "scripts/grpc/templates/client.usrv.cpp.jinja":"taxi/uservices/userver/scripts/grpc/templates/client.usrv.cpp.jinja", "scripts/grpc/templates/client.usrv.hpp.jinja":"taxi/uservices/userver/scripts/grpc/templates/client.usrv.hpp.jinja", "scripts/grpc/templates/service.usrv.cpp.jinja":"taxi/uservices/userver/scripts/grpc/templates/service.usrv.cpp.jinja", @@ -4088,12 +4297,35 @@ "scripts/perf-blocking-syscall":"taxi/uservices/userver/scripts/perf-blocking-syscall", "scripts/postgres/pg_sql_codes.py":"taxi/uservices/userver/scripts/postgres/pg_sql_codes.py", "scripts/postgres/ubuntu-install-postgresql-includes.sh":"taxi/uservices/userver/scripts/postgres/ubuntu-install-postgresql-includes.sh", + "scripts/proto_structs/README.md":"taxi/uservices/userver/scripts/proto_structs/README.md", "scripts/proto_structs/__init__.py":"taxi/uservices/userver/scripts/proto_structs/__init__.py", + "scripts/proto_structs/descriptors/__init__.py":"taxi/uservices/userver/scripts/proto_structs/descriptors/__init__.py", + "scripts/proto_structs/descriptors/descriptor_proto.py":"taxi/uservices/userver/scripts/proto_structs/descriptors/descriptor_proto.py", + "scripts/proto_structs/descriptors/node_parsers.py":"taxi/uservices/userver/scripts/proto_structs/descriptors/node_parsers.py", + "scripts/proto_structs/descriptors/option_parsers.py":"taxi/uservices/userver/scripts/proto_structs/descriptors/option_parsers.py", + "scripts/proto_structs/descriptors/type_mapping.py":"taxi/uservices/userver/scripts/proto_structs/descriptors/type_mapping.py", "scripts/proto_structs/generator.py":"taxi/uservices/userver/scripts/proto_structs/generator.py", + "scripts/proto_structs/models/__init__.py":"taxi/uservices/userver/scripts/proto_structs/models/__init__.py", + "scripts/proto_structs/models/forward_decls.py":"taxi/uservices/userver/scripts/proto_structs/models/forward_decls.py", + "scripts/proto_structs/models/gen_node.py":"taxi/uservices/userver/scripts/proto_structs/models/gen_node.py", + "scripts/proto_structs/models/includes.py":"taxi/uservices/userver/scripts/proto_structs/models/includes.py", + "scripts/proto_structs/models/includes_bundles.py":"taxi/uservices/userver/scripts/proto_structs/models/includes_bundles.py", + "scripts/proto_structs/models/io.py":"taxi/uservices/userver/scripts/proto_structs/models/io.py", + "scripts/proto_structs/models/names.py":"taxi/uservices/userver/scripts/proto_structs/models/names.py", + "scripts/proto_structs/models/options.py":"taxi/uservices/userver/scripts/proto_structs/models/options.py", + "scripts/proto_structs/models/reserved_identifiers.py":"taxi/uservices/userver/scripts/proto_structs/models/reserved_identifiers.py", + "scripts/proto_structs/models/sort_dependencies.py":"taxi/uservices/userver/scripts/proto_structs/models/sort_dependencies.py", + "scripts/proto_structs/models/toposort.py":"taxi/uservices/userver/scripts/proto_structs/models/toposort.py", + "scripts/proto_structs/models/type_ref.py":"taxi/uservices/userver/scripts/proto_structs/models/type_ref.py", + "scripts/proto_structs/models/type_ref_consts.py":"taxi/uservices/userver/scripts/proto_structs/models/type_ref_consts.py", + "scripts/proto_structs/models/vanilla.py":"taxi/uservices/userver/scripts/proto_structs/models/vanilla.py", + "scripts/proto_structs/templates/io.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/io.inc.jinja", + "scripts/proto_structs/templates/io_forward_declarations.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/io_forward_declarations.inc.jinja", "scripts/proto_structs/templates/structs.usrv.cpp.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/structs.usrv.cpp.jinja", "scripts/proto_structs/templates/structs.usrv.hpp.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/structs.usrv.hpp.jinja", + "scripts/proto_structs/templates/types.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/types.inc.jinja", + "scripts/proto_structs/templates/types_forward_declarations.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/types_forward_declarations.inc.jinja", "scripts/proto_structs/templates/utils.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/utils.inc.jinja", - "scripts/proto_structs/ya.make":"taxi/uservices/userver/scripts/proto_structs/ya.make", "scripts/rabbitmq/ubuntu_install_rabbitmq_dev.sh":"taxi/uservices/userver/scripts/rabbitmq/ubuntu_install_rabbitmq_dev.sh", "scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh":"taxi/uservices/userver/scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh", "scripts/sql/__init__.py":"taxi/uservices/userver/scripts/sql/__init__.py", @@ -4308,6 +4540,7 @@ "testsuite/requirements-grpc-3.txt":"taxi/uservices/userver/testsuite/requirements-grpc-3.txt", "testsuite/requirements-grpc-4.txt":"taxi/uservices/userver/testsuite/requirements-grpc-4.txt", "testsuite/requirements-grpc-5.txt":"taxi/uservices/userver/testsuite/requirements-grpc-5.txt", + "testsuite/requirements-grpc-6.txt":"taxi/uservices/userver/testsuite/requirements-grpc-6.txt", "testsuite/requirements-internal-tests.txt":"taxi/uservices/userver/testsuite/requirements-internal-tests.txt", "testsuite/requirements-mongo.txt":"taxi/uservices/userver/testsuite/requirements-mongo.txt", "testsuite/requirements-postgres.txt":"taxi/uservices/userver/testsuite/requirements-postgres.txt", @@ -4603,6 +4836,7 @@ "universal/include/userver/crypto/public_key.hpp":"taxi/uservices/userver/universal/include/userver/crypto/public_key.hpp", "universal/include/userver/crypto/random.hpp":"taxi/uservices/userver/universal/include/userver/crypto/random.hpp", "universal/include/userver/crypto/signers.hpp":"taxi/uservices/userver/universal/include/userver/crypto/signers.hpp", + "universal/include/userver/crypto/ssl_ctx.hpp":"taxi/uservices/userver/universal/include/userver/crypto/ssl_ctx.hpp", "universal/include/userver/crypto/verifiers.hpp":"taxi/uservices/userver/universal/include/userver/crypto/verifiers.hpp", "universal/include/userver/decimal64/decimal64.hpp":"taxi/uservices/userver/universal/include/userver/decimal64/decimal64.hpp", "universal/include/userver/decimal64/format_options.hpp":"taxi/uservices/userver/universal/include/userver/decimal64/format_options.hpp", @@ -4696,6 +4930,7 @@ "universal/include/userver/logging/format.hpp":"taxi/uservices/userver/universal/include/userver/logging/format.hpp", "universal/include/userver/logging/fwd.hpp":"taxi/uservices/userver/universal/include/userver/logging/fwd.hpp", "universal/include/userver/logging/impl/formatters/base.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/formatters/base.hpp", + "universal/include/userver/logging/impl/log_extra_tskv_formatter.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/log_extra_tskv_formatter.hpp", "universal/include/userver/logging/impl/logger_base.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/logger_base.hpp", "universal/include/userver/logging/impl/mem_logger.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/mem_logger.hpp", "universal/include/userver/logging/impl/tag_writer.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/tag_writer.hpp", @@ -4791,6 +5026,7 @@ "universal/include/userver/utils/string_literal.hpp":"taxi/uservices/userver/universal/include/userver/utils/string_literal.hpp", "universal/include/userver/utils/string_to_duration.hpp":"taxi/uservices/userver/universal/include/userver/utils/string_to_duration.hpp", "universal/include/userver/utils/strong_typedef.hpp":"taxi/uservices/userver/universal/include/userver/utils/strong_typedef.hpp", + "universal/include/userver/utils/strong_typedef_fwd.hpp":"taxi/uservices/userver/universal/include/userver/utils/strong_typedef_fwd.hpp", "universal/include/userver/utils/struct_subsets.hpp":"taxi/uservices/userver/universal/include/userver/utils/struct_subsets.hpp", "universal/include/userver/utils/swappingsmart.hpp":"taxi/uservices/userver/universal/include/userver/utils/swappingsmart.hpp", "universal/include/userver/utils/text_light.hpp":"taxi/uservices/userver/universal/include/userver/utils/text_light.hpp", @@ -4848,6 +5084,7 @@ "universal/src/crypto/random.cpp":"taxi/uservices/userver/universal/src/crypto/random.cpp", "universal/src/crypto/signature_test.cpp":"taxi/uservices/userver/universal/src/crypto/signature_test.cpp", "universal/src/crypto/signers.cpp":"taxi/uservices/userver/universal/src/crypto/signers.cpp", + "universal/src/crypto/ssl_ctx.cpp":"taxi/uservices/userver/universal/src/crypto/ssl_ctx.cpp", "universal/src/crypto/verifiers.cpp":"taxi/uservices/userver/universal/src/crypto/verifiers.cpp", "universal/src/decimal64/decimal64.cpp":"taxi/uservices/userver/universal/src/decimal64/decimal64.cpp", "universal/src/decimal64/decimal64_errors_test.cpp":"taxi/uservices/userver/universal/src/decimal64/decimal64_errors_test.cpp", @@ -4975,6 +5212,7 @@ "universal/src/logging/impl/formatters/struct.hpp":"taxi/uservices/userver/universal/src/logging/impl/formatters/struct.hpp", "universal/src/logging/impl/formatters/tskv.cpp":"taxi/uservices/userver/universal/src/logging/impl/formatters/tskv.cpp", "universal/src/logging/impl/formatters/tskv.hpp":"taxi/uservices/userver/universal/src/logging/impl/formatters/tskv.hpp", + "universal/src/logging/impl/log_extra_tskv_formatter.cpp":"taxi/uservices/userver/universal/src/logging/impl/log_extra_tskv_formatter.cpp", "universal/src/logging/impl/logger_base.cpp":"taxi/uservices/userver/universal/src/logging/impl/logger_base.cpp", "universal/src/logging/impl/mem_logger.cpp":"taxi/uservices/userver/universal/src/logging/impl/mem_logger.cpp", "universal/src/logging/impl/tag_writer.cpp":"taxi/uservices/userver/universal/src/logging/impl/tag_writer.cpp", @@ -5140,6 +5378,7 @@ "universal/utest/src/utest/current_process_open_files_test.cpp":"taxi/uservices/userver/universal/utest/src/utest/current_process_open_files_test.cpp", "universal/utest/src/utest/log_capture_fixture.cpp":"taxi/uservices/userver/universal/utest/src/utest/log_capture_fixture.cpp", "universal/utest/src/utest/parameter_names_test.cpp":"taxi/uservices/userver/universal/utest/src/utest/parameter_names_test.cpp", + "version.txt":"taxi/uservices/userver/version.txt", "ydb/CMakeLists.txt":"taxi/uservices/userver/ydb/CMakeLists.txt", "ydb/dynamic_configs/YDB_DEADLINE_PROPAGATION_VERSION.yaml":"taxi/uservices/userver/ydb/dynamic_configs/YDB_DEADLINE_PROPAGATION_VERSION.yaml", "ydb/dynamic_configs/YDB_QUERIES_COMMAND_CONTROL.yaml":"taxi/uservices/userver/ydb/dynamic_configs/YDB_QUERIES_COMMAND_CONTROL.yaml", diff --git a/.roorules b/.roorules new file mode 100644 index 000000000000..8109bc724043 --- /dev/null +++ b/.roorules @@ -0,0 +1,107 @@ +# Roo Rules (.roorules) +# +# Basic guide for AI agents working with userver service development. +# Contains essential project structure, examples location, and workflow commands. + +project_info: + description: "Basic guide for userver service development" + purpose: "Provides essential structure and workflow for creating userver services" + +# Project structure for userver services +project_structure: + description: "Standard directory structure for userver services" + service_template_location: "service_template/ - Use this as base for new services" + examples_location: "samples/ - Find working examples of various userver features here" + + standard_service_structure: + - "CMakeLists.txt # Build configuration" + - "configs/ # Configuration files" + - " └── static_config.yaml # Static service configuration" + - "src/ # Source code" + - " ├── main.cpp # Service entry point" + - " ├── handlers/ # HTTP handlers" + - " │ ├── handler_name.hpp # Handler declaration" + - " │ └── handler_name.cpp # Handler implementation" + - " └── components/ # Custom components (optional)" + - "tests/ # Tests (optional)" + +# Key files and their purposes +key_files: + main_cpp: + description: "Service entry point - registers components and starts service" + cmake_lists_txt: + description: "Build configuration - links userver components" + static_config_yaml: + description: "Configuration - defines task processors, components, URL mappings" + +# Basic workflow commands for service development +workflow: + description: "Essential commands for userver service development" + + create_service: + description: "Create new service from template" + commands: + - "cp -r service_template/ my_new_service/" + - "cd my_new_service/" + - "# Edit CMakeLists.txt to change project name" + - "# Edit src/main.cpp to register your handlers" + - "# Edit configs/static_config.yaml to configure service" + + build_service: + description: "Build the service" + commands: + - "mkdir build && cd build" + - "cmake .." + - "make -j$(nproc)" + + run_service: + description: "Run the service" + commands: + - "./my_service --config ../configs/static_config.yaml" + + run_tests: + description: "Run tests" + commands: + - "make test" + - "# For functional tests with pytest:" + - "cd tests && python3 -m pytest" + +# Where to find examples +examples: + description: "Look in samples/ directory for working examples" + basic_examples: + - "samples/hello_service/ - Simple HTTP service" + - "samples/postgres_service/ - Service with PostgreSQL" + - "samples/redis_service/ - Service with Redis" + - "samples/grpc_service/ - gRPC service example" + +# Stubs for future expansion +http_client_usage: + description: "TODO: Add HTTP client patterns" + +caching_strategies: + description: "TODO: Add caching patterns" + +middleware_development: + description: "TODO: Add middleware patterns" + +error_handling_logging: + description: "TODO: Add error handling patterns" + +security_best_practices: + description: "TODO: Add security guidelines" + +debugging_profiling: + description: "TODO: Add debugging workflows" + +deployment_release: + description: "TODO: Add deployment processes" + +monitoring_observability: + description: "TODO: Add monitoring setup" + +api_design_principles: + description: "TODO: Add API design guidelines" + +code_review_standards: + description: "TODO: Add code review standards" diff --git a/CMakeLists.txt b/CMakeLists.txt index 221aec71706a..26e86dcafeac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,8 +56,8 @@ if(USERVER_BUILD_TESTS AND NOT USERVER_FEATURE_UTEST) message(FATAL_ERROR "Running unit tests for userver requires utest, disabling it is meaningless") endif() -if(USERVER_INSTALL AND (USERVER_BUILD_TESTS OR USERVER_BUILD_SAMPLES)) - message(FATAL_ERROR "For USERVER_INSTALL, please turn off USERVER_BUILD_TESTS and USERVER_BUILD_SAMPLES " +if(USERVER_INSTALL AND USERVER_BUILD_TESTS) + message(FATAL_ERROR "For USERVER_INSTALL, please turn off USERVER_BUILD_TESTS " "to avoid accidentally installing them" ) endif() @@ -175,6 +175,9 @@ include(UserverSetupEnvironment) userver_setup_environment() include(PrepareInstall) + +include(GetUserverVersion) + include(UserverModule) if(USERVER_INSTALL) @@ -182,7 +185,6 @@ if(USERVER_INSTALL) endif() include(ModuleHelpers) -include(GetUserverVersion) include(AddGoogleTests) include(FindPackageRequired) include(IncludeWhatYouUse) @@ -192,6 +194,9 @@ include(UserverGenerateDynamicConfigsDocs) include(CheckCompileFlags) include(CMakePackageConfigHelpers) +_userver_macos_set_default_dir(OPENSSL_ROOT_DIR "brew;--prefix;openssl") +include(SetupOpenssl) + set(USERVER_THIRD_PARTY_DIRS ${USERVER_ROOT_DIR}/third_party CACHE INTERNAL "" @@ -199,6 +204,7 @@ set(USERVER_THIRD_PARTY_DIRS init_debian_depends() +include(SetupBoost) include(SetupGTest) if(USERVER_FEATURE_GRPC) @@ -324,10 +330,6 @@ if(USERVER_BUILD_SAMPLES AND USERVER_FEATURE_CORE) add_subdirectory(samples) endif() -if(USERVER_INSTALL) - include(cmake/UserverPack.cmake) -endif() - if(USERVER_FEATURE_CORE) _userver_directory_install( COMPONENT core @@ -367,3 +369,11 @@ endif() _userver_add_target_gen_dynamic_configs_docs() _userver_print_features_list() +if(CPM_PACKAGES) + include(DownloadUsingCPM) + _userver_print_cpm_packages() +endif() + +if(USERVER_INSTALL) + include(cmake/UserverPack.cmake) +endif() diff --git a/Makefile b/Makefile index 7162fe89b4e9..146530bbb36b 100644 --- a/Makefile +++ b/Makefile @@ -75,4 +75,13 @@ docker-kill: # clean build folders .PHONY: dist-clean dist-clean: - rm -rf build_* + rm -rf build_*/ + rm -rf debian/ + rm -rf .ruff_cache/ + rm -rf _CPack_Packages/ + find -name .mypy_cache | xargs rm -rf + find -name __pycache__ | xargs rm -rf + +.PHONY: gen-debian-directory +gen-debian-directory: + scripts/generate-debian-directory.sh diff --git a/README.md b/README.md index f24126ff9686..4f03ba8e9bfc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # userver [](https://userver.tech/) [![Ubuntu](https://github.com/userver-framework/userver/actions/workflows/ci.yml/badge.svg)](https://github.com/userver-framework/userver/actions/workflows/ci.yml) +[![Fedora](https://github.com/userver-framework/userver/actions/workflows/fedora.yml/badge.svg)](https://github.com/userver-framework/userver/actions/workflows/fedora.yml) +[![Debian](https://github.com/userver-framework/userver/actions/workflows/debian.yml/badge.svg)](https://github.com/userver-framework/userver/actions/workflows/debian.yml) [![MacOS](https://github.com/userver-framework/userver/actions/workflows/macos.yml/badge.svg)](https://github.com/userver-framework/userver/actions/workflows/macos.yml) [![Alpine](https://github.com/userver-framework/userver/actions/workflows/alpine.yml/badge.svg)](https://github.com/userver-framework/userver/actions/workflows/alpine.yml) [![Docker CI](https://github.com/userver-framework/userver/actions/workflows/docker.yaml/badge.svg)](https://github.com/userver-framework/userver/actions/workflows/docker.yaml) diff --git a/chaotic-openapi/CMakeLists.txt b/chaotic-openapi/CMakeLists.txt index b6ada59033e8..185bb249c759 100644 --- a/chaotic-openapi/CMakeLists.txt +++ b/chaotic-openapi/CMakeLists.txt @@ -8,6 +8,7 @@ userver_module( LINK_LIBRARIES userver-core userver-chaotic UTEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*_test.cpp" UTEST_LINK_LIBRARIES userver-core userver-chaotic + DEPENDS core ) if(USERVER_BUILD_TESTS) @@ -25,7 +26,9 @@ if(USERVER_BUILD_TESTS) ) add_subdirectory(integration_tests) - add_subdirectory(golden_tests) + if (USERVER_CHAOTIC_GOLDEN_TESTS) + add_subdirectory(golden_tests) + endif() endif() _userver_directory_install( diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/middleware.py b/chaotic-openapi/chaotic_openapi/back/cpp_client/middleware.py new file mode 100644 index 000000000000..5ee1b421d054 --- /dev/null +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/middleware.py @@ -0,0 +1,96 @@ +import abc + + +class Middleware(abc.ABC): + """ + Per-operation middleware. + It generates a single struct member for `Request` type. + If we want to represent the structure as a jinja template, + it will look something like that: + + struct Request { + {{ middleware.request_member_cpp_type }} {{ middleware.request_member_name }}; + }; + + void SerializeRequest( + const Request& request, + const std::string& base_url, + USERVER_NAMESPACE::clients::http::Request& http_request + ) + { + openapi::ParameterSinkHttpClient sink( + http_request, + base_url + "{{ op.path }}" + ); + ... + {{ middleware.write_member_command('request', 'sink', 'http_request') }} + } + """ + + @property + @abc.abstractmethod + def request_member_name(self) -> str: + """ + Generated `Request` member name for middleware data. + """ + pass + + @property + @abc.abstractmethod + def request_member_cpp_type(self) -> str: + """ + C++ type of `request_member_name`. + """ + pass + + @property + @abc.abstractmethod + def requests_hpp_includes(self) -> list[str]: + """ + C++ include path used to access `request_member_cpp_type`. + """ + pass + + @abc.abstractmethod + def write_member_command( + self, + request: str, + sink: str, + http_request: str, + ) -> str: + """ + C++ code that will be called as part of `Request` serialization. + + request - variable name of `Request` type + sink - variable name of `ParameterSinkHttpClient` type + http_request - variable name of `clients::http::Request` type + """ + pass + + +class MiddlewarePlugin(abc.ABC): + """ + Implement `MiddlewarePlugin` if you want to extend OpenAPI schema + with your middleware. The schema is extended with + `x-usrv-middleware.` + MiddlewarePlugin.field object member. + + A plugin is created per-parser. + """ + + @property + @abc.abstractmethod + def field(self) -> str: + """ + Returns `x-usrv-middleware` property name that is responsible for + middleware activation and parameters storage. + """ + pass + + @abc.abstractmethod + def create(self, args: dict) -> Middleware: + """ + The method is called every time a HTTP operation with + "x-usrv-middleware." + field property is discovered. + It creates a per-operation `Middleware` object. + """ + pass diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/output.py b/chaotic-openapi/chaotic_openapi/back/cpp_client/output.py index 6987bb990995..f290b0280477 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/output.py +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/output.py @@ -1,11 +1,15 @@ import collections +import dataclasses +import os import pathlib -from typing import Dict -from typing import List +from typing import Any from typing import Optional import yaml +from chaotic.back.cpp import types as cpp_types +from chaotic.front import parser as chaotic_parser +from chaotic.front import ref from . import renderer TYPES_INCLUDES = [ @@ -16,7 +20,11 @@ ] -def _get_template_includes(name: str, client_name: str, graph: Dict[str, List[str]]) -> List[str]: +def filepath_with_stem(fpath: str) -> str: + return fpath.rsplit('.', 1)[0] + + +def _get_template_includes(name: str, client_name: str, graph: dict[str, list[str]]) -> list[str]: includes = { 'client.cpp': [ f'clients/{client_name}/client.hpp', @@ -77,7 +85,6 @@ def _get_template_includes(name: str, client_name: str, graph: Dict[str, List[st 'string', 'variant', *[f'clients/{client_name}/{dep}' for dep in graph], - # TODO ], 'responses.cpp': [ f'clients/{client_name}/responses.hpp', @@ -92,63 +99,95 @@ def _get_template_includes(name: str, client_name: str, graph: Dict[str, List[st f'clients/{client_name}/exceptions.hpp', 'userver/chaotic/openapi/client/exceptions.hpp', *[f'clients/{client_name}/{dep}' for dep in graph], - # TODO ], } return includes[name] -def trim_suffix(string: str, suffix: str) -> Optional[str]: - if string.endswith(suffix): - return string[: -len(suffix)] - return None - - -def extract_includes(name: str, path: pathlib.Path) -> Optional[List[str]]: +def extract_includes(name: str, path: pathlib.Path, schemas_dir: pathlib.Path) -> Optional[list[str]]: with open(path) as ifile: content = yaml.safe_load(ifile) - includes: List[str] = [] + includes: list[str] = [] + is_file_produced = False + + relpath = os.path.relpath(os.path.dirname(path), schemas_dir) def visit(data) -> None: + nonlocal is_file_produced if isinstance(data, dict): for v in data.values(): visit(v) if '$ref' in data: - ref = data['$ref'] - ref = ref.split('#')[0] - ref_fname = ref.rsplit('/', 1)[-1] - if ref_fname: - stem = ref_fname.rsplit('.', 1)[0] + ref_ = data['$ref'] + filename = ref.Ref(ref_).file + if filename: + stem = os.path.join(relpath, filepath_with_stem(filename)) + stem = chaotic_parser.SchemaParser._normalize_ref(stem) includes.append(f'clients/{name}/{stem}.hpp') - if 'x-taxi-cpp-type' in data: - pass - if 'x-usrv-cpp-type' in data: - pass + + for field in ('x-taxi-cpp-type', 'x-usrv-cpp-type'): + if field not in data: + continue + header = extract_cpp_type_header(data[field]) + if header: + includes.append(header) + + if is_file_produced_feature(data): + is_file_produced = True elif isinstance(data, list): for v in data: visit(v) - if 'definitions' in content or 'components' in content or 'paths' in content: - visit(content) + visit(content) + if content.get('definitions') or content.get('components', {}).get('schemas'): + is_file_produced = True + + if includes or is_file_produced: return includes - else: + + # No schemas in file, it will generate no .hpp/.cpp + return None + + +def is_file_produced_feature(data: dict[str, Any]) -> bool: + if data.get('type') == 'object': + return True + if data.get('oneOf'): + return True + if data.get('allOf'): + return True + return False + + +def extract_cpp_type_header(cpp_type: str) -> Optional[str]: + parts = cpp_type.split('::') + if len(parts) < 2: + return None + library = parts[0].replace('_', '-') + if library == 'std': return None + filename = cpp_types.camel_to_snake_case(parts[1]) + return '{}/{}.hpp'.format(library, filename) -def include_graph(name: str, schemas_dir: pathlib.Path) -> Dict[str, List[str]]: +def include_graph(name: str, schemas_dir: pathlib.Path) -> dict[str, list[str]]: result = {} for root, _, filenames in schemas_dir.walk(): for filename in filenames: + if not filename.endswith('.yaml'): + continue filepath = pathlib.Path(root) / filename if filepath == schemas_dir / 'client.yaml' or filename == 'a.yaml': continue - result[filepath.stem + '.hpp'] = extract_includes(name, filepath) + + stem = filepath_with_stem(os.path.relpath(filepath, schemas_dir)) + result[stem + '.hpp'] = extract_includes(name, filepath, schemas_dir) return {key: result[key] for key in result if result[key] is not None} # type: ignore -def get_includes(client_name: str, schemas_dir: str) -> Dict[str, List[str]]: +def get_includes(client_name: str, schemas_dir: str) -> dict[str, list[str]]: graph = include_graph(client_name, pathlib.Path(schemas_dir)) output = collections.defaultdict(list) @@ -160,7 +199,7 @@ def get_includes(client_name: str, schemas_dir: str) -> Dict[str, List[str]]: output[rel_path] = _get_template_includes(name, client_name, graph) for file in graph: - stem = pathlib.Path(file).stem + stem = filepath_with_stem(file) output[f'include/clients/{client_name}/{stem}_fwd.hpp'] = [] output[f'include/clients/{client_name}/{stem}.hpp'] = [ f'clients/{client_name}/{stem}_fwd.hpp', @@ -182,9 +221,19 @@ def get_includes(client_name: str, schemas_dir: str) -> Dict[str, List[str]]: return output -def external_libraries(schemas_dir: str) -> List[str]: +@dataclasses.dataclass +class External: + libraries: list[str] + userver_modules: list[str] + + +# For Yandex uservices only, not for OSS +def external_libraries(schemas_dir: str) -> External: types = set() + libraries = [] + userver_modules = [] + def visit(data) -> None: if isinstance(data, dict): for v in data.values(): @@ -193,6 +242,18 @@ def visit(data) -> None: types.add(data['x-taxi-cpp-type']) if 'x-usrv-cpp-type' in data: types.add(data['x-usrv-cpp-type']) + if 'x-taxi-cpp-typedef-tag' in data: + types.add(data['x-taxi-cpp-typedef-tag']) + if 'x-usrv-cpp-typedef-tag' in data: + types.add(data['x-usrv-cpp-typedef-tag']) + + if data.get('x-taxi-middlewares', {}).get('api-4.0'): + libraries.append('passenger-authorizer-backend') + if data.get('x-taxi-middlewares', {}).get('eats') == 'v1': + libraries.append('eats-authproxy-backend') + if data.get('x-taxi-middlewares', {}).get('bank-authproxy'): + libraries.append('bank-authproxy-backend') + elif isinstance(data, list): for v in data: visit(v) @@ -202,10 +263,14 @@ def visit(data) -> None: content = yaml.safe_load(ifile) visit(content) - libraries = [] for type_ in types: + if type_.startswith('::'): + type_ = type_[2:] library = type_.split('::')[0].replace('_', '-') # special namespaces (and unsigned) which are defined in userver/, not in libraries/ if library not in {'std', 'storages', 'decimal64', 'unsigned'}: libraries.append(library) - return libraries + if type_.startswith('storages::postgres::'): + userver_modules.append('postgresql') + + return External(libraries=libraries, userver_modules=userver_modules) diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/renderer.py b/chaotic-openapi/chaotic_openapi/back/cpp_client/renderer.py index 51fa73fdb543..060ce66a4f73 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/renderer.py +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/renderer.py @@ -1,7 +1,6 @@ import dataclasses import os import pathlib -from typing import List import jinja2 @@ -45,7 +44,7 @@ class CppOutput: content: str @staticmethod - def save(outputs: List['CppOutput'], prefix: str) -> None: + def save(outputs: list['CppOutput'], prefix: str) -> None: for output in outputs: path = os.path.join(prefix, output.rel_path) content = THIS_FILE_IS_AUTOGENERATED + output.content @@ -81,7 +80,7 @@ def make_env() -> jinja2.Environment: JINJA_ENV = make_env() -def render(spec: types.ClientSpec, context: Context) -> List[CppOutput]: +def render(spec: types.ClientSpec, context: Context) -> list[CppOutput]: assert '-' not in spec.cpp_namespace env = { 'spec': spec, @@ -113,7 +112,7 @@ def render(spec: types.ClientSpec, context: Context) -> List[CppOutput]: filepath = cpp_type.json_schema.source_location().filepath vfilepath_map[filepath] = 'clients/{}/{}'.format( spec.client_name, - filepath.split('/')[-1], + filepath, ) # C++ types files diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.cpp.jinja b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.cpp.jinja index 9baab975dfd9..ac3719233454 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.cpp.jinja +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.cpp.jinja @@ -21,6 +21,9 @@ ClientImpl::ClientImpl(const USERVER_NAMESPACE::chaotic::openapi::client::Config const USERVER_NAMESPACE::chaotic::openapi::client::CommandControl& command_control ) { auto r = http_client_.CreateRequest(); + if (plugins_) { + r.SetPluginsList(*plugins_); + } ApplyConfig(r, command_control, config_); {% if not op.empty_request() -%} diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.hpp.jinja b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.hpp.jinja index 85ae34f81813..26ee612ee375 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.hpp.jinja +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/client_impl.hpp.jinja @@ -47,11 +47,16 @@ public: middleware_manager_.RegisterMiddleware(middleware); } + void SetPlugins(std::optional>> plugins) { + plugins_ = std::move(plugins); + } + private: USERVER_NAMESPACE::chaotic::openapi::client::Config config_; USERVER_NAMESPACE::clients::http::Client& http_client_; USERVER_NAMESPACE::chaotic::openapi::MiddlewareManager middleware_manager_; std::unordered_map> middlewares_; + std::optional>> plugins_; }; } diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/component.cpp.jinja b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/component.cpp.jinja index fed655854bd3..a30c75de6073 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/component.cpp.jinja +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/component.cpp.jinja @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include @@ -26,11 +28,23 @@ Component::Component(const USERVER_NAMESPACE::components::ComponentConfig& confi if (config.HasMember("middlewares")) { const auto& mw_config = config["middlewares"]; - for (const auto& [name, config] : Items(mw_config)) { + for (const auto& [name, config] : USERVER_NAMESPACE::formats::common::Items(mw_config)) { auto &factory = context.FindComponent("chaotic-client-middleware-" + name); auto middleware = factory.Create(config); client_.RegisterMiddleware(middleware); } + + const auto& names = config["plugins"].As>>(std::nullopt); + std::optional>> plugins; + if (names) { + plugins.emplace(); + for (const auto& name : *names) { + auto& component = + context.FindComponent(std::string{"http-client-plugin-"} + name); + plugins->emplace_back(&component.GetPlugin()); + } + } + client_.SetPlugins(std::move(plugins)); } } diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/exceptions.hpp.jinja b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/exceptions.hpp.jinja index 8f432cd5e31d..ec5f62ff28ed 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/exceptions.hpp.jinja +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/exceptions.hpp.jinja @@ -38,7 +38,7 @@ class TimeoutException: public USERVER_NAMESPACE::chaotic::openapi::client::Time namespace {{ op.cpp_namespace() }} { /// @brief Base exception class for all client {{ op.method }} operations with URL '{{ op.path }}' -class Exception: public {{ namespace }}::Exception { +class Exception: public ::{{ namespace }}::Exception { public: const char* what() const noexcept override; @@ -50,17 +50,17 @@ class Exception: public {{ namespace }}::Exception { /// @brief Error response with ErrorKind for all client {{ op.method }} operations with URL '{{ op.path }}' class HttpException : public Exception - , public {{ namespace }}::HttpException + , public ::{{ namespace }}::HttpException { public: - using {{ namespace }}::HttpException::HttpException; + using ::{{ namespace }}::HttpException::HttpException; ~HttpException(); }; /// @brief Timeout exception class for all client {{ op.method }} operations with URL '{{ op.path }}' class TimeoutException : public HttpException - , public {{ namespace }}::TimeoutException + , public ::{{ namespace }}::TimeoutException { public: TimeoutException(); @@ -70,10 +70,10 @@ class TimeoutException /// @brief Error response with HTTP status code for all client {{ op.method }} operations with URL '{{ op.path }}' class ExceptionWithStatusCode : public Exception - , public {{ namespace }}::ExceptionWithStatusCode + , public ::{{ namespace }}::ExceptionWithStatusCode { public: - using {{ namespace }}::ExceptionWithStatusCode::ExceptionWithStatusCode; + using ::{{ namespace }}::ExceptionWithStatusCode::ExceptionWithStatusCode; ~ExceptionWithStatusCode(); }; diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.cpp.jinja b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.cpp.jinja index 772147b235b5..d0cc7d4e1a6e 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.cpp.jinja +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.cpp.jinja @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,22 @@ namespace openapi = USERVER_NAMESPACE::chaotic::openapi; static constexpr openapi::Name k{{ parameter.cpp_name }} = "{{ parameter.raw_name }}"; {% endfor %} +{% if op.has_any_hidden_query_parameters() %} + constexpr utils::TrivialSet kHiddenQueries = [](auto selector) { + return selector() + {% for parameter in op.parameters %} + {% if parameter.query_log_mode_hide %} + .Case(k{{ parameter.cpp_name }}) + {% endif %} + {% endfor %} + ; + }; + + bool IsHiddenQueryArg(std::string_view str) { + return kHiddenQueries.Contains(str); + } +{% endif %} + void SerializeRequest(const Request& request, const std::string& base_url, USERVER_NAMESPACE::clients::http::Request& http_request) { {# parameters #} @@ -32,6 +49,11 @@ void SerializeRequest(const Request& request, const std::string& base_url, USERV http_request, base_url + "{{ op.path }}" ); + + {% if op.has_any_hidden_query_parameters() %} + sink.SetHiddenQueryArgNamesFunc(&IsHiddenQueryArg); + {% endif %} + {% for parameter in op.parameters %} {% if parameter.required %} openapi::WriteParameter<{{ parameter.parser }}>(request.{{ parameter.cpp_name}}, sink); @@ -71,6 +93,12 @@ void SerializeRequest(const Request& request, const std::string& base_url, USERV {% endif %} sink.Flush(); + + {# middlewares #} + {% for mw in op.middlewares %} + {{ mw.write_member_command('request', 'sink', 'http_request') }} + {% endfor %} + } } // namespace diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.hpp.jinja b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.hpp.jinja index 5119ff7189ef..fbf4d4b4534a 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.hpp.jinja +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/requests.hpp.jinja @@ -5,6 +5,9 @@ #pragma once #include +{% if spec.has_array_in_request_body() %} +#include +{% endif %} {% if spec.has_multiple_content_type_request() %} #include {% endif %} @@ -46,6 +49,7 @@ namespace {{ namespace }} { {% endfor %} >; {% endif %} + struct Request { {% for parameter in op.parameters %} {% if parameter.cpp_type.nullable %} @@ -55,6 +59,11 @@ namespace {{ namespace }} { {% endif -%} {{ parameter.cpp_name }}; {% endfor %} + + {% for mw in op.middlewares %} + {{ mw.request_member_cpp_type }} {{ mw.request_member_name }}; + {% endfor %} + {% if op.request_bodies %} Body body; {% endif %} diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/responses.cpp.jinja b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/responses.cpp.jinja index d5d7728be1b9..78d74ad335cb 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/responses.cpp.jinja +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/templates/responses.cpp.jinja @@ -5,6 +5,7 @@ #include #include #include +#include {% for include in spec.responses_definitions_includes() %} #include <{{ include }}> @@ -67,7 +68,10 @@ namespace {{ namespace }} { static const USERVER_NAMESPACE::http::headers::PredefinedHeader kHeader("{{ header.raw_name }}"); auto it = http_response.headers().find(kHeader); if (it != http_response.headers().end()) { - r.{{ header.cpp_name }} = it->second; + namespace openapi = chaotic::openapi; + static constexpr openapi::Name k{{ header.cpp_name }} = "{{ header.raw_name }}"; + using Header = {{ header.parser }}; + r.{{ header.cpp_name }} = openapi::ParseParameter::Parse(std::string{it->second}); } } {% endfor %} diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/translator.py b/chaotic-openapi/chaotic_openapi/back/cpp_client/translator.py index 745e78c7c6b8..f9747de6d204 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/translator.py +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/translator.py @@ -1,6 +1,7 @@ import re +import sys +import traceback import typing -from typing import List from typing import Union from chaotic import cpp_names @@ -9,7 +10,9 @@ from chaotic.back.cpp import types as cpp_types from chaotic.front import ref_resolver from chaotic.front import types as chaotic_types +from chaotic_openapi.back.cpp_client import middleware from chaotic_openapi.back.cpp_client import types +from chaotic_openapi.front import base_model from chaotic_openapi.front import model @@ -20,7 +23,8 @@ def __init__( *, cpp_namespace: str, dynamic_config: str, - include_dirs: List[str], + include_dirs: list[str], + middleware_plugins: list[middleware.MiddlewarePlugin], ) -> None: self._spec = types.ClientSpec( client_name=service.name, @@ -30,7 +34,26 @@ def __init__( operations=[], schemas={}, ) + self._include_dirs = include_dirs + self._middleware_plugins = middleware_plugins + + try: + self.translate(service) + except chaotic_error.BaseError: + raise + except BaseException as exc: + print(traceback.format_exc(), file=sys.stderr) + raise chaotic_error.BaseError( + full_filepath=f'<{service.name}>', + infile_path='', + schema_type='openapi/swagger', + msg=str(exc), + ) + def translate( + self, + service: model.Service, + ) -> None: # components/schemas parsed_schemas = chaotic_types.ParsedSchemas( schemas={str(schema.source_location()): schema for schema in service.schemas.values()}, @@ -40,7 +63,7 @@ def __init__( chaotic_translator.GeneratorConfig( namespaces={schema.source_location().filepath: '' for schema in service.schemas.values()}, infile_to_name_func=self.map_infile_path_to_cpp_type, - include_dirs=include_dirs, + include_dirs=self._include_dirs, ) ) self._spec.schemas = gen.generate_types(resolved_schemas) @@ -63,10 +86,12 @@ def __init__( op = types.Operation( method=operation.method.upper(), path=operation.path, + operation_id=operation.operationId, description=operation.description, parameters=[self._translate_parameter(parameter) for parameter in operation.parameters], request_bodies=request_bodies, responses=[self._translate_response(r, status) for status, r in operation.responses.items()], + middlewares=self._translate_middlewares(operation.x_middlewares), ) self._spec.operations.append(op) @@ -77,13 +102,21 @@ def map_infile_path_to_cpp_type(self, name: str, stem: str) -> str: if name.startswith('/components/schemas/'): return '{}::{}'.format( self._spec.cpp_namespace, - name.split('/')[-1], + cpp_names.cpp_identifier(name.split('/')[-1]), ) if name.startswith('/definitions/'): return '{}::{}'.format( self._spec.cpp_namespace, - name.split('/')[-1], + cpp_names.cpp_identifier(name.split('/')[-1]), + ) + + match = re.fullmatch('/paths/\\[([^\\]]*)\\]/([a-zA-Z]*)/requestBody(/schema)?', name) + if match: + return '{}::{}::{}::Body'.format( + self._spec.cpp_namespace, + cpp_names.namespace(match.group(1)), + types.map_method(match.group(2)), ) match = re.fullmatch('/paths/\\[([^\\]]*)\\]/([a-zA-Z]*)/requestBody/content/\\[([^\\]]*)\\]/schema', name) @@ -121,6 +154,16 @@ def map_infile_path_to_cpp_type(self, name: str, stem: str) -> str: ), ) + match = re.fullmatch('/components/responses/([-a-zA-Z_0-9]*)/headers/([^/]*)/schema', name) + if match: + return '{}::Response{}Header{}'.format( + self._spec.cpp_namespace, + match.group(1), + cpp_names.camel_case( + cpp_names.cpp_identifier(match.group(2)), + ), + ) + match = re.fullmatch('/paths/\\[([^\\]]*)\\]/([a-zA-Z]*)/parameters/([0-9]*)(/schema)?', name) if match: return '{}::{}::{}::Parameter{}'.format( @@ -205,6 +248,7 @@ def _translate_single_schema(self, schema: chaotic_types.Schema) -> cpp_types.Cp chaotic_translator.GeneratorConfig( namespaces={schema.source_location().filepath: ''}, infile_to_name_func=self.map_infile_path_to_cpp_type, + include_dirs=self._include_dirs, ) ) gen_types = gen.generate_types( @@ -231,13 +275,14 @@ def _translate_response( for name, header in response.headers.items(): headers.append(self._translate_parameter(header)) + body = {} + for content_type, schema in response.content.items(): + assert isinstance(schema.schema, chaotic_types.Schema) + body[content_type] = self._translate_single_schema(schema.schema) return types.Response( status=status, headers=headers, - body={ - content_type: self._translate_single_schema(schema.schema) - for content_type, schema in response.content.items() - }, + body=body, ) def _translate_request_body(self, request_body: model.RequestBody) -> types.Body: @@ -274,7 +319,9 @@ def _validate_primitive_object(self, schema: cpp_types.CppType) -> None: def _translate_parameter(self, parameter: model.Parameter) -> types.Parameter: in_ = parameter.in_ in_str = in_[0].title() + in_[1:] - cpp_name = cpp_names.cpp_identifier(parameter.name) + cpp_name = cpp_names.cpp_identifier( + parameter.x_cpp_name or parameter.name, + ) if isinstance(parameter.schema, chaotic_types.Array): delimiter = { @@ -309,7 +356,26 @@ def _translate_parameter(self, parameter: model.Parameter) -> types.Parameter: cpp_type=cpp_type, parser=parser, required=parameter.required, + query_log_mode_hide=parameter.x_query_log_mode_hide, ) + def _translate_middlewares(self, middlewares: base_model.XMiddlewares) -> list[middleware.Middleware]: + extra_members = middlewares.__pydantic_extra__ or {} + + result = [] + for plugin in self._middleware_plugins: + field = plugin.field + if field not in extra_members: + continue + + mw = plugin.create({field: extra_members[field]}) + result.append(mw) + del extra_members[field] + + if extra_members: + raise Exception(f'Unknown member(s) in x-taxi-middlewares: {extra_members}') + + return result + def spec(self) -> types.ClientSpec: return self._spec diff --git a/chaotic-openapi/chaotic_openapi/back/cpp_client/types.py b/chaotic-openapi/chaotic_openapi/back/cpp_client/types.py index 3f13c23230d2..be4e6d5a99a8 100644 --- a/chaotic-openapi/chaotic_openapi/back/cpp_client/types.py +++ b/chaotic-openapi/chaotic_openapi/back/cpp_client/types.py @@ -1,13 +1,12 @@ import dataclasses -from typing import Dict from typing import Generator -from typing import List from typing import Optional -from typing import Set +from typing import Union from chaotic import cpp_names from chaotic import error from chaotic.back.cpp import types as cpp_types +from . import middleware @dataclasses.dataclass @@ -19,14 +18,15 @@ class Parameter: parser: str required: bool - def declaration_includes(self) -> List[str]: + query_log_mode_hide: bool + + def declaration_includes(self) -> list[str]: # TODO if not self.required: return ['optional'] return [] - @classmethod - def _validate_schema(cls, schema: cpp_types.CppType, *, is_array_allowed: bool) -> None: + def _validate_schema(self, schema: cpp_types.CppType, *, is_array_allowed: bool) -> None: assert schema.json_schema if not is_array_allowed and isinstance(schema, cpp_types.CppArray): source_location = schema.json_schema.source_location() @@ -53,17 +53,28 @@ def _validate_schema(cls, schema: cpp_types.CppType, *, is_array_allowed: bool) full_filepath=source_location.filepath, infile_path=source_location.location, schema_type='jsonschema', - msg='Unsupported parameter type', + msg=f'Unsupported parameter type for parameter "{self.raw_name}"', ) if isinstance(schema, cpp_types.CppRef): - cls._validate_schema(schema.orig_cpp_type, is_array_allowed=is_array_allowed) + self._validate_schema(schema.orig_cpp_type, is_array_allowed=is_array_allowed) if isinstance(schema, cpp_types.CppArray): - cls._validate_schema(schema.items, is_array_allowed=False) + self._validate_schema(schema.items, is_array_allowed=False) def __post_init__(self) -> None: self._validate_schema(self.cpp_type, is_array_allowed=True) + # TODO: for handler only + def span_value(self) -> str: + if self.required: + arg = self.cpp_name + else: + arg = '*' + self.cpp_name + if self.cpp_type.cpp_user_name() == 'std::string': + return arg + else: + return f'::fmt::format("{{}}", {arg})' + @dataclasses.dataclass class Body: @@ -79,8 +90,8 @@ def cpp_name(self) -> str: @dataclasses.dataclass class Response: status: int - body: Dict[str, cpp_types.CppType] - headers: List[Parameter] = dataclasses.field(default_factory=list) + body: dict[str, cpp_types.CppType] + headers: list[Parameter] = dataclasses.field(default_factory=list) def is_error(self) -> bool: return self.status >= 400 @@ -108,18 +119,25 @@ def map_method(method: str) -> str: class Operation: method: str path: str + operation_id: Union[str, None] description: str = '' - parameters: List[Parameter] = dataclasses.field(default_factory=list) - request_bodies: List[Body] = dataclasses.field(default_factory=list) - responses: List[Response] = dataclasses.field(default_factory=list) + parameters: list[Parameter] = dataclasses.field(default_factory=list) + request_bodies: list[Body] = dataclasses.field(default_factory=list) + responses: list[Response] = dataclasses.field(default_factory=list) client_generate: bool = True + middlewares: list[middleware.Middleware] = dataclasses.field(default_factory=list) + def cpp_namespace(self) -> str: + if self.operation_id: + return cpp_names.namespace(self.operation_id) return cpp_names.namespace(self.path) + '::' + map_method(self.method) def cpp_method_name(self) -> str: + if self.operation_id: + return self.operation_id return cpp_names.camel_case( cpp_names.namespace(self.path) + '_' + map_method(self.method), ) @@ -148,6 +166,12 @@ def iter_error_responses(self) -> Generator[Response, None, None]: if response.is_error(): yield response + def has_any_hidden_query_parameters(self) -> bool: + for parameter in self.parameters: + if parameter.query_log_mode_hide: + return True + return False + @dataclasses.dataclass class ClientSpec: @@ -155,12 +179,12 @@ class ClientSpec: cpp_namespace: str dynamic_config: str description: str = '' - operations: List[Operation] = dataclasses.field(default_factory=list) + operations: list[Operation] = dataclasses.field(default_factory=list) # Internal types which cannot be referred to - internal_schemas: Dict[str, cpp_types.CppType] = dataclasses.field(default_factory=dict) + internal_schemas: dict[str, cpp_types.CppType] = dataclasses.field(default_factory=dict) # Types which can be referred to - schemas: Dict[str, cpp_types.CppType] = dataclasses.field(default_factory=dict) + schemas: dict[str, cpp_types.CppType] = dataclasses.field(default_factory=dict) def has_multiple_content_type_request(self) -> bool: for op in self.operations: @@ -168,8 +192,15 @@ def has_multiple_content_type_request(self) -> bool: return True return False - def requests_declaration_includes(self) -> List[str]: - includes: Set[str] = set() + def has_array_in_request_body(self) -> bool: + for op in self.operations: + for body in op.request_bodies: + if isinstance(body.schema, cpp_types.CppArray): + return True + return False + + def requests_declaration_includes(self) -> list[str]: + includes: set[str] = set() for op in self.operations: if not op.client_generate: continue @@ -180,10 +211,13 @@ def requests_declaration_includes(self) -> List[str]: for param in op.parameters: includes.update(param.declaration_includes()) + for mw in op.middlewares: + includes.update(mw.requests_hpp_includes) + return sorted(includes) - def responses_declaration_includes(self) -> List[str]: - includes: Set[str] = set() + def responses_declaration_includes(self) -> list[str]: + includes: set[str] = set() for op in self.operations: if not op.client_generate: continue @@ -195,8 +229,8 @@ def responses_declaration_includes(self) -> List[str]: includes.update(header.declaration_includes()) return sorted(includes) - def responses_definitions_includes(self) -> List[str]: - includes: Set[str] = set() + def responses_definitions_includes(self) -> list[str]: + includes: set[str] = set() for op in self.operations: if not op.client_generate: continue @@ -206,7 +240,7 @@ def responses_definitions_includes(self) -> List[str]: includes.update(body.definition_includes()) return sorted(includes) - def cpp_includes(self) -> List[str]: + def cpp_includes(self) -> list[str]: includes = [] for cpp_type in self.extract_cpp_types().values(): assert cpp_type.json_schema @@ -214,12 +248,12 @@ def cpp_includes(self) -> List[str]: includes.append( 'clients/{}/{}.hpp'.format( self.client_name, - filepath.split('/')[-1].split('.')[0], + filepath.rsplit('.', 1)[0], ), ) return includes - def extract_cpp_types(self) -> Dict[str, cpp_types.CppType]: + def extract_cpp_types(self) -> dict[str, cpp_types.CppType]: types = self.schemas.copy() types.update(self.internal_schemas) diff --git a/chaotic-openapi/chaotic_openapi/back/linter/validators.py b/chaotic-openapi/chaotic_openapi/back/linter/validators.py index e0ae6d6ee1c3..10df136ec8a5 100644 --- a/chaotic-openapi/chaotic_openapi/back/linter/validators.py +++ b/chaotic-openapi/chaotic_openapi/back/linter/validators.py @@ -26,7 +26,6 @@ def validate_nonobject_body(service: model.Service) -> None: if not isinstance( schema, (types.SchemaObject, types.OneOfWithDiscriminator, types.OneOfWithoutDiscriminator) ): - print(schema, file=sys.stderr) report_error('non-object-body', schema.source_location(), 'Non-object type in body root is forbidden') diff --git a/chaotic-openapi/chaotic_openapi/front/base_model.py b/chaotic-openapi/chaotic_openapi/front/base_model.py index 527cfbe141e7..fd147d5c1a81 100644 --- a/chaotic-openapi/chaotic_openapi/front/base_model.py +++ b/chaotic-openapi/chaotic_openapi/front/base_model.py @@ -1,12 +1,11 @@ from typing import Any -from typing import List import pydantic class BaseModel(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra='allow') - _model_userver_tags: List[str] = [] + _model_userver_tags: list[str] = [] def model_post_init(self, context: Any) -> None: super().model_post_init(context) @@ -14,7 +13,9 @@ def model_post_init(self, context: Any) -> None: if not self.__pydantic_extra__: return for field in self.__pydantic_extra__: - if field.startswith('x-taxi-py3-'): + if field.startswith('x-taxi-py3'): + continue + if field.startswith('x-taxi-go-'): continue if field.startswith('x-taxi-') or field.startswith('x-usrv-'): assert field in self._model_userver_tags, f'Field {field} is not allowed in this context' @@ -23,3 +24,11 @@ def model_post_init(self, context: Any) -> None: if field.startswith('x-'): continue raise Exception(f'Unknown field "{field}"') + + +class XMiddlewares(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra='allow') + + tvm: bool = True + + # TODO: validate field set diff --git a/chaotic-openapi/chaotic_openapi/front/model.py b/chaotic-openapi/chaotic_openapi/front/model.py index 73799c1b9fbd..b6f17b696438 100644 --- a/chaotic-openapi/chaotic_openapi/front/model.py +++ b/chaotic-openapi/chaotic_openapi/front/model.py @@ -1,12 +1,11 @@ import dataclasses import enum from typing import Any -from typing import Dict -from typing import List from typing import Optional from typing import Union from chaotic.front import types +from . import base_model class In(str, enum.Enum): @@ -32,7 +31,7 @@ class Parameter: name: str in_: In description: str - examples: Dict[str, Any] + examples: dict[str, Any] deprecated: bool required: bool @@ -41,18 +40,21 @@ class Parameter: style: Style schema: types.Schema + x_cpp_name: Optional[str] + x_query_log_mode_hide: bool + @dataclasses.dataclass class MediaType: - schema: Union[types.Schema, types.Ref] - examples: Dict[str, Any] + schema: Union[types.Schema, types.Ref, None] + examples: dict[str, Any] @dataclasses.dataclass class Response: description: str - headers: Dict[str, Parameter] - content: Dict[str, MediaType] + headers: dict[str, Parameter] + content: dict[str, MediaType] @dataclasses.dataclass @@ -87,7 +89,7 @@ class ApiKeySecurity(Security): @dataclasses.dataclass class Flow: refreshUrl: str - scopes: Dict[str, str] + scopes: dict[str, str] @dataclasses.dataclass @@ -113,7 +115,7 @@ class AuthCodeFlow(Flow): @dataclasses.dataclass class OAuthSecurity(Security): - flows: List[Flow] + flows: list[Flow] @dataclasses.dataclass @@ -132,22 +134,25 @@ class Operation: description: str path: str method: str - operationId: str - parameters: List[Parameter] - requestBody: Union[List[RequestBody], Ref] - responses: Dict[int, Union[Response, Ref]] - security: List[Security] + operationId: Union[str, None] + parameters: list[Parameter] + requestBody: Union[list[RequestBody], Ref] + responses: dict[int, Union[Response, Ref]] + security: list[Security] + + x_client_codegen: bool + x_middlewares: base_model.XMiddlewares @dataclasses.dataclass class Service: name: str description: str = '' - operations: List[Operation] = dataclasses.field(default_factory=list) - - schemas: Dict[str, types.Schema] = dataclasses.field(default_factory=dict) - responses: Dict[str, Response] = dataclasses.field(default_factory=dict) - parameters: Dict[str, Parameter] = dataclasses.field(default_factory=dict) - headers: Dict[str, Parameter] = dataclasses.field(default_factory=dict) - requestBodies: Dict[str, List[RequestBody]] = dataclasses.field(default_factory=dict) - security: Dict[str, Security] = dataclasses.field(default_factory=dict) + operations: list[Operation] = dataclasses.field(default_factory=list) + + schemas: dict[str, types.Schema] = dataclasses.field(default_factory=dict) + responses: dict[str, Response] = dataclasses.field(default_factory=dict) + parameters: dict[str, Parameter] = dataclasses.field(default_factory=dict) + headers: dict[str, Parameter] = dataclasses.field(default_factory=dict) + requestBodies: dict[str, list[RequestBody]] = dataclasses.field(default_factory=dict) + security: dict[str, Security] = dataclasses.field(default_factory=dict) diff --git a/chaotic-openapi/chaotic_openapi/front/openapi.py b/chaotic-openapi/chaotic_openapi/front/openapi.py index 9e8216aa8dd2..718f8d6f60b3 100644 --- a/chaotic-openapi/chaotic_openapi/front/openapi.py +++ b/chaotic-openapi/chaotic_openapi/front/openapi.py @@ -1,7 +1,5 @@ import enum from typing import Any -from typing import Dict -from typing import List from typing import Optional from typing import Union @@ -25,7 +23,7 @@ class Info(base_model.BaseModel): class Server(base_model.BaseModel): url: str description: Optional[str] = None - variables: Dict[str, Any] = pydantic.Field(default_factory=dict) + variables: dict[str, Any] = pydantic.Field(default_factory=dict) Schema = Any @@ -54,15 +52,20 @@ class Header(base_model.BaseModel): allowReserved: bool = False schema_: Schema = pydantic.Field(alias='schema') example: Any = None - examples: Dict[str, Any] = pydantic.Field(default_factory=dict) + examples: dict[str, Any] = pydantic.Field(default_factory=dict) # https://spec.openapis.org/oas/v3.0.0.html#media-type-object class MediaType(base_model.BaseModel): schema_: Schema = pydantic.Field(alias='schema', default=None) example: Any = None - examples: Dict[str, Any] = pydantic.Field(default_factory=dict) - # encoding: Dict[str, Encoding] = {} + examples: dict[str, Any] = pydantic.Field(default_factory=dict) + # encoding: dict[str, Encoding] = {} + + _model_userver_tags: list[str] = [ + 'x-taxi-non-std-type-reason', + 'x-usrv-non-std-type-reason', + ] # https://spec.openapis.org/oas/v3.0.0.html#reference-object @@ -73,10 +76,15 @@ class Ref(base_model.BaseModel): # https://spec.openapis.org/oas/v3.0.0.html#responses-object class Response(base_model.BaseModel): description: str - headers: Dict[str, Union[Header, Ref]] = pydantic.Field(default_factory=dict) - content: Dict[str, MediaType] = pydantic.Field(default_factory=dict) + headers: dict[str, Union[Header, Ref]] = pydantic.Field(default_factory=dict) + content: dict[str, MediaType] = pydantic.Field(default_factory=dict) # TODO: links + def model_post_init(self, context: Any, /) -> None: + if 'application/json' in self.content and not self.content['application/json'].schema_: + # empty application/json means the same "no body" + del self.content['application/json'] + class In(str, enum.Enum): path = 'path' @@ -85,6 +93,11 @@ class In(str, enum.Enum): cookie = 'cookie' +class QueryLogMode(str, enum.Enum): + show = 'show' + hide = 'hide' + + # https://spec.openapis.org/oas/v3.0.0.html#parameter-object class Parameter(base_model.BaseModel): name: str @@ -99,9 +112,26 @@ class Parameter(base_model.BaseModel): allowReserved: bool = False schema_: Schema = pydantic.Field(alias='schema') example: Any = None - examples: Dict[str, Any] = pydantic.Field(default_factory=dict) + examples: dict[str, Any] = pydantic.Field(default_factory=dict) - # content: Dict[str, MediaType] = {} + # content: dict[str, MediaType] = {} + + x_handler_tag: Optional[str] = pydantic.Field( + default=None, + validation_alias=pydantic.AliasChoices('x-taxi-handler-tag', 'x-usrv-handler-tag'), + ) + x_cpp_name: Optional[str] = pydantic.Field( + default=None, + validation_alias=pydantic.AliasChoices('x-taxi-cpp-name', 'x-usrv-cpp-name'), + ) + x_query_log_mode: QueryLogMode = pydantic.Field( + default=QueryLogMode.show, + validation_alias=pydantic.AliasChoices('x-taxi-query-log-mode', 'x-usrv-query-log-mode'), + ) + x_explode_true_reason: str = pydantic.Field( + default='', + validation_alias=pydantic.AliasChoices('x-taxi-explode-true-reason', 'x-usrv-explode-true-reason'), + ) def model_post_init(self, context: Any, /) -> None: super().model_post_init(context) @@ -119,7 +149,7 @@ def model_post_init(self, context: Any, /) -> None: # https://spec.openapis.org/oas/v3.0.0.html#request-body-object class RequestBody(base_model.BaseModel): description: Optional[str] = None - content: Dict[str, MediaType] + content: dict[str, MediaType] required: bool = False @@ -138,25 +168,25 @@ class SecurityIn(str, enum.Enum): class ImplicitFlow(base_model.BaseModel): refreshUrl: Optional[str] = None - scopes: Dict[str, str] = pydantic.Field(default_factory=dict) + scopes: dict[str, str] = pydantic.Field(default_factory=dict) authorizationUrl: str class PasswordFlow(base_model.BaseModel): refreshUrl: Optional[str] = None - scopes: Dict[str, str] = pydantic.Field(default_factory=dict) + scopes: dict[str, str] = pydantic.Field(default_factory=dict) tokenUrl: str class ClientCredFlow(base_model.BaseModel): refreshUrl: Optional[str] = None - scopes: Dict[str, str] = pydantic.Field(default_factory=dict) + scopes: dict[str, str] = pydantic.Field(default_factory=dict) tokenUrl: str class AuthCodeFlow(base_model.BaseModel): refreshUrl: Optional[str] = None - scopes: Dict[str, str] = pydantic.Field(default_factory=dict) + scopes: dict[str, str] = pydantic.Field(default_factory=dict) authorizationUrl: str tokenUrl: str @@ -199,47 +229,65 @@ def model_post_init(self, context: Any, /) -> None: raise ValueError(errors.missing_field_msg('openIdConnectUrl')) -SecuritySchemes = Dict[str, Union[SecurityScheme, Ref]] +SecuritySchemes = dict[str, Union[SecurityScheme, Ref]] # https://spec.openapis.org/oas/v3.0.0.html#security-requirement-object -Security = Dict[str, List[str]] +Security = dict[str, list[str]] # https://spec.openapis.org/oas/v3.0.0.html#components-object class Components(base_model.BaseModel): - schemas: Dict[str, Schema] = pydantic.Field(default_factory=dict) - requests: Dict[str, Any] = pydantic.Field(default_factory=dict) # TODO - responses: Dict[str, Response] = pydantic.Field(default_factory=dict) - parameters: Dict[str, Parameter] = pydantic.Field(default_factory=dict) - headers: Dict[str, Header] = pydantic.Field(default_factory=dict) - requestBodies: Dict[str, RequestBody] = pydantic.Field(default_factory=dict) + schemas: dict[str, Schema] = pydantic.Field(default_factory=dict) + requests: dict[str, Any] = pydantic.Field(default_factory=dict) # TODO + responses: dict[str, Response] = pydantic.Field(default_factory=dict) + parameters: dict[str, Parameter] = pydantic.Field(default_factory=dict) + headers: dict[str, Header] = pydantic.Field(default_factory=dict) + requestBodies: dict[str, RequestBody] = pydantic.Field(default_factory=dict) securitySchemes: SecuritySchemes = pydantic.Field(default_factory=dict) -class XTaxiMiddlewares(base_model.BaseModel): - tvm: bool = True - - # https://spec.openapis.org/oas/v3.0.0.html#operation-object class Operation(base_model.BaseModel): - tags: List[str] = pydantic.Field(default_factory=list) + tags: list[str] = pydantic.Field(default_factory=list) summary: Optional[str] = None description: str = '' externalDocs: Any = None operationId: Optional[str] = None - parameters: List[Union[Parameter, Ref]] = pydantic.Field(default_factory=list) + parameters: list[Union[Parameter, Ref]] = pydantic.Field(default_factory=list) requestBody: Optional[Union[RequestBody, Ref]] = None - responses: Dict[Union[str, int], Union[Response, Ref]] + responses: dict[Union[str, int], Union[Response, Ref]] deprecated: bool = False security: Optional[Security] = None - servers: List[Server] = pydantic.Field(default_factory=list) + servers: list[Server] = pydantic.Field(default_factory=list) - x_taxi_middlewares: Optional[XTaxiMiddlewares] = pydantic.Field( + x_taxi_middlewares: Optional[base_model.XMiddlewares] = pydantic.Field( default=None, - alias='x-taxi-middlewares', + validation_alias=pydantic.AliasChoices('x-taxi-middlewares', 'x-usrv-middlewares'), + ) + x_taxi_handler_codegen: bool = pydantic.Field( + default=True, + validation_alias=pydantic.AliasChoices('x-taxi-handler-codegen', 'x-usrv-handler-codegen'), + ) + x_query_log_mode: QueryLogMode = pydantic.Field( + default=QueryLogMode.show, + validation_alias=pydantic.AliasChoices('x-taxi-query-log-mode', 'x-usrv-query-log-mode'), ) + x_client_codegen: bool = pydantic.Field( + default=True, + validation_alias=pydantic.AliasChoices('x-taxi-client-codegen', 'x-usrv-client-codegen'), + ) + + def model_post_init(self, context: Any, /) -> None: + super().model_post_init(context) + + if self.x_query_log_mode == QueryLogMode.hide: + for parameter in self.parameters: + if not isinstance(parameter, Parameter): + continue + if parameter.in_ == In.query: + parameter.x_query_log_mode = QueryLogMode.hide # https://spec.openapis.org/oas/v3.0.0.html#path-item-object @@ -256,8 +304,8 @@ class Path(base_model.BaseModel): patch: Optional[Operation] = None trace: Optional[Operation] = None - servers: List[Server] = pydantic.Field(default_factory=list) - parameters: List[Union[Parameter, Ref]] = pydantic.Field(default_factory=list) + servers: list[Server] = pydantic.Field(default_factory=list) + parameters: list[Union[Parameter, Ref]] = pydantic.Field(default_factory=list) class XTaxiClientQos(base_model.BaseModel): @@ -268,20 +316,20 @@ class XTaxiClientQos(base_model.BaseModel): class OpenApi(base_model.BaseModel): openapi: str = '3.0.0' info: Optional[Info] = None - servers: List[Server] = pydantic.Field(default_factory=list) - paths: Dict[str, Path] = pydantic.Field(default_factory=dict) + servers: list[Server] = pydantic.Field(default_factory=list) + paths: dict[str, Path] = pydantic.Field(default_factory=dict) components: Components = Components() security: Security = pydantic.Field(default_factory=dict) - tags: List[Any] = pydantic.Field(default_factory=list) + tags: list[Any] = pydantic.Field(default_factory=list) externalDocs: Any = None x_taxi_client_qos: Optional[XTaxiClientQos] = pydantic.Field( default=None, - alias='x-taxi-client-qos', + validation_alias=pydantic.AliasChoices('x-taxi-client-qos', 'x-usrv-client-qos'), ) - x_taxi_middlewares: Optional[XTaxiMiddlewares] = pydantic.Field( + x_taxi_middlewares: Optional[base_model.XMiddlewares] = pydantic.Field( default=None, - alias='x-taxi-middlewares', + validation_alias=pydantic.AliasChoices('x-taxi-middlewares', 'x-usrv-middlewares'), ) def validate_security(self, security: Optional[Security]) -> None: diff --git a/chaotic-openapi/chaotic_openapi/front/parser.py b/chaotic-openapi/chaotic_openapi/front/parser.py index fecc7696d962..caa831714e29 100644 --- a/chaotic-openapi/chaotic_openapi/front/parser.py +++ b/chaotic-openapi/chaotic_openapi/front/parser.py @@ -3,17 +3,16 @@ import typing from typing import Any from typing import Callable -from typing import Dict -from typing import List from typing import Optional -from typing import Tuple from typing import Union import pydantic -from chaotic import cpp_names +from chaotic import error as chaotic_error from chaotic.front import parser as chaotic_parser +from chaotic.front import ref from chaotic.front import types +from . import base_model from . import errors from . import model from . import openapi @@ -24,7 +23,10 @@ @dataclasses.dataclass class ParserState: service: model.Service + # Full filepath in filesystem as-is full_filepath: str = '' + # Virtual filepath for .cpp/.hpp file generation (e.g. relative paths) + full_vfilepath: str = '' class Parser: @@ -34,8 +36,9 @@ def __init__( ) -> None: self._state = ParserState(service=model.Service(name=name)) - def parse_schema(self, schema: dict, full_filepath: str) -> None: + def parse_schema(self, schema: dict, full_filepath: str, full_vfilepath: str) -> None: self._state.full_filepath = full_filepath + self._state.full_vfilepath = full_vfilepath parser = self._guess_parser(schema) try: parsed = parser(**schema) @@ -82,6 +85,8 @@ def _convert_swagger_header(self, name: str, header: swagger.Header, infile_path allowEmptyValue=False, style=model.Style.simple, schema=self._parse_schema(header_dict, infile_path + '/schema'), + x_cpp_name=None, + x_query_log_mode_hide=False, ) def _convert_media_type( @@ -99,11 +104,11 @@ def _convert_media_type( RIGHT_SLASH_RE = re.compile('/[^/]*$') - def _locate_ref(self, ref: str) -> str: - if ref.startswith('#'): - return self._state.full_filepath + ref + def _locate_ref(self, ref_: str) -> str: + if not ref.Ref(ref_).file: + return self._state.full_filepath + ref_ cur = re.sub(self.RIGHT_SLASH_RE, '/', self._state.full_filepath) - return chaotic_parser.SchemaParser._normalize_ref(cur + ref) + return chaotic_parser.SchemaParser._normalize_ref(cur + ref_) def _convert_openapi_parameter( self, @@ -122,13 +127,15 @@ def _convert_openapi_parameter( allowEmptyValue=parameter.allowEmptyValue, style=model.Style(parameter.style), schema=self._parse_schema(parameter.schema_, infile_path + '/schema'), + x_cpp_name=parameter.x_cpp_name, + x_query_log_mode_hide=(parameter.x_query_log_mode == openapi.QueryLogMode.hide), ) return p def _is_swagger_request_body( self, parameter: Union[swagger.Parameter, swagger.Ref], - global_params: Dict[str, Union[model.Parameter, List[model.RequestBody]]], + global_params: dict[str, Union[model.Parameter, list[model.RequestBody]]], ) -> bool: if isinstance(parameter, swagger.Ref): return isinstance(global_params[self._locate_ref(parameter.ref)], list) @@ -139,14 +146,14 @@ def _convert_swagger_request_body( self, request_body: Union[swagger.Parameter, swagger.Ref], infile_path: str, - consumes: List[str] = [], - ) -> Union[List[model.RequestBody], model.Ref]: + consumes: list[str] = [], + ) -> Union[list[model.RequestBody], model.Ref]: if isinstance(request_body, swagger.Ref): - ref = ref_resolver.normalize_ref( + ref_ = ref_resolver.normalize_ref( self._state.full_filepath, request_body.ref, ) - return model.Ref(ref) + return model.Ref(ref_) if request_body.in_ == swagger.In.body: return [ @@ -160,6 +167,15 @@ def _convert_swagger_request_body( assert request_body.in_ == swagger.In.formData + if request_body.type == 'file': + if 'multipart/form-data' not in consumes and 'application/x-www-form-urlencoded' not in consumes: + raise chaotic_error.BaseError( + full_filepath=self._state.full_filepath, + infile_path=infile_path, + schema_type='swagger', + msg='"consumes" must be either "multipart/form-data" or "application/x-www-form-urlencoded" for "type: file"', + ) + schema = self._parse_schema( request_body.model_dump( by_alias=True, @@ -167,6 +183,7 @@ def _convert_swagger_request_body( exclude_unset=True, ), infile_path, + allow_file=True, ) return [ model.RequestBody( @@ -190,6 +207,7 @@ def _convert_swagger_parameter( exclude_unset=True, ), infile_path, + allow_file=True, ) style: model.Style @@ -208,6 +226,8 @@ def _convert_swagger_parameter( allowEmptyValue=parameter.allowEmptyValue, style=style, schema=schema, + x_cpp_name=parameter.x_cpp_name, + x_query_log_mode_hide=False, ) return p @@ -220,12 +240,15 @@ def _convert_openapi_response( assert infile_path.count('#') <= 1 if isinstance(response, openapi.Ref): - ref = ref_resolver.normalize_ref( + ref_ = ref_resolver.normalize_ref( self._state.full_filepath, response.ref, ) - assert ref.count('#') == 1, ref - return model.Ref(ref) + + # validate + ref.Ref(ref_) + + return model.Ref(ref_) content = {} for content_type, openapi_content in response.content.items(): @@ -243,17 +266,20 @@ def _convert_openapi_response( ) def _convert_swagger_response( - self, response: Union[swagger.Response, swagger.Ref], produces: List[str], infile_path: str + self, response: Union[swagger.Response, swagger.Ref], produces: list[str], infile_path: str ) -> Union[model.Response, model.Ref]: assert infile_path.count('#') <= 1 if isinstance(response, swagger.Ref): - ref = ref_resolver.normalize_ref( + ref_ = ref_resolver.normalize_ref( self._state.full_filepath, response.ref, ) - assert ref.count('#') == 1, ref - return model.Ref(ref) + + # validate + ref.Ref(ref_) + + return model.Ref(ref_) if response.schema_: schema = self._parse_schema(response.schema_, infile_path + '/schema') @@ -275,13 +301,13 @@ def _convert_openapi_request_body( self, request_body: Union[openapi.RequestBody, openapi.Ref], infile_path: str, - ) -> Union[List[model.RequestBody], model.Ref]: + ) -> Union[list[model.RequestBody], model.Ref]: if isinstance(request_body, openapi.Ref): - ref = ref_resolver.normalize_ref( + ref_ = ref_resolver.normalize_ref( self._state.full_filepath, request_body.ref, ) - return model.Ref(ref) + return model.Ref(ref_) requestBody = [] for content_type, media_type in request_body.content.items(): @@ -298,8 +324,8 @@ def _convert_openapi_request_body( ) return requestBody - def _convert_openapi_flows(self, flows: openapi.OAuthFlows) -> List[model.Flow]: - model_flows: List[model.Flow] = [] + def _convert_openapi_flows(self, flows: openapi.OAuthFlows) -> list[model.Flow]: + model_flows: list[model.Flow] = [] if flows.implicit: implicit = flows.implicit refreshUrl = implicit.refreshUrl or '' @@ -322,7 +348,7 @@ def _convert_openapi_flows(self, flows: openapi.OAuthFlows) -> List[model.Flow]: return model_flows def _convert_openapi_securuty( - self, security_scheme: Union[openapi.SecurityScheme, openapi.Ref], flows_scopes: Optional[List[str]] = None + self, security_scheme: Union[openapi.SecurityScheme, openapi.Ref], flows_scopes: Optional[list[str]] = None ) -> model.Security: if isinstance(security_scheme, openapi.Ref): return self._state.service.security[self._locate_ref(security_scheme.ref)] @@ -350,7 +376,7 @@ def _convert_openapi_securuty( assert False def _convert_swagger_security( - self, security_def: swagger.SecurityDef, flows_scopes: Optional[List[str]] = None + self, security_def: swagger.SecurityDef, flows_scopes: Optional[list[str]] = None ) -> model.Security: description = security_def.description or '' if security_def.type == swagger.SecurityType.basic: @@ -388,7 +414,7 @@ def _append_schema( self, parsed: Union[openapi.OpenApi, swagger.Swagger], ) -> None: - components_schemas: Dict[str, Any] = {} + components_schemas: dict[str, Any] = {} components_schemas_path = '' if isinstance(parsed, openapi.OpenApi): parsed = typing.cast(openapi.OpenApi, parsed) @@ -414,11 +440,11 @@ def _append_schema( security_scheme = self._convert_openapi_securuty(sec_scheme) self._state.service.security[self._state.full_filepath + '#' + infile_path] = security_scheme - def _convert_op_security(security: Optional[openapi.Security]) -> List[model.Security]: + def _convert_op_security(security: Optional[openapi.Security]) -> list[model.Security]: if not security: security = default_security - securities: List[model.Security] = [] + securities: list[model.Security] = [] for name, scopes in security.items(): securities.append(self._convert_openapi_securuty(security_schemas[name], scopes)) @@ -451,7 +477,7 @@ def _convert_op_security(security: Optional[openapi.Security]) -> List[model.Sec # paths for path, path_item in parsed.paths.items(): infile_path = f'/paths/[{path}]' - path_params: Dict[Tuple[str, model.In], model.Parameter] = {} + path_params: dict[tuple[str, model.In], model.Parameter] = {} for i, path_parameter in enumerate(path_item.parameters): param = self._convert_openapi_parameter(path_parameter, infile_path + f'/parameters/{i}') path_params[(param.name, param.in_)] = param @@ -474,7 +500,7 @@ def _convert_op_security(security: Optional[openapi.Security]) -> List[model.Sec consumes = parsed.consumes # parameters - global_params: Dict[str, Union[model.Parameter, List[model.RequestBody]]] = {} + global_params: dict[str, Union[model.Parameter, list[model.RequestBody]]] = {} for name, sw_parameter in parsed.parameters.items(): if self._is_swagger_request_body(sw_parameter, global_params): infile_path = '/requestBodies/' + name @@ -506,11 +532,11 @@ def _convert_op_security(security: Optional[openapi.Security]) -> List[model.Sec security_def = self._convert_swagger_security(sec_def) self._state.service.security[self._state.full_filepath + '#' + infile_path] = security_def - def _convert_op_security(security: Optional[swagger.Security]) -> List[model.Security]: + def _convert_op_security(security: Optional[swagger.Security]) -> list[model.Security]: if not security: security = default_security - securities: List[model.Security] = [] + securities: list[model.Security] = [] for name, scopes in security.items(): securities.append(self._convert_swagger_security(security_defs[name], scopes)) @@ -519,8 +545,8 @@ def _convert_op_security(security: Optional[swagger.Security]) -> List[model.Sec # paths for sw_path, sw_path_item in parsed.paths.items(): infile_path = f'/paths/[{sw_path}]' - sw_path_params: Dict[Tuple[str, model.In], model.Parameter] = {} - sw_path_body: Union[List[model.RequestBody], model.Ref] = [] + sw_path_params: dict[tuple[str, model.In], model.Parameter] = {} + sw_path_body: Union[list[model.RequestBody], model.Ref] = [] for i, sw_path_parameter in enumerate(sw_path_item.parameters): if self._is_swagger_request_body(sw_path_parameter, global_params): sw_path_body = self._convert_swagger_request_body( @@ -533,8 +559,8 @@ def _convert_op_security(security: Optional[swagger.Security]) -> List[model.Sec def _convert_op_params( op_params: swagger.Parameters, infile_path: str, - consumes: List[str], - ) -> Tuple[List[model.Parameter], Union[List[model.RequestBody], model.Ref]]: + consumes: list[str], + ) -> tuple[list[model.Parameter], Union[list[model.RequestBody], model.Ref]]: params = sw_path_params.copy() body = sw_path_body for i, sw_parameter in enumerate(op_params): @@ -577,7 +603,7 @@ def _convert_op_params( parser = chaotic_parser.SchemaParser( config=chaotic_parser.ParserConfig(erase_prefix=''), full_filepath=self._state.full_filepath, - full_vfilepath=self._state.full_filepath, + full_vfilepath=self._state.full_vfilepath, ) for name, schema in components_schemas.items(): infile_path = components_schemas_path + '/' + name @@ -594,15 +620,14 @@ def _make_sure_operations_are_unique(self) -> None: raise Exception(f'Operation {operation.method.upper()} {operation.path} is duplicated') seen.add(new) - @staticmethod - def _gen_operation_id(path: str, method: str) -> str: - return cpp_names.camel_case((path + '_' + method).replace('/', '_')) - - def _parse_schema(self, schema: Any, infile_path: str) -> Union[types.Schema, types.Ref]: + def _parse_schema(self, schema: Any, infile_path: str, allow_file=False) -> Union[types.Schema, types.Ref]: parser = chaotic_parser.SchemaParser( - config=chaotic_parser.ParserConfig(erase_prefix=''), + config=chaotic_parser.ParserConfig( + erase_prefix='', + allow_file=allow_file, + ), full_filepath=self._state.full_filepath, - full_vfilepath=self._state.full_filepath, + full_vfilepath=self._state.full_vfilepath, ) parser.parse_schema(infile_path, schema) parsed_schemas = parser.parsed_schemas() @@ -610,13 +635,13 @@ def _parse_schema(self, schema: Any, infile_path: str) -> Union[types.Schema, ty schema_ref = list(parsed_schemas.schemas.values())[0] if isinstance(schema_ref, types.Ref): - ref = types.Ref( + ref_ = types.Ref( chaotic_parser.SchemaParser._normalize_ref(schema_ref.ref), indirect=schema_ref.indirect, self_ref=schema_ref.self_ref, ) - ref._source_location = schema_ref._source_location # type: ignore - return ref + ref_._source_location = schema_ref._source_location # type: ignore + return ref_ else: return schema_ref @@ -625,8 +650,8 @@ def _append_openapi_operation( path: str, method: str, operation: Optional[openapi.Operation], - security_converter: Callable[[Optional[openapi.Security]], List[model.Security]], - path_params: Dict[Tuple[str, model.In], model.Parameter], + security_converter: Callable[[Optional[openapi.Security]], list[model.Security]], + path_params: dict[tuple[str, model.In], model.Parameter], ) -> None: if not operation: return @@ -651,7 +676,7 @@ def _append_openapi_operation( description=operation.description, path=path, method=method, - operationId=(operation.operationId or self._gen_operation_id(path, method)), + operationId=operation.operationId, parameters=list(params.values()), requestBody=requestBody, responses={ @@ -659,6 +684,8 @@ def _append_openapi_operation( for status, response in operation.responses.items() }, security=security_converter(operation.security), + x_middlewares=operation.x_taxi_middlewares or base_model.XMiddlewares(tvm=True), + x_client_codegen=operation.x_client_codegen, ) ) @@ -667,10 +694,10 @@ def _append_swagger_operation( path: str, method: str, operation: Optional[swagger.Operation], - security_converter: Callable[[Optional[swagger.Security]], List[model.Security]], + security_converter: Callable[[Optional[swagger.Security]], list[model.Security]], params_converter: Callable[ - [swagger.Parameters, str, List[str]], - Tuple[List[model.Parameter], Union[List[model.RequestBody], model.Ref]], + [swagger.Parameters, str, list[str]], + tuple[list[model.Parameter], Union[list[model.RequestBody], model.Ref]], ], ) -> None: if not operation: @@ -685,7 +712,7 @@ def _append_swagger_operation( description=operation.description, path=path, method=method, - operationId=(operation.operationId or self._gen_operation_id(path, method)), + operationId=operation.operationId, parameters=params, requestBody=body, responses={ @@ -695,6 +722,8 @@ def _append_swagger_operation( for status, response in operation.responses.items() }, security=security_converter(operation.security), + x_middlewares=operation.x_taxi_middlewares or base_model.XMiddlewares(tvm=True), + x_client_codegen=operation.x_client_codegen, ) ) diff --git a/chaotic-openapi/chaotic_openapi/front/ref_resolver.py b/chaotic-openapi/chaotic_openapi/front/ref_resolver.py index 997976c152f8..1893aa58197c 100644 --- a/chaotic-openapi/chaotic_openapi/front/ref_resolver.py +++ b/chaotic-openapi/chaotic_openapi/front/ref_resolver.py @@ -1,31 +1,26 @@ import collections -import re from typing import Any -from typing import Dict -from typing import List -from typing import Tuple from chaotic.front import parser as chaotic_parser +from chaotic.front import ref from chaotic.front import ref_resolver -REF_SHRINK_RE = re.compile('/[^/]+/../') - -def normalize_ref(filepath: str, ref: str) -> str: - if ref.startswith('#'): - return filepath + ref +def normalize_ref(filepath: str, ref_: str) -> str: + if not ref.Ref(ref_).file: + return filepath + ref_ return chaotic_parser.SchemaParser._normalize_ref( '{}/{}'.format( filepath.rsplit('/', 1)[0], - ref, + ref_, ) ) # Extracts list of external (non-'#ref') $refs in global form # (e.g. 'other.yaml#ref' becomes '/path/to/other.yaml#ref') -def _extract_refs(filepath: str, content: Any) -> List[str]: +def _extract_refs(filepath: str, content: Any) -> list[str]: refs = [] def visit(value: Any) -> None: @@ -36,15 +31,16 @@ def visit(value: Any) -> None: for item in value.values(): visit(item) if '$ref' in value: - ref = value['$ref'] - if not ref.startswith('#'): - refs.append(normalize_ref(filepath, ref).split('#')[0]) + ref_ = value['$ref'] + reference = ref.Ref(ref_) + if reference.file: + refs.append(ref.Ref(normalize_ref(filepath, ref_)).file) visit(content) return refs -def sort_openapis(contents: Dict[str, Any]) -> List[Tuple[str, Any]]: +def sort_openapis(contents: dict[str, Any]) -> list[tuple[str, Any]]: nodes = set() edges = collections.defaultdict(list) @@ -52,8 +48,8 @@ def sort_openapis(contents: Dict[str, Any]) -> List[Tuple[str, Any]]: nodes.add(filepath) refs = _extract_refs(filepath, content) - for ref in refs: - edges[filepath].append(ref) + for ref_ in refs: + edges[filepath].append(ref_) sorted_nodes = ref_resolver.sort_dfs(nodes, edges) return [(filepath, contents[filepath]) for filepath in sorted_nodes] diff --git a/chaotic-openapi/chaotic_openapi/front/swagger.py b/chaotic-openapi/chaotic_openapi/front/swagger.py index 2692ff83c229..2ce02e4c1b6a 100644 --- a/chaotic-openapi/chaotic_openapi/front/swagger.py +++ b/chaotic-openapi/chaotic_openapi/front/swagger.py @@ -1,7 +1,5 @@ import enum from typing import Any -from typing import Dict -from typing import List from typing import Optional from typing import Union @@ -44,12 +42,17 @@ class Parameter(base_model.BaseModel): type: Optional[str] = None format: Optional[str] = None allowEmptyValue: bool = False - items: Optional[Dict] = None + items: Optional[dict] = None collectionFormat: Optional[str] = None default: Any = None # TODO: validators + x_cpp_name: Optional[str] = pydantic.Field( + default=None, + validation_alias=pydantic.AliasChoices('x-taxi-cpp-name', 'x-usrv-cpp-name'), + ) + def model_post_init(self, context: Any, /) -> None: if self.in_ == In.body: if not self.schema_: @@ -67,7 +70,7 @@ class Header(base_model.BaseModel): description: Optional[str] = None type: str format: Optional[str] = None - items: Optional[Dict] = None + items: Optional[dict] = None collectionFormat: Optional[str] = None default: Any = None @@ -88,11 +91,11 @@ def model_post_init(self, context: Any, /) -> None: class Response(base_model.BaseModel): description: str schema_: Schema = pydantic.Field(alias='schema', default=None) - headers: Dict[str, Header] = pydantic.Field(default_factory=dict) - examples: Dict[str, Any] = pydantic.Field(default_factory=dict) + headers: dict[str, Header] = pydantic.Field(default_factory=dict) + examples: dict[str, Any] = pydantic.Field(default_factory=dict) -Responses = Dict[Union[str, int], Union[Response, Ref]] +Responses = dict[Union[str, int], Union[Response, Ref]] class SecurityType(str, enum.Enum): @@ -122,7 +125,7 @@ class SecurityDef(base_model.BaseModel): flow: Optional[OAuthFlow] = None authorizationUrl: Optional[str] = None tokenUrl: Optional[str] = None - scopes: Dict[str, str] = pydantic.Field(default_factory=dict) + scopes: dict[str, str] = pydantic.Field(default_factory=dict) def model_post_init(self, context: Any, /) -> None: if self.type == SecurityType.apiKey: @@ -150,26 +153,39 @@ def model_post_init(self, context: Any, /) -> None: # https://spec.openapis.org/oas/v2.0.html#security-requirement-object -Security = Dict[str, List[str]] +Security = dict[str, list[str]] -Parameters = List[Union[Parameter, Ref]] +Parameters = list[Union[Parameter, Ref]] # https://spec.openapis.org/oas/v2.0.html#operation-object class Operation(base_model.BaseModel): - tags: Optional[List[str]] = None + tags: Optional[list[str]] = None summary: Optional[str] = None description: str = '' - externalDocs: Optional[Dict] = None + externalDocs: Optional[dict] = None operationId: Optional[str] = None - consumes: List[str] = pydantic.Field(default_factory=list) - produces: List[str] = pydantic.Field(default_factory=list) + consumes: list[str] = pydantic.Field(default_factory=list) + produces: list[str] = pydantic.Field(default_factory=list) parameters: Parameters = pydantic.Field(default_factory=list) responses: Responses - schemes: List[str] = pydantic.Field(default_factory=list) + schemes: list[str] = pydantic.Field(default_factory=list) deprecated: bool = False security: Optional[Security] = None + x_taxi_middlewares: Optional[base_model.XMiddlewares] = pydantic.Field( + default=None, + validation_alias=pydantic.AliasChoices('x-taxi-middlewares', 'x-usrv-middlewares'), + ) + x_taxi_handler_codegen: bool = pydantic.Field( + default=True, + validation_alias=pydantic.AliasChoices('x-taxi-handler-codegen', 'x-usrv-handler-codegen'), + ) + x_client_codegen: bool = pydantic.Field( + default=True, + validation_alias=pydantic.AliasChoices('x-taxi-client-codegen', 'x-usrv-client-codegen'), + ) + # https://spec.openapis.org/oas/v2.0.html#paths-object class Path(base_model.BaseModel): @@ -183,7 +199,7 @@ class Path(base_model.BaseModel): parameters: Parameters = pydantic.Field(default_factory=list) -Paths = Dict[str, Path] +Paths = dict[str, Path] # https://spec.openapis.org/oas/v2.0.html#schema @@ -192,14 +208,14 @@ class Swagger(base_model.BaseModel): info: Optional[Info] = None host: Optional[str] = None basePath: str = '' - schemes: List[str] = pydantic.Field(default_factory=list) - consumes: List[str] = pydantic.Field(default_factory=list) - produces: List[str] = pydantic.Field(default_factory=list) + schemes: list[str] = pydantic.Field(default_factory=list) + consumes: list[str] = pydantic.Field(default_factory=list) + produces: list[str] = pydantic.Field(default_factory=list) paths: Paths = pydantic.Field(default_factory=dict) - definitions: Dict[str, Schema] = pydantic.Field(default_factory=dict) - parameters: Dict[str, Parameter] = pydantic.Field(default_factory=dict) - responses: Dict[str, Response] = pydantic.Field(default_factory=dict) - securityDefinitions: Dict[str, SecurityDef] = pydantic.Field(default_factory=dict) + definitions: dict[str, Schema] = pydantic.Field(default_factory=dict) + parameters: dict[str, Parameter] = pydantic.Field(default_factory=dict) + responses: dict[str, Response] = pydantic.Field(default_factory=dict) + securityDefinitions: dict[str, SecurityDef] = pydantic.Field(default_factory=dict) security: Security = pydantic.Field(default_factory=dict) def validate_security(self, security: Optional[Security]) -> None: diff --git a/chaotic-openapi/chaotic_openapi/main.py b/chaotic-openapi/chaotic_openapi/main.py index 89a524aea959..6e003096da52 100644 --- a/chaotic-openapi/chaotic_openapi/main.py +++ b/chaotic-openapi/chaotic_openapi/main.py @@ -1,4 +1,5 @@ import argparse +import os import sys import yaml @@ -23,7 +24,7 @@ def do_main(): # sort contents = {} - for file in args.file: + for file in args.files: with open(file) as ifile: content = yaml.safe_load(ifile) contents[file] = content @@ -32,14 +33,15 @@ def do_main(): # parse parser = front_parser.Parser(args.name) for file, content in sorted_contents: - parser.parse_schema(content, file) + parser.parse_schema(content, file, os.path.basename(file)) # translate spec = translator.Translator( parser.service(), dynamic_config=args.dynamic_config, cpp_namespace=(args.namespace or f'clients::{args.name}'), - include_dirs=[], + include_dirs=args.include_dirs or [], + middleware_plugins=[], ).spec() # render @@ -67,7 +69,14 @@ def parse_args() -> argparse.Namespace: ), ) parser.add_argument( - 'file', + '-I', + '--include-dir', + dest='include_dirs', + action='append', + help='Path to search for include files for x-usrv-cpp-type', + ) + parser.add_argument( + 'files', nargs='+', help='openapi/swagger yaml/json schemas', ) diff --git a/chaotic-openapi/include/userver/chaotic/openapi/form.hpp b/chaotic-openapi/include/userver/chaotic/openapi/form.hpp index 537ff5997dfa..5d8b933f2ce1 100644 --- a/chaotic-openapi/include/userver/chaotic/openapi/form.hpp +++ b/chaotic-openapi/include/userver/chaotic/openapi/form.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include USERVER_NAMESPACE_BEGIN diff --git a/chaotic-openapi/include/userver/chaotic/openapi/parameters.hpp b/chaotic-openapi/include/userver/chaotic/openapi/parameters.hpp index e72578e00424..1ff3d86a6d21 100644 --- a/chaotic-openapi/include/userver/chaotic/openapi/parameters.hpp +++ b/chaotic-openapi/include/userver/chaotic/openapi/parameters.hpp @@ -29,41 +29,41 @@ inline constexpr bool kIsTrivialRawType = true; template <> inline constexpr bool kIsTrivialRawType = true; -template +template struct TrivialParameterBase { - using RawType = RawType_; - using UserType = UserType_; + using RawType = TRawType; + using UserType = TUserType; static_assert(kIsTrivialRawType); }; -template +template struct TrivialParameter { - static constexpr auto kIn = In_; - static constexpr auto& kName = Name_; + static constexpr auto kIn = TIn; + static constexpr auto& kName = Name; - using Base = TrivialParameterBase; + using Base = TrivialParameterBase; static_assert(kIn != In::kQueryExplode); }; -template +template struct ArrayParameterBase { - static constexpr char kDelimiter = Delimiter_; - static constexpr auto kIn = In_; + static constexpr char kDelimiter = Delimiter; + static constexpr auto kIn = TIn; using RawType = std::vector; - using RawItemType = RawItemType_; - using UserType = std::vector; - using UserItemType = UserItemType_; + using RawItemType = TRawItemType; + using UserType = std::vector; + using UserItemType = TUserItemType; }; -template +template struct ArrayParameter { - static constexpr auto& kName = Name_; - static constexpr auto kIn = In_; + static constexpr auto& kName = Name; + static constexpr auto kIn = TIn; - using Base = ArrayParameterBase; + using Base = ArrayParameterBase; }; } // namespace chaotic::openapi diff --git a/chaotic-openapi/include/userver/chaotic/openapi/parameters_read.hpp b/chaotic-openapi/include/userver/chaotic/openapi/parameters_read.hpp index c21911619773..34d1f14abe2b 100644 --- a/chaotic-openapi/include/userver/chaotic/openapi/parameters_read.hpp +++ b/chaotic-openapi/include/userver/chaotic/openapi/parameters_read.hpp @@ -24,18 +24,18 @@ namespace chaotic::openapi { * */ -template +template auto GetParameter(std::string_view name, const server::http::HttpRequest& source) { - if constexpr (In_ == In::kPath) { + if constexpr (TIn == In::kPath) { return source.GetPathArg(name); - } else if constexpr (In_ == In::kCookie) { + } else if constexpr (TIn == In::kCookie) { return source.GetCookie(std::string{name}); - } else if constexpr (In_ == In::kHeader) { + } else if constexpr (TIn == In::kHeader) { return source.GetHeader(name); - } else if constexpr (In_ == In::kQuery) { + } else if constexpr (TIn == In::kQuery) { return source.GetArg(name); } else { - static_assert(In_ == In::kQueryExplode, "Unknown 'In'"); + static_assert(TIn == In::kQueryExplode, "Unknown 'In'"); return source.GetArgVector(name); } } @@ -78,8 +78,8 @@ void SplitByDelimiter(std::string_view str, char delimiter, utils::function_ref< } -template -struct ParseParameter> { +template +struct ParseParameter> { static auto Parse(std::string&& str_value) { openapi::ParseParameter> parser; diff --git a/chaotic-openapi/include/userver/chaotic/openapi/parameters_write.hpp b/chaotic-openapi/include/userver/chaotic/openapi/parameters_write.hpp index 81e16cad89c6..f2ec49e854d4 100644 --- a/chaotic-openapi/include/userver/chaotic/openapi/parameters_write.hpp +++ b/chaotic-openapi/include/userver/chaotic/openapi/parameters_write.hpp @@ -54,32 +54,39 @@ class ParameterSinkHttpClient final : public ParameterSinkBase { void Flush(); + const clients::http::Headers& GetHeaders() const; + + using HiddenQueryArgNamesFunc = bool (*)(std::string_view); + + void SetHiddenQueryArgNamesFunc(HiddenQueryArgNamesFunc func); + private: std::string url_pattern_; clients::http::Request& request_; clients::http::Headers headers_; http::MultiArgs query_args_; + HiddenQueryArgNamesFunc hidden_query_arg_names_func_{}; std::unordered_map cookies_; fmt::dynamic_format_arg_store path_vars_; }; void ValidatePathVariableValue(std::string_view name, std::string_view value); -template +template void SetParameter(Name& name, StrType&& str_value, ParameterSinkBase& dest) { static_assert(std::is_same_v || std::is_same_v>); - if constexpr (In == In::kPath) { + if constexpr (TIn == In::kPath) { USERVER_NAMESPACE::chaotic::openapi::ValidatePathVariableValue(name, str_value); dest.SetPath(name, std::forward(str_value)); - } else if constexpr (In == In::kCookie) { + } else if constexpr (TIn == In::kCookie) { dest.SetCookie(name, std::forward(str_value)); - } else if constexpr (In == In::kHeader) { + } else if constexpr (TIn == In::kHeader) { dest.SetHeader(name, std::forward(str_value)); - } else if constexpr (In == In::kQuery) { + } else if constexpr (TIn == In::kQuery) { dest.SetQuery(name, std::forward(str_value)); } else { - static_assert(In == In::kQueryExplode, "Unknown 'In'"); + static_assert(TIn == In::kQueryExplode, "Unknown 'In'"); dest.SetMultiQuery(name, std::forward(str_value)); } } @@ -145,9 +152,9 @@ struct SerializeParameter } }; -template -struct SerializeParameter> { - static_assert(In_ != In::kQueryExplode); +template +struct SerializeParameter> { + static_assert(TIn != In::kQueryExplode); std::string operator()(const UserType& item) const { return ToStrParameter(Convert(item, convert::To{})); } diff --git a/chaotic-openapi/integration_tests/clients/body-with-array/openapi.yaml b/chaotic-openapi/integration_tests/clients/body-with-array/openapi.yaml new file mode 100644 index 000000000000..4f47e43b77d1 --- /dev/null +++ b/chaotic-openapi/integration_tests/clients/body-with-array/openapi.yaml @@ -0,0 +1,18 @@ +openapi: 3.0.0 +info: + title: test-object client + version: '1.0' + +paths: + /test1: + post: + requestBody: + content: + application/json: + schema: + type: array + items: + type: string + responses: + '200': + description: OK diff --git a/chaotic-openapi/integration_tests/clients/empty-openapi/openapi.yaml b/chaotic-openapi/integration_tests/clients/empty-openapi/openapi.yaml new file mode 100644 index 000000000000..30776fa4e2a5 --- /dev/null +++ b/chaotic-openapi/integration_tests/clients/empty-openapi/openapi.yaml @@ -0,0 +1,12 @@ +openapi: 3.0.0 +info: + title: test-object client + version: '1.0' + +paths: + /test1: + post: + responses: + '200': + description: OK +components: {} diff --git a/chaotic-openapi/integration_tests/clients/handler-tag/openapi.yaml b/chaotic-openapi/integration_tests/clients/handler-tag/openapi.yaml new file mode 100644 index 000000000000..e094a1e0279f --- /dev/null +++ b/chaotic-openapi/integration_tests/clients/handler-tag/openapi.yaml @@ -0,0 +1,51 @@ +openapi: 3.0.0 +info: + title: test-object client + version: '1.0' + +paths: + /test1: + get: + parameters: + - name: param1 + in: query + required: true + x-usrv-handler-tag: param1 + schema: + type: integer + - name: param2 + in: query + required: true + x-usrv-handler-tag: param2 + schema: + type: number + - name: param3 + in: query + required: true + x-usrv-handler-tag: param3 + schema: + type: string + + - name: param4 + in: query + required: false + x-usrv-handler-tag: param4 + schema: + type: integer + - name: param5 + in: query + required: false + x-usrv-handler-tag: param5 + schema: + type: number + - name: param6 + in: query + required: false + x-usrv-handler-tag: param6 + schema: + type: string + + responses: + '200': + description: OK +components: {} diff --git a/chaotic-openapi/integration_tests/clients/non-alphanum/openapi.yaml b/chaotic-openapi/integration_tests/clients/non-alphanum/openapi.yaml new file mode 100644 index 000000000000..a7eadf154ba7 --- /dev/null +++ b/chaotic-openapi/integration_tests/clients/non-alphanum/openapi.yaml @@ -0,0 +1,24 @@ +openapi: 3.0.0 +info: + title: test-object client + version: '1.0' + +paths: + /some.method-name: + x-tag: test + post: + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + additionalProperties: false + properties: {} +components: + schemas: + My.Super-Object: + type: object + additionalProperties: false + properties: {} diff --git a/chaotic-openapi/integration_tests/clients/operation/non_std_type_reason.yaml b/chaotic-openapi/integration_tests/clients/operation/non_std_type_reason.yaml new file mode 100644 index 000000000000..d9646f7d3e72 --- /dev/null +++ b/chaotic-openapi/integration_tests/clients/operation/non_std_type_reason.yaml @@ -0,0 +1,26 @@ +openapi: 3.0.0 +info: + title: test-object client + version: '1.0' + +paths: + /test2: + get: + operationId: MyTest2 + requestBody: + content: + application/json: + x-usrv-non-std-type-reason: bla bla + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + x-usrv-non-std-type-reason: bla bla bla + schema: + type: object + properties: {} + additionalProperties: false +components: {} diff --git a/chaotic-openapi/integration_tests/clients/operation/openapi.yaml b/chaotic-openapi/integration_tests/clients/operation/openapi.yaml new file mode 100644 index 000000000000..3d4e8a3b7e2a --- /dev/null +++ b/chaotic-openapi/integration_tests/clients/operation/openapi.yaml @@ -0,0 +1,26 @@ +openapi: 3.0.0 +info: + title: test-object client + version: '1.0' + +paths: + /test1: + get: + operationId: MyTest + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: {} + additionalProperties: false + /test3: + get: + responses: + '200': + description: OK + content: + application/json: {} +components: {} diff --git a/chaotic-openapi/integration_tests/clients/parameters/openapi.yaml b/chaotic-openapi/integration_tests/clients/parameters/openapi.yaml index 8813f946e46f..b27cf61542a5 100644 --- a/chaotic-openapi/integration_tests/clients/parameters/openapi.yaml +++ b/chaotic-openapi/integration_tests/clients/parameters/openapi.yaml @@ -15,6 +15,12 @@ paths: - one - two - three + - name: class + x-usrv-cpp-name: myclass + in: query + required: false + schema: + type: string requestBody: content: application/json: @@ -41,4 +47,41 @@ paths: description: Some error '500': description: Some error + + /test1/query-log-mode: + get: + x-taxi-query-log-mode: hide + parameters: + - in: query + name: password + required: true + schema: + type: string + - in: query + name: secret + required: false + schema: + type: string + responses: + '200': + description: OK + + /test1/query-log-mode/parameter: + get: + parameters: + - in: query + name: password + required: true + x-taxi-query-log-mode: hide + schema: + type: string + - in: query + name: secret + required: false + schema: + type: string + responses: + '200': + description: OK + components: {} diff --git a/chaotic-openapi/integration_tests/clients/response-headers/openapi.yaml b/chaotic-openapi/integration_tests/clients/response-headers/openapi.yaml index 1ece697b68a4..fd4161df5be7 100644 --- a/chaotic-openapi/integration_tests/clients/response-headers/openapi.yaml +++ b/chaotic-openapi/integration_tests/clients/response-headers/openapi.yaml @@ -19,4 +19,18 @@ paths: X-Header: schema: type: string -components: {} + X-Integer: + schema: + type: integer + X-Seconds: + schema: + type: integer + x-taxi-cpp-type: std::chrono::seconds +components: + responses: + ResponseX: + description: OK + headers: + X-Header: + schema: + type: string diff --git a/chaotic-openapi/integration_tests/clients/swagger/swagger.yaml b/chaotic-openapi/integration_tests/clients/swagger/swagger.yaml index 137e8471f8b4..a7a1fd19e3c3 100644 --- a/chaotic-openapi/integration_tests/clients/swagger/swagger.yaml +++ b/chaotic-openapi/integration_tests/clients/swagger/swagger.yaml @@ -21,6 +21,42 @@ paths: '404': $ref: '#/responses/Response404' + /test1/no-tvm: + post: + x-usrv-middlewares: + tvm: false + responses: + '200': + description: OK + + /test1/body: + get: + consumes: + - application/json + produces: + - application/json + parameters: + - in: body + name: body + required: true + schema: + type: string + responses: + '200': + description: OK + + /test1/file: + post: + consumes: + - multipart/form-data + parameters: + - in: formData + name: file + required: true + type: file + responses: + '200': + description: OK parameters: X-Header-456: diff --git a/chaotic-openapi/integration_tests/clients/test-object/openapi.yaml b/chaotic-openapi/integration_tests/clients/test-object/openapi.yaml index df35b5aaf772..becc3c8269c6 100644 --- a/chaotic-openapi/integration_tests/clients/test-object/openapi.yaml +++ b/chaotic-openapi/integration_tests/clients/test-object/openapi.yaml @@ -37,6 +37,9 @@ components: Type: type: object properties: + bar: + type: integer + x-usrv-cpp-type: unsigned baz: type: string additionalProperties: false diff --git a/chaotic-openapi/integration_tests/src/operation_test.cpp b/chaotic-openapi/integration_tests/src/operation_test.cpp new file mode 100644 index 000000000000..4368fd3d190e --- /dev/null +++ b/chaotic-openapi/integration_tests/src/operation_test.cpp @@ -0,0 +1,28 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +UTEST(Operation, OperationId) { + auto http_client = utest::CreateHttpClient(); + ::clients::operation::ClientImpl client( + { + "", + }, + *http_client + ); + + if (false) { + // Dead code intentionally + [[maybe_unused]] ::clients::operation::mytest::Response200 response = client.MyTest({}); + } +} + +} // namespace + +USERVER_NAMESPACE_END diff --git a/chaotic-openapi/integration_tests/src/parameters_test.cpp b/chaotic-openapi/integration_tests/src/parameters_test.cpp new file mode 100644 index 000000000000..1b347ebca618 --- /dev/null +++ b/chaotic-openapi/integration_tests/src/parameters_test.cpp @@ -0,0 +1,20 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +namespace client = ::clients::parameters::test1::post; + +UTEST(Parameters, CppName) { + client::Request request; + + // compilation test + request.myclass = "123"; +} + +} // namespace + +USERVER_NAMESPACE_END diff --git a/chaotic-openapi/integration_tests/src/requests_test.cpp b/chaotic-openapi/integration_tests/src/requests_test.cpp index 632802933cab..99980d1ac1c0 100644 --- a/chaotic-openapi/integration_tests/src/requests_test.cpp +++ b/chaotic-openapi/integration_tests/src/requests_test.cpp @@ -5,8 +5,11 @@ #include #include #include +#include +#include #include +#include USERVER_NAMESPACE_BEGIN @@ -96,6 +99,43 @@ UTEST(RequestsMultipleContentTypes, OctetStream) { auto response = request.perform(); EXPECT_EQ(response->status_code(), 200); } + +class RequestsQueryLogMode : public utest::LogCaptureFixture<> {}; + +UTEST_F(RequestsQueryLogMode, HideOperation) { + const utest::HttpServerMock http_server([&](const utest::HttpServerMock::HttpRequest&) { + return utest::HttpServerMock::HttpResponse{200}; + }); + auto http_client_ptr = utest::CreateHttpClient(); + auto request = http_client_ptr->CreateRequest(); + + namespace client = ::clients::parameters::test1_query_log_mode::get; + client::SerializeRequest({client::Request{"foo", "bar"}}, http_server.GetBaseUrl(), request); + auto response = request.perform(); + + EXPECT_EQ(response->status_code(), 200); + + auto text = GetLogCapture().GetAll().back().GetTag("http_url"); + EXPECT_TRUE(utils::text::EndsWith(text, "test1/query-log-mode?password=***&secret=***")); +} + +UTEST_F(RequestsQueryLogMode, HideParameter) { + const utest::HttpServerMock http_server([&](const utest::HttpServerMock::HttpRequest&) { + return utest::HttpServerMock::HttpResponse{200}; + }); + auto http_client_ptr = utest::CreateHttpClient(); + auto request = http_client_ptr->CreateRequest(); + + namespace client = ::clients::parameters::test1_query_log_mode_parameter::get; + client::SerializeRequest({client::Request{"foo", "bar"}}, http_server.GetBaseUrl(), request); + auto response = request.perform(); + + EXPECT_EQ(response->status_code(), 200); + + auto text = GetLogCapture().GetAll().back().GetTag("http_url"); + EXPECT_TRUE(utils::text::EndsWith(text, "test1/query-log-mode/parameter?password=***&secret=bar")) << text; +} + } // namespace USERVER_NAMESPACE_END diff --git a/chaotic-openapi/integration_tests/src/responses_test.cpp b/chaotic-openapi/integration_tests/src/responses_test.cpp index 2a67d547f5bf..6e0372f49f54 100644 --- a/chaotic-openapi/integration_tests/src/responses_test.cpp +++ b/chaotic-openapi/integration_tests/src/responses_test.cpp @@ -190,6 +190,8 @@ UTEST(ResponsesMultipleContentType, HeaderParse) { r.body = R"({})"; r.headers[std::string{"Content-Type"}] = "application/json"; r.headers[std::string{"X-Header"}] = "string"; + r.headers[std::string{"X-Integer"}] = "42"; + r.headers[std::string{"X-Seconds"}] = "100"; return r; }); auto http_client = utest::CreateHttpClient(); @@ -198,6 +200,8 @@ UTEST(ResponsesMultipleContentType, HeaderParse) { namespace client = ::clients::response_headers::test1::post; auto response = client::ParseResponse(*http_response); EXPECT_EQ(response.X_Header, "string"); + EXPECT_EQ(response.X_Integer, 42); + EXPECT_EQ(response.X_Seconds, 100); } } // namespace diff --git a/chaotic-openapi/src/chaotic/openapi/parameters_write.cpp b/chaotic-openapi/src/chaotic/openapi/parameters_write.cpp index 2d36488072be..a198cde0fdf7 100644 --- a/chaotic-openapi/src/chaotic/openapi/parameters_write.cpp +++ b/chaotic-openapi/src/chaotic/openapi/parameters_write.cpp @@ -1,11 +1,34 @@ #include +#include + #include +#include + USERVER_NAMESPACE_BEGIN namespace chaotic::openapi { +namespace { + +constexpr std::string_view kMask = "***"; + +auto MaskQueryMultiArgs(const http::MultiArgs& args, ParameterSinkHttpClient::HiddenQueryArgNamesFunc func) { + using Pair = std::pair; + + auto masked = args | boost::adaptors::transformed([&func](const auto& pair) { + const auto& [name, value] = pair; + if (func(name)) + return Pair(name, kMask); + else + return Pair(name, value); + }); + return utils::AsContainer>(std::move(masked)); +} + +} // namespace + ParameterSinkHttpClient::ParameterSinkHttpClient(clients::http::Request& request, std::string&& url_pattern) : url_pattern_(std::move(url_pattern)), request_(request) {} @@ -35,10 +58,21 @@ void ParameterSinkHttpClient::SetMultiQuery(std::string_view name, std::vector') + parser.parse_schema(schema, '', '') service = parser.service() tr = translator.Translator( @@ -15,6 +15,7 @@ def func(schema): dynamic_config='', cpp_namespace='test_namespace', include_dirs=[], + middleware_plugins=[], ) return tr.spec() diff --git a/chaotic-openapi/tests/back/test_headers.py b/chaotic-openapi/tests/back/test_headers.py index 89931db57b26..8e1279c1ef31 100644 --- a/chaotic-openapi/tests/back/test_headers.py +++ b/chaotic-openapi/tests/back/test_headers.py @@ -37,6 +37,7 @@ def test_headers(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, request_bodies=[], responses=[ types.Response( @@ -59,6 +60,7 @@ def test_headers(translate_single_schema): ), parser='openapi::TrivialParameter', required=False, + query_log_mode_hide=False, ) ], body={}, @@ -108,6 +110,7 @@ def test_header_ref(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, request_bodies=[], responses=[ types.Response( @@ -129,6 +132,7 @@ def test_header_ref(translate_single_schema): ), parser='openapi::TrivialParameter', required=False, + query_log_mode_hide=False, ), ], ) diff --git a/chaotic-openapi/tests/back/test_parameters.py b/chaotic-openapi/tests/back/test_parameters.py index 0fdfaec0c239..d8af7a024076 100644 --- a/chaotic-openapi/tests/back/test_parameters.py +++ b/chaotic-openapi/tests/back/test_parameters.py @@ -36,6 +36,7 @@ def test_parameters(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, parameters=[ types.Parameter( description='parameter description', @@ -54,6 +55,7 @@ def test_parameters(translate_single_schema): ), parser='openapi::TrivialParameter', required=False, + query_log_mode_hide=False, ) ], request_bodies=[], @@ -100,6 +102,7 @@ def test_parameters_ref(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, parameters=[ types.Parameter( description='parameter description', @@ -118,6 +121,7 @@ def test_parameters_ref(translate_single_schema): ), parser='openapi::TrivialParameter', required=False, + query_log_mode_hide=False, ) ], request_bodies=[], @@ -164,6 +168,7 @@ def test_parameters_schemas_ref(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, parameters=[ types.Parameter( description='parameter description', @@ -198,6 +203,7 @@ def test_parameters_schemas_ref(translate_single_schema): ), parser='openapi::TrivialParameter', required=False, + query_log_mode_hide=False, ) ], request_bodies=[], @@ -309,6 +315,6 @@ def test_parameters_too_complex_schema(translate_single_schema): Unhandled error while processing Path "/paths/[/]/get/parameters/0/schema", Format "jsonschema" Error: -Unsupported parameter type +Unsupported parameter type for parameter "param" ===============================================================""" ) diff --git a/chaotic-openapi/tests/back/test_request_body.py b/chaotic-openapi/tests/back/test_request_body.py index 948931f40ebe..871deac6603d 100644 --- a/chaotic-openapi/tests/back/test_request_body.py +++ b/chaotic-openapi/tests/back/test_request_body.py @@ -35,6 +35,7 @@ def test_request_body(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, request_bodies=[ types.Body( content_type='application/json', @@ -91,6 +92,7 @@ def test_request_body_ref(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, request_bodies=[ types.Body( content_type='application/json', @@ -140,6 +142,7 @@ def test_request_body_nonrequired(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, request_bodies=[ types.Body( content_type='application/json', diff --git a/chaotic-openapi/tests/back/test_responses.py b/chaotic-openapi/tests/back/test_responses.py index cf2264ae88bf..f1f85b61fa31 100644 --- a/chaotic-openapi/tests/back/test_responses.py +++ b/chaotic-openapi/tests/back/test_responses.py @@ -36,6 +36,7 @@ def test_response(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, request_bodies=[], responses=[ types.Response( @@ -97,6 +98,7 @@ def test_response_ref(translate_single_schema): types.Operation( method='GET', path='/', + operation_id=None, request_bodies=[], responses=[ types.Response( diff --git a/chaotic-openapi/tests/front/conftest.py b/chaotic-openapi/tests/front/conftest.py index 58561e15e228..1a662dfaec78 100644 --- a/chaotic-openapi/tests/front/conftest.py +++ b/chaotic-openapi/tests/front/conftest.py @@ -6,7 +6,7 @@ def simple_parser(): def _simple_parser(schema): parser = front_parser.Parser('test') - parser.parse_schema(schema, '') + parser.parse_schema(schema, '', '') return parser.service() return _simple_parser diff --git a/chaotic-openapi/tests/front/test_openapi.py b/chaotic-openapi/tests/front/test_openapi.py index a1d0677d471d..fb18a7aa760b 100644 --- a/chaotic-openapi/tests/front/test_openapi.py +++ b/chaotic-openapi/tests/front/test_openapi.py @@ -1,3 +1,4 @@ +from chaotic_openapi.front import base_model from chaotic_openapi.front import model import pytest @@ -42,7 +43,7 @@ def test_openapi_body_schema(simple_parser): description='', path='/', method='get', - operationId='Get', + operationId=None, parameters=[], requestBody=[ model.RequestBody( @@ -53,6 +54,8 @@ def test_openapi_body_schema(simple_parser): ], responses={}, security=[], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ) ], ) @@ -140,7 +143,7 @@ def test_openapi_security(simple_parser): description='', path='/', method='get', - operationId='Get', + operationId=None, parameters=[], responses={}, requestBody=[], @@ -163,12 +166,14 @@ def test_openapi_security(simple_parser): ], ), ], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ), model.Operation( description='', path='/', method='post', - operationId='Post', + operationId=None, parameters=[], responses={}, requestBody=[], @@ -191,12 +196,14 @@ def test_openapi_security(simple_parser): ], ), ], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ), model.Operation( description='', path='/', method='put', - operationId='Put', + operationId=None, parameters=[], responses={}, requestBody=[], @@ -219,6 +226,8 @@ def test_openapi_security(simple_parser): ], ), ], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ), ], ) @@ -309,6 +318,8 @@ def test_openapi_parameters(simple_parser): examples={}, deprecated=False, allowEmptyValue=False, + x_cpp_name=None, + x_query_log_mode_hide=False, ) }, operations=[ @@ -316,10 +327,12 @@ def test_openapi_parameters(simple_parser): description='', path='/', method='get', - operationId='Get', + operationId=None, responses={}, requestBody=[], security=[], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, parameters=[ model.Parameter( name='pamparam1', @@ -331,6 +344,8 @@ def test_openapi_parameters(simple_parser): examples={}, deprecated=False, allowEmptyValue=False, + x_cpp_name=None, + x_query_log_mode_hide=False, ), model.Parameter( name='pamparam2', @@ -342,6 +357,8 @@ def test_openapi_parameters(simple_parser): examples={}, deprecated=False, allowEmptyValue=False, + x_cpp_name=None, + x_query_log_mode_hide=False, ), model.Parameter( name='pamparam2', @@ -353,6 +370,8 @@ def test_openapi_parameters(simple_parser): examples={}, deprecated=False, allowEmptyValue=False, + x_cpp_name=None, + x_query_log_mode_hide=False, ), ], ) diff --git a/chaotic-openapi/tests/front/test_swagger.py b/chaotic-openapi/tests/front/test_swagger.py index 353949c529e3..47d3180638e9 100644 --- a/chaotic-openapi/tests/front/test_swagger.py +++ b/chaotic-openapi/tests/front/test_swagger.py @@ -1,5 +1,8 @@ +from chaotic_openapi.front import base_model from chaotic_openapi.front import model +import pytest +from chaotic import error from chaotic.front import types @@ -43,7 +46,7 @@ def test_swagger_body_schema(simple_parser): description='', path='/', method='get', - operationId='Get', + operationId=None, parameters=[], requestBody=[ model.RequestBody( @@ -54,6 +57,8 @@ def test_swagger_body_schema(simple_parser): ], responses={}, security=[], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ) ], ) @@ -118,6 +123,8 @@ def test_swagger_responses(simple_parser): allowEmptyValue=False, style=model.Style.simple, schema=types.String(), + x_cpp_name=None, + x_query_log_mode_hide=False, ) }, content={ @@ -137,7 +144,7 @@ def test_swagger_responses(simple_parser): description='', path='/', method='get', - operationId='Get', + operationId=None, parameters=[], requestBody=[], responses={ @@ -154,6 +161,8 @@ def test_swagger_responses(simple_parser): allowEmptyValue=False, style=model.Style.simple, schema=types.String(), + x_cpp_name=None, + x_query_log_mode_hide=False, ), 'X-Header-Name-2': model.Parameter( name='X-Header-Name-2', @@ -165,6 +174,8 @@ def test_swagger_responses(simple_parser): allowEmptyValue=False, style=model.Style.simple, schema=types.Array(items=types.Integer()), + x_cpp_name=None, + x_query_log_mode_hide=False, ), }, content={ @@ -181,6 +192,8 @@ def test_swagger_responses(simple_parser): 500: model.Ref('#/responses/500'), }, security=[], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ) ], ) @@ -274,7 +287,7 @@ def test_swagger_securuty(simple_parser): description='', path='/', method='get', - operationId='Get', + operationId=None, parameters=[], responses={}, requestBody=[], @@ -302,12 +315,14 @@ def test_swagger_securuty(simple_parser): ], ), ], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ), model.Operation( description='', path='/', method='post', - operationId='Post', + operationId=None, parameters=[], responses={}, requestBody=[], @@ -335,12 +350,14 @@ def test_swagger_securuty(simple_parser): ], ), ], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ), model.Operation( description='', path='/', method='put', - operationId='Put', + operationId=None, parameters=[], responses={}, requestBody=[], @@ -368,6 +385,8 @@ def test_swagger_securuty(simple_parser): ], ), ], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, ), ], ) @@ -449,13 +468,15 @@ def test_swagger_parameters(simple_parser): description='', path='/', method='get', - operationId='Get', + operationId=None, responses={}, requestBody=[ model.RequestBody(content_type='text/plain; charset=utf-8', required=True, schema=types.Number()), model.RequestBody(content_type='application/json', required=True, schema=types.Number()), ], security=[], + x_middlewares=base_model.XMiddlewares(tvm=True), + x_client_codegen=True, parameters=[ model.Parameter( name='pamparam2', @@ -467,6 +488,8 @@ def test_swagger_parameters(simple_parser): examples={}, deprecated=False, allowEmptyValue=False, + x_cpp_name=None, + x_query_log_mode_hide=False, ), model.Parameter( name='pamparam2', @@ -478,8 +501,73 @@ def test_swagger_parameters(simple_parser): examples={}, deprecated=False, allowEmptyValue=False, + x_cpp_name=None, + x_query_log_mode_hide=False, ), ], ) ], ) + + +@pytest.mark.parametrize( + 'consumes', + [ + 'multipart/form-data', + 'application/x-www-form-urlencoded', + ], +) +def test_swagger_file_in_body_ok(simple_parser, consumes): + simple_parser( + { + 'swagger': '2.0', + 'info': {}, + 'paths': { + '/': { + 'get': { + 'consumes': [consumes], + 'parameters': [ + { + 'name': 'pamparam', + 'in': 'formData', + 'required': True, + 'type': 'file', + }, + ], + 'responses': {}, + }, + }, + }, + }, + ) + + +def test_swagger_file_in_body_wrong_consumes(simple_parser): + with pytest.raises(error.BaseError) as exc_info: + simple_parser( + { + 'swagger': '2.0', + 'info': {}, + 'paths': { + '/': { + 'get': { + 'consumes': ['application/json'], + 'parameters': [ + { + 'name': 'pamparam', + 'in': 'formData', + 'required': True, + 'type': 'file', + }, + ], + 'responses': {}, + }, + }, + }, + }, + ) + + assert ( + exc_info.value.msg + == '"consumes" must be either "multipart/form-data" or "application/x-www-form-urlencoded" for "type: file"' + ) diff --git a/chaotic/CMakeLists.txt b/chaotic/CMakeLists.txt index 475eceb28abd..b8b749c24c7f 100644 --- a/chaotic/CMakeLists.txt +++ b/chaotic/CMakeLists.txt @@ -7,6 +7,7 @@ userver_module( SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" UTEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*_test.cpp" LINK_LIBRARIES userver::universal + DEPENDS universal NO_CORE_LINK ) @@ -24,7 +25,11 @@ if(USERVER_BUILD_TESTS) ) add_subdirectory(integration_tests) - add_subdirectory(golden_tests) + + option(USERVER_CHAOTIC_GOLDEN_TESTS "Whether to run golden tests of chaotic" ON) + if (USERVER_CHAOTIC_GOLDEN_TESTS) + add_subdirectory(golden_tests) + endif() endif() _userver_directory_install( diff --git a/chaotic/chaotic/back/cpp/renderer.py b/chaotic/chaotic/back/cpp/renderer.py index c05fa3229bec..88a898b4db69 100644 --- a/chaotic/chaotic/back/cpp/renderer.py +++ b/chaotic/chaotic/back/cpp/renderer.py @@ -2,8 +2,6 @@ import dataclasses import os import pathlib -from typing import Dict -from typing import List from typing import Optional import jinja2 @@ -26,7 +24,7 @@ class CppOutputFile: @dataclasses.dataclass class CppOutput: filepath_wo_ext: str - files: List[CppOutputFile] + files: list[CppOutputFile] current_namespace = '' # pylint: disable=invalid-name @@ -76,14 +74,14 @@ def cpp_type(name: str) -> str: return name_part -def declaration_includes(types: List[cpp_types.CppType]) -> List[str]: +def declaration_includes(types: list[cpp_types.CppType]) -> list[str]: includes = set() for type_ in types: includes.update(set(type_.declaration_includes())) return sorted(includes) -def definition_includes(types: List[cpp_types.CppType]) -> List[str]: +def definition_includes(types: list[cpp_types.CppType]) -> list[str]: includes = set() for type_ in types: includes.update(set(type_.definition_includes())) @@ -144,7 +142,7 @@ def __init__( self, *, relative_to: str, - vfilepath_to_relfilepath: Dict[str, str], + vfilepath_to_relfilepath: dict[str, str], clang_format_bin: str, parse_extra_formats: bool = False, generate_serializer: bool = False, @@ -165,9 +163,9 @@ def _vfilepath_to_relfilepath(self, vfilepath: str) -> str: def extract_external_includes( self, - types_cpp: Dict[str, cpp_types.CppType], + types_cpp: dict[str, cpp_types.CppType], ignore_filepath_wo_ext: str, - ) -> List[str]: + ) -> list[str]: result = set() def visitor( @@ -200,11 +198,11 @@ def filepath_to_include(self, filepath_wo_ext: str) -> str: def render( self, - types: Dict[str, cpp_types.CppType], + types: dict[str, cpp_types.CppType], local_pair_header=True, pair_header: Optional[str] = None, - ) -> List[CppOutput]: - files: Dict[str, Dict[str, cpp_types.CppType]] = collections.defaultdict(dict) + ) -> list[CppOutput]: + files: dict[str, dict[str, cpp_types.CppType]] = collections.defaultdict(dict) for name, type_ in types.items(): assert type_.json_schema @@ -297,7 +295,7 @@ def render( return output @staticmethod - def get_output_files(stem: str, path: str) -> List[str]: + def get_output_files(stem: str, path: str) -> list[str]: return [ f'include/{path}/{stem}_fwd.hpp', f'include/{path}/{stem}_parsers.ipp', diff --git a/chaotic/chaotic/back/cpp/translator.py b/chaotic/chaotic/back/cpp/translator.py index fefa06cbcc3e..4bea33ef8212 100644 --- a/chaotic/chaotic/back/cpp/translator.py +++ b/chaotic/chaotic/back/cpp/translator.py @@ -4,27 +4,25 @@ import pathlib import re from typing import Callable -from typing import Dict -from typing import List from typing import NoReturn from typing import Optional -from typing import Set from chaotic import cpp_names from chaotic import error from chaotic.back.cpp import type_name from chaotic.back.cpp import types as cpp_types +from chaotic.front import ref from chaotic.front import types @dataclasses.dataclass class GeneratorConfig: # vfull -> namespace - namespaces: Dict[str, str] + namespaces: dict[str, str] # infile_path -> cpp type infile_to_name_func: Callable # type: ignore - include_dirs: Optional[List[str]] = dataclasses.field( + include_dirs: Optional[list[str]] = dataclasses.field( # type: ignore default_factory=list, ) @@ -34,11 +32,11 @@ class GeneratorConfig: @dataclasses.dataclass class GeneratorState: - types: Dict[str, cpp_types.CppType] - refs: Dict[types.Schema, str] # type: ignore - ref_objects: List[cpp_types.CppRef] - external_types: Dict[types.Schema, cpp_types.CppType] # type: ignore - seen_includes: Set[str] + types: dict[str, cpp_types.CppType] + refs: dict[types.Schema, str] # type: ignore + ref_objects: list[cpp_types.CppRef] + external_types: dict[types.Schema, cpp_types.CppType] # type: ignore + seen_includes: set[str] NON_NAME_SYMBOL_RE = re.compile('[^_0-9a-zA-Z]') @@ -47,11 +45,11 @@ class GeneratorState: class FormatChooser: - def __init__(self, types: List[cpp_types.CppType]) -> None: + def __init__(self, types: list[cpp_types.CppType]) -> None: self.types = types - self.parent: Dict[ + self.parent: dict[ cpp_types.CppType, - List[Optional[cpp_types.CppType]], + list[Optional[cpp_types.CppType]], ] = collections.defaultdict(list) def check_for_json_onlyness(self) -> None: @@ -117,8 +115,8 @@ def __init__(self, config: GeneratorConfig) -> None: def generate_types( self, schemas: types.ResolvedSchemas, - external_schemas: Dict[str, cpp_types.CppType] = {}, - ) -> Dict[str, cpp_types.CppType]: + external_schemas: dict[str, cpp_types.CppType] = {}, + ) -> dict[str, cpp_types.CppType]: self._state.seen_includes = set() for cpp_type in external_schemas.values(): @@ -128,6 +126,16 @@ def generate_types( for name, schema in schemas.schemas.items(): fq_cpp_name = self._gen_fq_cpp_name(name) + if fq_cpp_name in self._state.types: + sl1 = schema.source_location() + path1 = f'{sl1.filepath}#{sl1.location}' + + existing_schema = self._state.types[fq_cpp_name].json_schema + assert existing_schema + sl2 = existing_schema.source_location() + path2 = f'{sl2.filepath}#{sl2.location}' + raise Exception(f'Duplicate type name: {fq_cpp_name}, generated from {path1} and {path2}') + self._state.refs[schema] = fq_cpp_name self._state.types[fq_cpp_name] = self._generate_type( type_name.TypeName(fq_cpp_name), @@ -140,7 +148,7 @@ def generate_types( return self._state.types @property - def seen_includes(self) -> Set[str]: + def seen_includes(self) -> set[str]: return self._state.seen_includes def _validate_type(self, type_: cpp_types.CppType) -> None: @@ -149,11 +157,15 @@ def _validate_type(self, type_: cpp_types.CppType) -> None: if type_.has_generated_user_cpp_type(): return + user_includes = type_.get_includes_by_cpp_type(type_.user_cpp_type) + for user_include in user_includes: + self._validate_user_include_exists(type_, user_include) + + def _validate_user_include_exists(self, type_: cpp_types.CppType, user_include: str) -> None: if self._config.include_dirs is None: # no check at all return - user_include = type_.cpp_type_to_user_include_path(type_.user_cpp_type) for include_dir in self._config.include_dirs: path = os.path.join(include_dir, user_include) if os.path.exists(path): @@ -217,19 +229,19 @@ def _extract_container(self, schema: types.Schema) -> str: return container def fixup_refs(self) -> None: - for ref in self._state.ref_objects: - assert isinstance(ref.json_schema, types.Ref) - schema = ref.json_schema.schema + for ref_ in self._state.ref_objects: + assert isinstance(ref_.json_schema, types.Ref) + schema = ref_.json_schema.schema name = self._state.refs.get(schema) if name: orig_cpp_type = self._state.types[name] else: orig_cpp_type = self._state.external_types[schema] - ref.orig_cpp_type = orig_cpp_type - ref.indirect = ref.json_schema.indirect - ref.self_ref = ref.json_schema.self_ref - ref.cpp_name = name + ref_.orig_cpp_type = orig_cpp_type + ref_.indirect = ref_.json_schema.indirect + ref_.self_ref = ref_.json_schema.self_ref + ref_.cpp_name = name def fixup_formats(self) -> None: chooser = FormatChooser(list(self._state.types.values())) @@ -247,9 +259,9 @@ def _generate_type( return cpp_type def _gen_fq_cpp_name(self, jsonschema_name: str) -> str: - vfile, infile = jsonschema_name.split('#') - name = self._config.infile_to_name_func(infile, pathlib.Path(vfile).stem) - namespace = self._config.namespaces[vfile] + reference = ref.Ref(jsonschema_name) + name = self._config.infile_to_name_func(reference.fragment, pathlib.Path(reference.file).stem) + namespace = self._config.namespaces[reference.file] if namespace: return '::' + namespace + '::' + name else: @@ -285,7 +297,7 @@ def _gen_integer( if 'x-enum-varnames' in schema.x_properties: enum_names = schema.x_properties['x-enum-varnames'] - emum_items: List[cpp_types.CppIntEnumItem] = [] + emum_items: list[cpp_types.CppIntEnumItem] = [] def to_camel_case(text: str) -> str: words = SPLIT_RE.findall(text) @@ -607,8 +619,8 @@ def _gen_one_of_with_discriminator( mapping_values = schema.mapping.as_ints() for field_value, refs in zip(schema.oneOf, mapping_values): - for ref in refs: - variants[ref] = self._gen_ref( + for ref_ in refs: + variants[ref_] = self._gen_ref( type_name.TypeName(''), field_value, ) @@ -698,7 +710,7 @@ def _gen_ref( name: type_name.TypeName, schema: types.Ref, ) -> cpp_types.CppType: - ref = cpp_types.CppRef( + ref_ = cpp_types.CppRef( json_schema=schema, nullable=False, # stub @@ -713,8 +725,8 @@ def _gen_ref( indirect=False, self_ref=False, ) - self._state.ref_objects.append(ref) - return ref + self._state.ref_objects.append(ref_) + return ref_ # pylint: disable=protected-access diff --git a/chaotic/chaotic/back/cpp/type_name.py b/chaotic/chaotic/back/cpp/type_name.py index dd7071be73b4..21970a445912 100644 --- a/chaotic/chaotic/back/cpp/type_name.py +++ b/chaotic/chaotic/back/cpp/type_name.py @@ -1,9 +1,8 @@ -from typing import List from typing import Union class TypeName: - def __init__(self, name: Union[str, List[str]]) -> None: + def __init__(self, name: Union[str, list[str]]) -> None: if isinstance(name, str): self._components = name.split('::') else: diff --git a/chaotic/chaotic/back/cpp/types.py b/chaotic/chaotic/back/cpp/types.py index 5b65341098ab..016c7092947b 100644 --- a/chaotic/chaotic/back/cpp/types.py +++ b/chaotic/chaotic/back/cpp/types.py @@ -2,8 +2,6 @@ import dataclasses import itertools from typing import Any -from typing import Dict -from typing import List from typing import Optional from typing import Union @@ -45,13 +43,13 @@ def without_json_schema(self) -> 'CppType': # Should return only direct subtypes, not recursively because # jinja's generate_*() is called recursively. - def subtypes(self) -> List['CppType']: + def subtypes(self) -> list['CppType']: return [] - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: raise NotImplementedError() - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: raise NotImplementedError() def parser_type(self, ns: str, name: str) -> str: @@ -123,26 +121,32 @@ def cpp_comment(self) -> str: assert self.json_schema kwargs = self.json_schema.x_properties - description = kwargs.get('description') + description = (kwargs.get('title', '') + '\n' + kwargs.get('description', '')).strip() if description: return '// ' + description.replace('\n', '\n//') else: return '' @classmethod - def cpp_type_to_user_include_path(cls, cpp_type: str) -> str: + def get_includes_by_cpp_type(cls, cpp_type: str) -> list[str]: + includes = [] + parts = cpp_type.split('<', 1) cpp_type = parts[0] if cpp_type == 'userver::utils::StrongTypedef' and parts[1][-1] == '>': cpp_type = parts[1].split(', ', 1)[0] + includes.append('userver/utils/strong_typedef.hpp') + elif cpp_type == 'utils::StrongTypedef' and parts[1][-1] == '>': + # legacy uservices + cpp_type = parts[1].split(', ', 1)[0] + includes.append('userver/utils/strong_typedef.hpp') if cpp_type.startswith('::'): cpp_type = cpp_type[2:] - return 'userver/chaotic/io/' + camel_to_snake_case(cpp_type.replace('::', '/')) + '.hpp' - @classmethod - def get_include_by_cpp_type(cls, cpp_type: str) -> List[str]: - return [cls.cpp_type_to_user_include_path(cpp_type)] + includes.append('userver/chaotic/io/' + camel_to_snake_case(cpp_type.replace('::', '/')) + '.hpp') + + return includes def _primitive_parser_type(self) -> str: raw_cpp_type = f'USERVER_NAMESPACE::chaotic::Primitive<{self.raw_cpp_type.in_global_scope()}>' @@ -247,10 +251,10 @@ class CppPrimitiveType(CppType): __hash__ = CppType.__hash__ - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: includes = [] if self.user_cpp_type: - includes += self.get_include_by_cpp_type(self.user_cpp_type) + includes += self.get_includes_by_cpp_type(self.user_cpp_type) assert self.json_schema type_ = self.json_schema.type # type: ignore @@ -258,13 +262,13 @@ def declaration_includes(self) -> List[str]: includes.append('cstdint') elif type_ in ('number', 'boolean'): pass - elif type_ == 'string': + elif type_ in ('string', 'file'): includes.append('string') else: raise NotImplementedError(type_) return includes - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: includes = ['userver/chaotic/primitive.hpp'] if not self.validators.is_none(): includes.append('userver/chaotic/validators.hpp') @@ -344,16 +348,16 @@ def cpp_user_name(self) -> str: cpp_type = 'USERVER_NAMESPACE::' + cpp_type[len(USERVER_COLONCOLON) :] return cpp_type - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: includes = [] if self.user_cpp_type: - includes += self.get_include_by_cpp_type(self.user_cpp_type) - includes += self.get_include_by_cpp_type(self.format_cpp_type) + includes += self.get_includes_by_cpp_type(self.user_cpp_type) + includes += self.get_includes_by_cpp_type(self.format_cpp_type) includes.append('string') return includes - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: includes = ['userver/chaotic/primitive.hpp'] includes.append('userver/chaotic/with_type.hpp') return includes @@ -406,7 +410,7 @@ def cpp_user_name(self) -> str: else: return self.cpp_name - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: if self.indirect: return ['userver/utils/box.hpp'] if self.self_ref: @@ -414,7 +418,7 @@ def declaration_includes(self) -> List[str]: else: return self.orig_cpp_type.declaration_includes() - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: if self.indirect: return ['userver/chaotic/ref.hpp'] if self.self_ref: @@ -444,21 +448,21 @@ class CppIntEnumItem: raw_name: str cpp_name: str - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: return ['fmt/core.h'] @dataclasses.dataclass class CppIntEnum(CppType): name: str - enums: List[CppIntEnumItem] + enums: list[CppIntEnumItem] __hash__ = CppType.__hash__ - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: return ['fmt/core.h'] - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: return [ 'cstdint', 'userver/chaotic/exception.hpp', @@ -484,22 +488,22 @@ class CppStringEnumItem: raw_name: str cpp_name: str - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: return ['fmt/core.h'] @dataclasses.dataclass class CppStringEnum(CppType): name: str - enums: List[CppStringEnumItem] + enums: list[CppStringEnumItem] default: Optional[EnumItemName] __hash__ = CppType.__hash__ - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: return ['string', 'fmt/core.h'] - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: return [ 'userver/chaotic/exception.hpp', 'userver/chaotic/primitive.hpp', @@ -604,7 +608,7 @@ def cpp_field_parse_type(self) -> str: @dataclasses.dataclass class CppStruct(CppType): - fields: Dict[str, CppStructField] + fields: dict[str, CppStructField] # 'None' means 'do not generate extra member' extra_type: Union[CppType, bool, None] = False autodiscover_default_dict: bool = False @@ -665,7 +669,7 @@ def parser_type(self, ns: str, name: str) -> str: return f'USERVER_NAMESPACE::chaotic::WithType<{parser_type}, {dict_type}>' return parser_type - def subtypes(self) -> List[CppType]: + def subtypes(self) -> list[CppType]: types = [field.schema for field in self.fields.values()] if isinstance(self.extra_type, CppType) and not self._is_default_dict(): types.append(self.extra_type) @@ -679,10 +683,10 @@ def extra_container(self) -> str: kwargs.get('x-taxi-cpp-extra-type', 'std::unordered_map'), ) - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: includes = ['optional'] if self.user_cpp_type: - includes += self.get_include_by_cpp_type(self.user_cpp_type) + includes += self.get_includes_by_cpp_type(self.user_cpp_type) for field in self.fields.values(): includes.extend(field.schema.declaration_includes()) @@ -690,7 +694,7 @@ def declaration_includes(self) -> List[str]: includes.append('string') if isinstance(self.extra_type, CppType): extra_container = self.extra_container() - includes += self.get_include_by_cpp_type(extra_container) + includes += self.get_includes_by_cpp_type(extra_container) includes.extend(self.extra_type.declaration_includes()) else: includes.append('userver/formats/json/value.hpp') @@ -699,7 +703,7 @@ def declaration_includes(self) -> List[str]: includes.append('userver/utils/default_dict.hpp') return includes - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: includes = [ 'userver/formats/parse/common_containers.hpp', 'userver/formats/serialize/common_containers.hpp', @@ -718,7 +722,7 @@ def definition_includes(self) -> List[str]: if isinstance(self.extra_type, CppType): includes.extend(self.extra_type.definition_includes()) if self._is_default_dict(): - includes += self.get_include_by_cpp_type( + includes += self.get_includes_by_cpp_type( 'userver::utils::DefaultDict<>', ) return includes @@ -767,7 +771,7 @@ def without_json_schema(self) -> 'CppArray': tmp.items = self.items.without_json_schema() return tmp - def subtypes(self) -> List[CppType]: + def subtypes(self) -> list[CppType]: return [self.items] def parser_type(self, ns: str, name: str) -> str: @@ -787,13 +791,13 @@ def parser_type(self, ns: str, name: str) -> str: parser_type = f'USERVER_NAMESPACE::chaotic::WithType<{parser_type}, {user_cpp_type}>' return parser_type - def declaration_includes(self) -> List[str]: - includes = self.get_include_by_cpp_type(self.container) + self.items.declaration_includes() + def declaration_includes(self) -> list[str]: + includes = self.get_includes_by_cpp_type(self.container) + self.items.declaration_includes() if self.user_cpp_type: - includes += self.get_include_by_cpp_type(self.user_cpp_type) + includes += self.get_includes_by_cpp_type(self.user_cpp_type) return includes - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: return [ 'userver/chaotic/array.hpp', 'userver/chaotic/with_type.hpp', @@ -812,20 +816,20 @@ def flatten(data: list) -> list: @dataclasses.dataclass class CppStructAllOf(CppType): - parents: List[CppType] + parents: list[CppType] KNOWN_X_PROPERTIES = ['x-usrv-cpp-type', 'x-taxi-cpp-type'] __hash__ = CppType.__hash__ - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: includes = [] if self.user_cpp_type: - includes += self.get_include_by_cpp_type(self.user_cpp_type) + includes += self.get_includes_by_cpp_type(self.user_cpp_type) includes += flatten([item.declaration_includes() for item in self.parents]) return includes - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: return [ 'userver/formats/common/merge.hpp', 'userver/chaotic/primitive.hpp', @@ -834,7 +838,7 @@ def definition_includes(self) -> List[str]: def parser_type(self, ns: str, name: str) -> str: return self._primitive_parser_type() - def subtypes(self) -> List[CppType]: + def subtypes(self) -> list[CppType]: return self.parents def need_dom_parser(self) -> bool: @@ -849,16 +853,16 @@ def need_operator_eq(self) -> bool: @dataclasses.dataclass class CppVariant(CppType): - variants: List[CppType] + variants: list[CppType] KNOWN_X_PROPERTIES = ['x-usrv-cpp-type', 'x-taxi-cpp-type'] __hash__ = CppType.__hash__ - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: includes = [] if self.user_cpp_type: - includes += self.get_include_by_cpp_type(self.user_cpp_type) + includes += self.get_includes_by_cpp_type(self.user_cpp_type) return ( includes + [ @@ -869,7 +873,7 @@ def declaration_includes(self) -> List[str]: + flatten([item.declaration_includes() for item in self.variants]) ) - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: return [ 'userver/chaotic/primitive.hpp', 'userver/chaotic/variant.hpp', @@ -885,7 +889,7 @@ def parser_type(self, ns: str, name: str) -> str: else: return parser_type - def subtypes(self) -> List[CppType]: + def subtypes(self) -> list[CppType]: return self.variants def need_operator_lshift(self) -> bool: @@ -895,21 +899,21 @@ def need_operator_lshift(self) -> bool: @dataclasses.dataclass class CppVariantWithDiscriminator(CppType): property_name: str - variants: Dict[str, CppType] + variants: dict[str, CppType] mapping_type: types.MappingType = types.MappingType.STR KNOWN_X_PROPERTIES = ['x-usrv-cpp-type', 'x-taxi-cpp-type'] __hash__ = CppType.__hash__ - def declaration_includes(self) -> List[str]: + def declaration_includes(self) -> list[str]: includes = ['variant', 'userver/chaotic/oneof_with_discriminator.hpp'] if self.user_cpp_type: - includes += self.get_include_by_cpp_type(self.user_cpp_type) + includes += self.get_includes_by_cpp_type(self.user_cpp_type) return includes + flatten([item.declaration_includes() for item in self.variants.values()]) - def definition_includes(self) -> List[str]: + def definition_includes(self) -> list[str]: return ['userver/formats/json/serialize_variant.hpp'] + flatten([ item.definition_includes() for item in self.variants.values() ]) diff --git a/chaotic/chaotic/compilers/dynamic_config.py b/chaotic/chaotic/compilers/dynamic_config.py index 6e37d5294174..ce6e7e0300d7 100644 --- a/chaotic/chaotic/compilers/dynamic_config.py +++ b/chaotic/chaotic/compilers/dynamic_config.py @@ -3,10 +3,6 @@ import pathlib import subprocess from typing import Any -from typing import Dict -from typing import List -from typing import Set -from typing import Tuple import jinja2 import yaml @@ -16,12 +12,16 @@ from chaotic.back.cpp import renderer from chaotic.back.cpp import translator from chaotic.back.cpp import types +from chaotic.front import ref from chaotic.front.types import ResolvedSchemas from chaotic.main import generate_cpp_name_func from chaotic.main import NameMapItem from chaotic.main import read_schemas -INCLUDE_DIR = str(pathlib.Path(__file__).parent.parent.parent / 'include') +INCLUDE_DIRS = [ + str(pathlib.Path(__file__).parent.parent.parent / 'include'), + str(pathlib.Path(__file__).parent.parent.parent.parent / 'universal' / 'include'), +] TEMPLATE_DIR = str(pathlib.Path(__file__).parent / 'dynamic_config' / 'templates') CLANG_FORMAT_BIN = None @@ -80,25 +80,25 @@ def enrich_jinja_env(env: jinja2.Environment) -> None: class CompilerBase: def __init__(self) -> None: - self._variables_types: Dict[str, Dict[str, types.CppType]] = {} - self._definitions: Dict[ + self._variables_types: dict[str, dict[str, types.CppType]] = {} + self._definitions: dict[ str, - Tuple[ResolvedSchemas, Dict[str, types.CppType]], + tuple[ResolvedSchemas, dict[str, types.CppType]], ] = {} - self._defaults: Dict[str, Any] = {} - self.seen_includes: Dict[str, Set[str]] = {} + self._defaults: dict[str, Any] = {} + self.seen_includes: dict[str, set[str]] = {} - def extract_definition_names(self, filepath: str) -> List[str]: + def extract_definition_names(self, filepath: str) -> list[str]: with open(filepath, 'r') as ifile: content = yaml.load(ifile, Loader=yaml.CLoader) return list(set(self._extract_definition_names(content)) - set([''])) - def _extract_definition_names(self, content: Any) -> List[str]: + def _extract_definition_names(self, content: Any) -> list[str]: if isinstance(content, dict): refs = [] - ref = content.get('$ref') - if isinstance(ref, str): - filename = ref.split('#')[0] + ref_ = content.get('$ref') + if isinstance(ref_, str): + filename = ref.Ref(ref_).file refs.append(filename) for value in content.values(): @@ -130,7 +130,7 @@ def parse_definition( self, filepath: str, name: str, - include_dirs: List[str] = [], + include_dirs: list[str] = [], ) -> None: name_lower = self.format_ns_name(name) name_map = [NameMapItem('/([^/]+)/={0}')] @@ -148,16 +148,16 @@ def parse_definition( self._definitions[name] = (schemas, types) self.seen_includes[filepath] = seen_includes - def definitions_includes_hpp(self) -> List[str]: + def definitions_includes_hpp(self) -> list[str]: types = self._collect_types() - includes: List[str] = [] + includes: list[str] = [] for type_ in types.values(): includes += type_.declaration_includes() return sorted(set(includes)) - def definitions_includes_cpp(self) -> List[str]: + def definitions_includes_cpp(self) -> list[str]: types = self._collect_types() - includes: List[str] = [] + includes: list[str] = [] for type_ in types.values(): includes += type_.definition_includes() return sorted(set(includes)) @@ -167,7 +167,7 @@ def parse_variable( filepath: str, name: str, namespace: str, - include_dirs: List[str] = [], + include_dirs: list[str] = [], ) -> None: name_lower = self.format_ns_name(name) name_map = [ @@ -193,13 +193,13 @@ def parse_variable( self._defaults[name] = self._read_default(filepath) self.seen_includes[filepath] = seen_includes - def _collect_schemas(self) -> List[ResolvedSchemas]: + def _collect_schemas(self) -> list[ResolvedSchemas]: schemas = [] for def_schemas, _definitions in self._definitions.values(): schemas.append(def_schemas) return schemas - def _collect_types(self) -> Dict[str, types.CppType]: + def _collect_types(self) -> dict[str, types.CppType]: types = {} for _def_schemas, definitions in self._definitions.values(): types.update(definitions) @@ -212,8 +212,8 @@ def _generate_types( erase_prefix: str, name_map, fname: str, - include_dirs: List[str], - ) -> Tuple[ResolvedSchemas, Dict[str, types.CppType], Set[str]]: + include_dirs: list[str], + ) -> tuple[ResolvedSchemas, dict[str, types.CppType], set[str]]: schemas = read_schemas( erase_path_prefix=erase_prefix, filepaths=[filepath], @@ -227,7 +227,7 @@ def _generate_types( ) gen = translator.Generator( config=translator.GeneratorConfig( - include_dirs=include_dirs + [INCLUDE_DIR], + include_dirs=include_dirs + INCLUDE_DIRS, namespaces={fname: namespace}, infile_to_name_func=cpp_name_func, autodiscover_default_dict=True, @@ -240,22 +240,22 @@ def _generate_types( ) return schemas, types, gen.seen_includes - def variables_includes_hpp(self, name: str) -> List[str]: + def variables_includes_hpp(self, name: str) -> list[str]: types = self._variables_types[name] - includes: List[str] = [] + includes: list[str] = [] for type_ in types.values(): includes += type_.declaration_includes() return sorted(set(includes)) - def variables_includes_cpp(self, name: str) -> List[str]: + def variables_includes_cpp(self, name: str) -> list[str]: types = self._variables_types[name] - includes: List[str] = [] + includes: list[str] = [] for type_ in types.values(): includes += type_.definition_includes() return sorted(set(includes)) - def variables_external_includes_hpp(self, name: str) -> List[str]: + def variables_external_includes_hpp(self, name: str) -> list[str]: types = self._variables_types[name] return self.renderer_for_variable( name, @@ -273,8 +273,8 @@ def _jinja(self) -> jinja2.Environment: def create_aliases( self, - types: Dict[str, types.CppType], - ) -> List[Tuple[str, str]]: + types: dict[str, types.CppType], + ) -> list[tuple[str, str]]: return [] def renderer_for_variable( diff --git a/chaotic/chaotic/front/parser.py b/chaotic/chaotic/front/parser.py index b7200b2dd8b4..a4592ff9448a 100644 --- a/chaotic/chaotic/front/parser.py +++ b/chaotic/chaotic/front/parser.py @@ -3,20 +3,22 @@ import dataclasses import os import re -from typing import Dict +from typing import Any from typing import Generator -from typing import List from typing import NoReturn from typing import Optional from typing import Union from chaotic import error +from chaotic.front import ref from chaotic.front import types @dataclasses.dataclass(frozen=True) class ParserConfig: erase_prefix: str + # Allow type: file + allow_file: bool = False @dataclasses.dataclass @@ -24,7 +26,7 @@ class ParserState: # Current location in file infile_path: str - schemas: Dict[str, types.Schema] + schemas: dict[str, types.Schema] schema_type: str = 'jsonschema' @@ -89,7 +91,7 @@ def do_parse_schema(self, input__: dict) -> Union[types.Schema, types.Ref]: def _parse_allof(self, variants: list, input__: dict) -> types.AllOf: raw = types.AllOfRaw(**input__) - variables: List[types.Schema] = [] + variables: list[types.Schema] = [] with self._path_enter('allOf') as _: for i, variant in enumerate(variants): with self._path_enter(str(i)) as _: @@ -113,7 +115,10 @@ def _parse_oneof(self, variants: list, input__: dict) -> types.Schema: with self._path_enter(str(i)) as _: type_ = self._parse_schema(variant) variables.append(type_) - obj = types.OneOfWithoutDiscriminator(oneOf=variables) + obj = types.OneOfWithoutDiscriminator( + oneOf=variables, + nullable=raw.nullable, + ) obj.x_properties = raw.x_properties # type:ignore return obj @@ -129,7 +134,10 @@ def _parse_oneof_i( self._raise('Not a $ref in oneOf with discriminator') assert self._state - dest_type = self._state.schemas.get(type_.ref) + dest_type: Any = type_ + while isinstance(dest_type, types.Ref): + dest_type = self._state.schemas.get(dest_type.ref) + # TODO: fix in TAXICOMMON-8910 # # if not dest_type: @@ -139,7 +147,7 @@ def _parse_oneof_i( # ) if dest_type: if not isinstance(dest_type, types.SchemaObject): - self._raise('oneOf $ref to non-object') + self._raise(f'oneOf $ref to non-object ({type(dest_type).__name__})') if discriminator_property not in (dest_type.properties or {}): self._raise( f'No discriminator property "{discriminator_property}"', @@ -150,7 +158,7 @@ def _parse_oneof_i( def _parse_oneof_disc_mapping( self, user_mapping: dict, - variables: List[types.Ref], + variables: list[types.Ref], ) -> types.DiscMapping: with self._path_enter('discriminator/mapping') as _: idx_mapping = collections.defaultdict(list) @@ -171,11 +179,11 @@ def _parse_oneof_disc_mapping( idx_mapping[abs_ref].append(key) - for ref in variables: - assert isinstance(ref, types.Ref) - map_value = idx_mapping.pop(ref.ref, None) + for ref_ in variables: + assert isinstance(ref_, types.Ref) + map_value = idx_mapping.pop(ref_.ref, None) if map_value is None: - self._raise(f'Missing $ref in mapping: {ref.ref}') + self._raise(f'Missing $ref in mapping: {ref_.ref}') mapping.append(map_value) @@ -195,7 +203,7 @@ def _parse_oneof_w_discriminator( discriminator_property = input_['discriminator']['propertyName'] - variables: List[types.Ref] = [] + variables: list[types.Ref] = [] with self._path_enter('oneOf') as _: for i, variant in enumerate(variants): with self._path_enter(str(i)) as _: @@ -209,11 +217,12 @@ def _parse_oneof_w_discriminator( if user_mapping is not None: mapping = self._parse_oneof_disc_mapping(user_mapping, variables) else: - mapping = types.DiscMapping(str_values=[[ref.ref.split('/')[-1]] for ref in variables], int_values=None) + mapping = types.DiscMapping(str_values=[[ref_.ref.split('/')[-1]] for ref_ in variables], int_values=None) obj = types.OneOfWithDiscriminator( oneOf=variables, discriminator_property=discriminator_property, mapping=mapping, + nullable=input_.get('nullable', False), ) del input_['discriminator'] @@ -229,7 +238,6 @@ def _path_enter(self, path_component: str) -> Generator[None, None, None]: if self._state.infile_path: self._state.infile_path += '/' self._state.infile_path += path_component - # print(f'enter => {self._state.infile_path}') try: yield @@ -240,7 +248,7 @@ def _path_enter(self, path_component: str) -> Generator[None, None, None]: except ParserError: raise except Exception as exc: # pylint: disable=broad-exception-caught - self._raise(exc.__repr__()) + self._raise(str(exc)) else: self._state.infile_path = old @@ -328,30 +336,40 @@ def _parse_string(self, input_: dict) -> types.String: fmt = None return types.String(**input_, format=fmt) + def _parse_file(self, input_: dict) -> types.String: + if not self._config.allow_file: + with self._path_enter('type') as _: + self._raise('"file" type is not allowed') + + return self._parse_string(input_) + REF_SHRINK_RE = re.compile('/[^/]+/\\.\\./') REF_SHRINK_DOT_RE = re.compile('/\\./') @staticmethod - def _normalize_ref(ref: str) -> str: - while SchemaParser.REF_SHRINK_RE.search(ref): - ref = re.sub(SchemaParser.REF_SHRINK_RE, '/', ref) - while SchemaParser.REF_SHRINK_DOT_RE.search(ref): - ref = re.sub(SchemaParser.REF_SHRINK_DOT_RE, '/', ref) - return ref - - def _make_abs_ref(self, ref: str) -> str: + def _normalize_ref(ref_: str) -> str: + ref_ = '/' + ref_ # for regex simplicity + + while SchemaParser.REF_SHRINK_RE.search(ref_): + ref_ = re.sub(SchemaParser.REF_SHRINK_RE, '/', ref_) + while SchemaParser.REF_SHRINK_DOT_RE.search(ref_): + ref_ = re.sub(SchemaParser.REF_SHRINK_DOT_RE, '/', ref_) + + return ref_[1:] + + def _make_abs_ref(self, ref_: str) -> str: assert self._state - if ref.startswith('#'): + if not ref.Ref(ref_).file: # Local $ref - return self.full_vfilepath + ref + return self.full_vfilepath + ref_ else: my_ref = '/'.join(self.full_vfilepath.split('/')[:-1]) - file, infile = ref.split('#') - out_file = self._normalize_ref(os.path.join(my_ref, file)) - return out_file + '#' + infile + reference = ref.Ref(ref_) + out_file = self._normalize_ref(os.path.join(my_ref, reference.file)) + return out_file + '#' + reference.fragment - def _parse_ref(self, ref: str, input_: dict) -> types.Ref: + def _parse_ref(self, ref_: str, input_: dict) -> types.Ref: assert self._state fields = set(input_.keys()) @@ -376,7 +394,7 @@ def _parse_ref(self, ref: str, input_: dict) -> types.Ref: self._raise(f'Unknown field(s) {list(fields)}') with self._path_enter('$ref') as _: - abs_ref = self._make_abs_ref(ref) + abs_ref = self._make_abs_ref(ref_) ref_value = types.Ref( ref=abs_ref, indirect=indirect, @@ -399,6 +417,7 @@ def parsed_schemas(self) -> types.ParsedSchemas: 'integer': SchemaParser._parse_int, 'number': SchemaParser._parse_number, 'string': SchemaParser._parse_string, + 'file': SchemaParser._parse_file, 'array': SchemaParser._parse_array, 'object': SchemaParser._parse_object, } diff --git a/chaotic/chaotic/front/ref.py b/chaotic/chaotic/front/ref.py new file mode 100644 index 000000000000..a56548c7b9b6 --- /dev/null +++ b/chaotic/chaotic/front/ref.py @@ -0,0 +1,10 @@ +class Ref: + def __init__(self, ref: str) -> None: + self._ref = ref + + parts = ref.split('#') + if len(parts) != 2: + raise Exception(f'Error in $ref ({ref}): there should be exactly one "#" inside') + + self.file = parts[0] + self.fragment = parts[1] diff --git a/chaotic/chaotic/front/ref_resolver.py b/chaotic/chaotic/front/ref_resolver.py index 9166b82b965c..4498d97b62f5 100644 --- a/chaotic/chaotic/front/ref_resolver.py +++ b/chaotic/chaotic/front/ref_resolver.py @@ -1,9 +1,6 @@ import collections from typing import Any -from typing import Dict -from typing import List from typing import Optional -from typing import Set from chaotic import error from chaotic.front import types @@ -13,9 +10,9 @@ class ResolverError(Exception): pass -def sort_dfs(nodes: Set[str], edges: Dict[str, List[str]]) -> List[str]: +def sort_dfs(nodes: set[str], edges: dict[str, list[str]]) -> list[str]: visited = set() - visiting: List[str] = [] + visiting: list[str] = [] sorted_nodes = [] def do_node(node: str): @@ -152,9 +149,9 @@ def _search_refs(cls, data: Any, *, inside_items: bool): def sort_json_types( self, - types: Dict[str, Any], + types: dict[str, Any], erase_path_prefix: str = '', - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Sorts not-yet-parsed schemas. Required for correct allOf/oneOf parsing. """ diff --git a/chaotic/chaotic/front/types.py b/chaotic/chaotic/front/types.py index 65d36ae5ca7e..777acfa1241e 100644 --- a/chaotic/chaotic/front/types.py +++ b/chaotic/chaotic/front/types.py @@ -4,8 +4,6 @@ import typing from typing import Any from typing import Callable -from typing import Dict -from typing import List from typing import Optional from typing import TypeVar from typing import Union @@ -22,7 +20,7 @@ def __init__(self, field: str, msg: str) -> None: def is_ignored_prefix(arg: str) -> bool: if arg.startswith('x-'): return True - if arg in ('description', 'example'): + if arg in ('description', 'example', 'title'): return True return False @@ -77,7 +75,7 @@ def validate_type(field_name: str, value, type_) -> None: f'field "{field_name}" has wrong type', ) except TypeError: - # TODO: type=List[str] + # TODO: type=list[str] pass @@ -105,7 +103,7 @@ def __post_init__(self) -> None: @dataclasses.dataclass class Schema(Base): - x_properties: Dict[str, Any] = dataclasses.field( + x_properties: dict[str, Any] = dataclasses.field( init=False, default_factory=dict, ) @@ -170,7 +168,7 @@ class Ref(Schema): schema: Schema = _NOT_IMPL def __post_init__(self): - assert self.ref.find('/../') == -1 + assert self.ref.find('/../') == -1, self.ref __hash__ = Schema.__hash__ @@ -181,6 +179,7 @@ class Boolean(Schema): type: str = 'boolean' default: Optional[bool] = None nullable: bool = False + deprecated: bool = False __hash__ = Schema.__hash__ @@ -217,8 +216,9 @@ class Integer(Schema): exclusiveMinimum: Optional[int] = None exclusiveMaximum: Optional[int] = None # TODO: multipleOf - enum: Optional[List[int]] = None + enum: Optional[list[int]] = None format: Optional[IntegerFormat] = None + deprecated: bool = False def __post_init__(self) -> None: super().__post_init__() @@ -244,6 +244,7 @@ class Number(Schema): exclusiveMinimum: Optional[Union[float, int]] = None exclusiveMaximum: Optional[Union[float, int]] = None format: Optional[str] = None + deprecated: bool = False # TODO: multipleOf __hash__ = Schema.__hash__ @@ -285,11 +286,12 @@ class String(Schema): type: str = 'string' default: Optional[str] = None nullable: bool = False - enum: Optional[List[str]] = None + enum: Optional[list[str]] = None pattern: Optional[str] = None format: Optional[StringFormat] = None minLength: Optional[int] = None maxLength: Optional[int] = None + deprecated: bool = False def __post_init__(self) -> None: super().__post_init__() @@ -313,6 +315,7 @@ class Array(Schema): nullable: bool = False minItems: Optional[int] = None maxItems: Optional[int] = None + deprecated: bool = False def visit_children(self, cb: Callable[[Schema, Schema], None]) -> None: cb(self.items, self) @@ -327,16 +330,17 @@ class SchemaObjectRaw: type: str additionalProperties: Any properties: Optional[dict] = None - required: Optional[List[str]] = None + required: Optional[list[str]] = None nullable: bool = False + deprecated: bool = False @smart_fields @dataclasses.dataclass class SchemaObject(Schema): additionalProperties: Union[Schema, bool] - properties: Dict[str, Schema] - required: Optional[List[str]] = None + properties: dict[str, Schema] + required: Optional[list[str]] = None nullable: bool = False def __post_init__(self) -> None: @@ -364,7 +368,7 @@ def visit_children(self, cb: Callable[[Schema, Schema], None]) -> None: @smart_fields @dataclasses.dataclass class AllOf(Schema): - allOf: List[Schema] # type: ignore + allOf: list[Schema] # type: ignore nullable: bool = False def visit_children(self, cb: Callable[[Schema, Schema], None]) -> None: @@ -380,6 +384,7 @@ def visit_children(self, cb: Callable[[Schema, Schema], None]) -> None: class AllOfRaw: allOf: list # type:ignore nullable: bool = False + deprecated: bool = False def __post_init__(self) -> None: if not self.allOf: @@ -389,7 +394,7 @@ def __post_init__(self) -> None: @smart_fields @dataclasses.dataclass class OneOfWithoutDiscriminator(Schema): - oneOf: List[Schema] # type:ignore + oneOf: list[Schema] # type:ignore nullable: bool = False def visit_children(self, cb: Callable[[Schema, Schema], None]) -> None: @@ -408,14 +413,14 @@ class MappingType(enum.Enum): @dataclasses.dataclass class DiscMapping: # only one list must be not none - str_values: Optional[List[List[str]]] = None - int_values: Optional[List[List[int]]] = None + str_values: Optional[list[list[str]]] = None + int_values: Optional[list[list[int]]] = None def append(self, value: list): if self.str_values is not None: - self.str_values.append(typing.cast(List[str], value)) + self.str_values.append(typing.cast(list[str], value)) elif self.int_values is not None: - self.int_values.append(typing.cast(List[int], value)) + self.int_values.append(typing.cast(list[int], value)) def enable_str(self): self.str_values = [] @@ -431,11 +436,11 @@ def get_type(self) -> MappingType: return MappingType.STR - def as_strs(self) -> List[List[str]]: - return typing.cast(List[List[str]], self.str_values) + def as_strs(self) -> list[list[str]]: + return typing.cast(list[list[str]], self.str_values) - def as_ints(self) -> List[List[int]]: - return typing.cast(List[List[int]], self.int_values) + def as_ints(self) -> list[list[int]]: + return typing.cast(list[list[int]], self.int_values) def is_int(self): return self.int_values is not None @@ -447,7 +452,7 @@ def is_str(self): @smart_fields @dataclasses.dataclass class OneOfWithDiscriminator(Schema): - oneOf: List[Ref] # type:ignore + oneOf: list[Ref] # type:ignore discriminator_property: Optional[str] = None mapping: DiscMapping = dataclasses.field(default_factory=DiscMapping) nullable: bool = False @@ -464,7 +469,7 @@ def visit_children(self, cb: Callable[[Schema, Schema], None]) -> None: @dataclasses.dataclass class OneOfDiscriminatorRaw: propertyName: str # type:ignore - mapping: Optional[Dict[str, str]] = None + mapping: Optional[dict[str, str]] = None @smart_fields @@ -472,6 +477,8 @@ class OneOfDiscriminatorRaw: class OneOfRaw: oneOf: list # type:ignore discriminator: Optional[dict] = None + nullable: bool = False + deprecated: bool = False def __post_init__(self) -> None: if not self.oneOf: @@ -480,12 +487,12 @@ def __post_init__(self) -> None: @dataclasses.dataclass class ParsedSchemas: - schemas: Dict[str, Schema] = dataclasses.field( + schemas: dict[str, Schema] = dataclasses.field( default_factory=collections.OrderedDict, ) @staticmethod - def merge(schemas: List['ParsedSchemas']) -> 'ParsedSchemas': + def merge(schemas: list['ParsedSchemas']) -> 'ParsedSchemas': result = ParsedSchemas() for schema in schemas: result.schemas.update(schema.schemas) @@ -494,4 +501,4 @@ def merge(schemas: List['ParsedSchemas']) -> 'ParsedSchemas': @dataclasses.dataclass class ResolvedSchemas: - schemas: Dict[str, Schema] + schemas: dict[str, Schema] diff --git a/chaotic/chaotic/main.py b/chaotic/chaotic/main.py index 271779187382..c8c56bc473c8 100644 --- a/chaotic/chaotic/main.py +++ b/chaotic/chaotic/main.py @@ -6,8 +6,6 @@ import sys from typing import Any from typing import Callable -from typing import Dict -from typing import List from typing import Optional import yaml @@ -115,7 +113,7 @@ def parse_args() -> argparse.Namespace: def generate_cpp_name_func( - name_map: List[NameMapItem], + name_map: list[NameMapItem], erase_prefix: str, ) -> Callable: def cpp_name_func(schema_name: str, stem: str) -> str: @@ -129,7 +127,7 @@ def cpp_name_func(schema_name: str, stem: str) -> str: return cpp_name_func -def vfilepath_from_filepath(filepath: str, file_map: List[NameMapItem]) -> str: +def vfilepath_from_filepath(filepath: str, file_map: list[NameMapItem]) -> str: for item in file_map: vfilepath = item.match(filepath, stem=pathlib.Path(filepath).stem) if vfilepath: @@ -168,8 +166,8 @@ def traverse_dfs(path: str, data: Any): def extract_schemas_to_scan( inp: dict, - name_map: List[NameMapItem], -) -> Dict[str, Any]: + name_map: list[NameMapItem], +) -> dict[str, Any]: schemas = [] gen = traverse_dfs('/', inp) @@ -192,10 +190,10 @@ def extract_schemas_to_scan( def read_schemas( erase_path_prefix: str, - filepaths: List[str], + filepaths: list[str], name_map, file_map, - dependencies: List[types.ResolvedSchemas] = [], + dependencies: list[types.ResolvedSchemas] = [], ) -> types.ResolvedSchemas: config = front_parser.ParserConfig(erase_prefix=erase_path_prefix) rr = ref_resolver.RefResolver() diff --git a/chaotic/include/userver/chaotic/io/boost/uuids/uuid.hpp b/chaotic/include/userver/chaotic/io/boost/uuids/uuid.hpp index 8365631f7e34..167ffac411e4 100644 --- a/chaotic/include/userver/chaotic/io/boost/uuids/uuid.hpp +++ b/chaotic/include/userver/chaotic/io/boost/uuids/uuid.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include diff --git a/chaotic/include/userver/chaotic/io/std/int32_t.hpp b/chaotic/include/userver/chaotic/io/std/int32_t.hpp new file mode 100644 index 000000000000..d0879af92bfb --- /dev/null +++ b/chaotic/include/userver/chaotic/io/std/int32_t.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace chaotic::convert { + +template +std::enable_if_t, T> Convert(const std::int32_t& value, To) { + return utils::numeric_cast(value); +} + +template +std::enable_if_t, std::int32_t> Convert(const T& value, To) { + return utils::numeric_cast(value); +} + +} // namespace chaotic::convert + +USERVER_NAMESPACE_END diff --git a/chaotic/include/userver/chaotic/io/std/int64_t.hpp b/chaotic/include/userver/chaotic/io/std/int64_t.hpp new file mode 100644 index 000000000000..d0fc56a02095 --- /dev/null +++ b/chaotic/include/userver/chaotic/io/std/int64_t.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace chaotic::convert { + +template +std::enable_if_t, T> Convert(const std::int64_t& value, To) { + return utils::numeric_cast(value); +} + +template +std::enable_if_t, std::int64_t> Convert(const T& value, To) { + return utils::numeric_cast(value); +} + +} // namespace chaotic::convert + +USERVER_NAMESPACE_END diff --git a/chaotic/integration_tests/CMakeLists.txt b/chaotic/integration_tests/CMakeLists.txt index 34c7fa4f01f8..91ea8afcf578 100644 --- a/chaotic/integration_tests/CMakeLists.txt +++ b/chaotic/integration_tests/CMakeLists.txt @@ -18,6 +18,7 @@ userver_target_generate_chaotic( INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${CMAKE_CURRENT_SOURCE_DIR}/../../universal/include LAYOUT "/definitions/([^/]*)/=ns::{0}" OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/src diff --git a/chaotic/integration_tests/schemas/container_format.yaml b/chaotic/integration_tests/schemas/container_format.yaml new file mode 100644 index 000000000000..54b60d13074d --- /dev/null +++ b/chaotic/integration_tests/schemas/container_format.yaml @@ -0,0 +1,11 @@ +definitions: + ContainerWithFormatItem: + type: object + additionalProperties: false + properties: + my_field: + type: array + x-usrv-cpp-container: std::unordered_set + items: + type: string + format: uuid diff --git a/chaotic/integration_tests/schemas/custom_cpp_type.yaml b/chaotic/integration_tests/schemas/custom_cpp_type.yaml index 118bc7eeec1e..e8545b6498e4 100644 --- a/chaotic/integration_tests/schemas/custom_cpp_type.yaml +++ b/chaotic/integration_tests/schemas/custom_cpp_type.yaml @@ -115,3 +115,10 @@ definitions: type: string x-usrv-cpp-type: my::CustomString x-usrv-cpp-type: userver::utils::DefaultDict + StrongTypedefObject: + type: object + additionalProperties: false + properties: + strong_typedef: + type: string + x-usrv-cpp-type: userver::utils::StrongTypedef diff --git a/chaotic/integration_tests/schemas/oneofdiscriminator.yaml b/chaotic/integration_tests/schemas/oneofdiscriminator.yaml index 41392f447a4d..6eb30a291281 100644 --- a/chaotic/integration_tests/schemas/oneofdiscriminator.yaml +++ b/chaotic/integration_tests/schemas/oneofdiscriminator.yaml @@ -56,3 +56,12 @@ definitions: properties: version: type: integer + + OneOfToRefToRef: + oneOf: + - $ref: '#/definitions/RefToC' + discriminator: + propertyName: version + + RefToC: + $ref: '#/definitions/C' diff --git a/chaotic/tests/back/cpp/conftest.py b/chaotic/tests/back/cpp/conftest.py index 3347723e616b..206b8ba47189 100644 --- a/chaotic/tests/back/cpp/conftest.py +++ b/chaotic/tests/back/cpp/conftest.py @@ -1,5 +1,4 @@ from collections import OrderedDict -from typing import Dict import pytest @@ -33,7 +32,7 @@ def func(input_: dict): @pytest.fixture(name='clean') def _clean(): - def func(ordered_dict: OrderedDict) -> Dict[str, CppType]: + def func(ordered_dict: OrderedDict) -> dict[str, CppType]: res = {} for key, val in ordered_dict.items(): res[key] = val.without_json_schema() diff --git a/chaotic/tests/back/cpp/test_external.py b/chaotic/tests/back/cpp/test_external.py index a06f4a962279..c1ce55acd35a 100644 --- a/chaotic/tests/back/cpp/test_external.py +++ b/chaotic/tests/back/cpp/test_external.py @@ -1,3 +1,5 @@ +import pytest + from chaotic.back.cpp import type_name from chaotic.back.cpp import types as cpp_types from chaotic.back.cpp.translator import Generator @@ -32,16 +34,14 @@ def parse(path, input_, external_schemas, external_types, cpp_name_func): return resolved_schemas, types -def test_import(cpp_name_func): +def test_import(cpp_name_func, cpp_primitive_type): ext_schemas, ext_types = parse('/type1', {'type': 'string'}, types.ResolvedSchemas(schemas={}), {}, cpp_name_func) assert ext_schemas.schemas == {'vfull#/type1': types.String(type='string')} assert ext_types == { - '::type1': cpp_types.CppPrimitiveType( - raw_cpp_type=type_name.TypeName('std::string'), - nullable=False, - user_cpp_type=None, - json_schema=types.String(type='string'), + '::type1': cpp_primitive_type( validators=cpp_types.CppPrimitiveValidator(prefix='type1'), + raw_cpp_type_str='std::string', + json_schema=types.String(type='string'), ), } @@ -65,3 +65,42 @@ def test_import(cpp_name_func): json_schema=new_schemas.schemas['vfull#/type2'], ), } + + +def test_duplicate_name(cpp_name_func): + config = parser.ParserConfig(erase_prefix='') + + schema_parser = parser.SchemaParser( + config=config, + full_filepath='full', + full_vfilepath='vfull', + ) + schema_parser.parse_schema('/type1', {'type': 'string'}) + schemas1 = schema_parser.parsed_schemas() + + schema_parser = parser.SchemaParser( + config=config, + full_filepath='full2', + full_vfilepath='vfull2', + ) + schema_parser.parse_schema('/type1', {'type': 'integer'}) + schemas2 = schema_parser.parsed_schemas() + + schemas = types.ParsedSchemas.merge([schemas1, schemas2]) + rr = ref_resolver.RefResolver() + resolved_schemas = rr.sort_schemas( + schemas, + ) + + gen = Generator( + config=GeneratorConfig( + namespaces={'vfull': '', 'vfull2': ''}, include_dirs=None, infile_to_name_func=cpp_name_func + ), + ) + + with pytest.raises( + BaseException, match='Duplicate type name: ::type1, generated from vfull2#/type1 and vfull#/type1' + ): + gen.generate_types( + resolved_schemas, + ) diff --git a/chaotic/tests/back/cpp/test_tr_array.py b/chaotic/tests/back/cpp/test_tr_array.py index 697923168028..0b0574679df7 100644 --- a/chaotic/tests/back/cpp/test_tr_array.py +++ b/chaotic/tests/back/cpp/test_tr_array.py @@ -1,11 +1,10 @@ from chaotic.back.cpp import type_name from chaotic.back.cpp.types import CppArray from chaotic.back.cpp.types import CppArrayValidator -from chaotic.back.cpp.types import CppPrimitiveType from chaotic.back.cpp.types import CppPrimitiveValidator -def test_array_int(simple_gen): +def test_array_int(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'array', 'items': {'type': 'integer'}}) assert types == { '::type': CppArray( @@ -13,12 +12,9 @@ def test_array_int(simple_gen): user_cpp_type=None, json_schema=None, nullable=False, - items=CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, - nullable=False, + items=cpp_primitive_type( validators=CppPrimitiveValidator(prefix='typeA'), + raw_cpp_type_str='int', ), container='std::vector', validators=CppArrayValidator(), @@ -26,7 +22,7 @@ def test_array_int(simple_gen): } -def test_array_array_with_validators(simple_gen): +def test_array_array_with_validators(simple_gen, cpp_primitive_type): types = simple_gen({ 'type': 'array', 'items': {'type': 'array', 'items': {'type': 'integer', 'minimum': 1}}, @@ -42,15 +38,12 @@ def test_array_array_with_validators(simple_gen): user_cpp_type=None, json_schema=None, nullable=False, - items=CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, - nullable=False, + items=cpp_primitive_type( validators=CppPrimitiveValidator( min=1, prefix='typeAA', ), + raw_cpp_type_str='int', ), container='std::vector', validators=CppArrayValidator(), diff --git a/chaotic/tests/back/cpp/test_tr_int.py b/chaotic/tests/back/cpp/test_tr_int.py index d383c33bf08a..2fa1c64cc23a 100644 --- a/chaotic/tests/back/cpp/test_tr_int.py +++ b/chaotic/tests/back/cpp/test_tr_int.py @@ -2,19 +2,15 @@ from chaotic.back.cpp import type_name from chaotic.back.cpp.types import CppIntEnum from chaotic.back.cpp.types import CppIntEnumItem -from chaotic.back.cpp.types import CppPrimitiveType from chaotic.back.cpp.types import CppPrimitiveValidator -def test_int(simple_gen): +def test_int(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer'}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, - nullable=False, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator(prefix='type'), + raw_cpp_type_str='int', ), } @@ -30,118 +26,99 @@ def test_wrong_type_x(simple_gen): assert exc.msg == 'Non-string x- property "x-usrv-cpp-type"' -def test_int_nullable(simple_gen): +def test_int_nullable(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer', 'nullable': True}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, - nullable=True, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator(prefix='type'), + raw_cpp_type_str='int', + nullable=True, ), } -def test_int_cpp_type(simple_gen): +def test_int_cpp_type(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer', 'x-usrv-cpp-type': 'X'}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type='X', - json_schema=None, - nullable=False, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator(prefix='type'), + raw_cpp_type_str='int', + user_cpp_type='X', ), } -def test_int_default(simple_gen): +def test_int_default(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer', 'default': 42}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - default=42, - json_schema=None, - nullable=False, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator(prefix='type'), + raw_cpp_type_str='int', + default=42, ), } -def test_int_min(simple_gen): +def test_int_min(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer', 'minimum': 1}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, - nullable=False, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator( min=1, prefix='type', ), + raw_cpp_type_str='int', ), } -def test_int_min_max(simple_gen): +def test_int_min_max(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer', 'minimum': 1, 'maximum': 10}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator( min=1, max=10, prefix='type', ), - nullable=False, + raw_cpp_type_str='int', ), } -def test_int_min_max_exclusive(simple_gen): +def test_int_min_max_exclusive(simple_gen, cpp_primitive_type): types = simple_gen({ 'type': 'integer', 'exclusiveMinimum': 1, 'exclusiveMaximum': 10, }) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator( exclusiveMin=1, exclusiveMax=10, prefix='type', ), - nullable=False, + raw_cpp_type_str='int', ), } -def test_int_min_max_exclusive_false(simple_gen): +def test_int_min_max_exclusive_false(simple_gen, cpp_primitive_type): types = simple_gen({ 'type': 'integer', 'exclusiveMinimum': False, 'exclusiveMaximum': False, }) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator(prefix='type'), - nullable=False, + raw_cpp_type_str='int', ), } -def test_int_min_max_exclusive_legacy(simple_gen): +def test_int_min_max_exclusive_legacy(simple_gen, cpp_primitive_type): types = simple_gen({ 'type': 'integer', 'exclusiveMinimum': True, @@ -150,42 +127,33 @@ def test_int_min_max_exclusive_legacy(simple_gen): 'maximum': 10, }) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - user_cpp_type=None, - json_schema=None, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator( exclusiveMin=2, exclusiveMax=10, prefix='type', ), - nullable=False, + raw_cpp_type_str='int', ), } -def test_int_format_int32(simple_gen): +def test_int_format_int32(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer', 'format': 'int32'}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('std::int32_t'), - user_cpp_type=None, - json_schema=None, - nullable=False, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator(prefix='type'), + raw_cpp_type_str='std::int32_t', ), } -def test_int_format_int64(simple_gen): +def test_int_format_int64(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'integer', 'format': 'int64'}) assert types == { - '::type': CppPrimitiveType( - raw_cpp_type=type_name.TypeName('std::int64_t'), - user_cpp_type=None, - json_schema=None, - nullable=False, + '::type': cpp_primitive_type( validators=CppPrimitiveValidator(prefix='type'), + raw_cpp_type_str='std::int64_t', ), } diff --git a/chaotic/tests/back/cpp/test_tr_object.py b/chaotic/tests/back/cpp/test_tr_object.py index 28d8d330c10c..587aa6be1976 100644 --- a/chaotic/tests/back/cpp/test_tr_object.py +++ b/chaotic/tests/back/cpp/test_tr_object.py @@ -6,7 +6,6 @@ from chaotic.back.cpp.types import CppPrimitiveValidator from chaotic.back.cpp.types import CppStruct from chaotic.back.cpp.types import CppStructField -from chaotic.back.cpp.types import CppStructPrimitiveField def test_empty(simple_gen): @@ -26,7 +25,7 @@ def test_empty(simple_gen): } -def test_additional_properties_simple(simple_gen): +def test_additional_properties_simple(simple_gen, cpp_primitive_type): schemas = simple_gen({ 'type': 'object', 'properties': {}, @@ -39,22 +38,19 @@ def test_additional_properties_simple(simple_gen): nullable=False, user_cpp_type=None, fields={}, - extra_type=CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - json_schema=None, - nullable=False, - user_cpp_type=None, + extra_type=cpp_primitive_type( validators=CppPrimitiveValidator( namespace='::type', prefix='Extra', ), + raw_cpp_type_str='int', ), ), } @pytest.mark.skip(reason='see comment in translator.py: _gen_field()') -def test_field_external(simple_gen): +def test_field_external(simple_gen, cpp_primitive_type): schemas = simple_gen({ 'type': 'object', 'properties': {'field': {'type': 'integer'}}, @@ -71,8 +67,12 @@ def test_field_external(simple_gen): 'field': CppStructField( name='field', required=False, - external_schema=CppStructPrimitiveField( - raw_cpp_type=type_name.TypeName('int'), + schema=cpp_primitive_type( + validators=CppPrimitiveValidator( + namespace='::type', + prefix='Field', + ), + raw_cpp_type_str='int', ), ), }, @@ -80,7 +80,7 @@ def test_field_external(simple_gen): } -def test_field_with_default(simple_gen): +def test_field_with_default(simple_gen, cpp_primitive_type): schemas = simple_gen({ 'type': 'object', 'properties': {'field': {'type': 'integer', 'default': 1}}, @@ -97,16 +97,13 @@ def test_field_with_default(simple_gen): 'field': CppStructField( name='field', required=False, - schema=CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - json_schema=None, - default=1, - nullable=False, - user_cpp_type=None, + schema=cpp_primitive_type( validators=CppPrimitiveValidator( namespace='::type', prefix='Field', ), + raw_cpp_type_str='int', + default=1, ), ), }, @@ -114,7 +111,7 @@ def test_field_with_default(simple_gen): } -def test_field_inplace(simple_gen): +def test_field_inplace(simple_gen, cpp_primitive_type): schemas = simple_gen({ 'type': 'object', 'properties': {'field': {'type': 'integer', 'minimum': 1}}, @@ -131,16 +128,13 @@ def test_field_inplace(simple_gen): 'field': CppStructField( name='field', required=False, - schema=CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - json_schema=None, - nullable=False, - user_cpp_type=None, + schema=cpp_primitive_type( validators=CppPrimitiveValidator( min=1, namespace='::type', prefix='Field', ), + raw_cpp_type_str='int', ), ), }, diff --git a/chaotic/tests/back/cpp/test_tr_oneof.py b/chaotic/tests/back/cpp/test_tr_oneof.py index 7ae6111ce156..8b217246b174 100644 --- a/chaotic/tests/back/cpp/test_tr_oneof.py +++ b/chaotic/tests/back/cpp/test_tr_oneof.py @@ -2,8 +2,6 @@ from chaotic.back.cpp.translator import Generator from chaotic.back.cpp.translator import GeneratorConfig from chaotic.front import ref_resolver -from chaotic.front.parser import ParserConfig -from chaotic.front.parser import SchemaParser from chaotic.front.types import MappingType from chaotic.main import generate_cpp_name_func from chaotic.main import NameMapItem @@ -36,13 +34,8 @@ def test_simple(simple_gen): assert foo_schema.variants[2].raw_cpp_type == type_name.TypeName('double') -def test_empty_mapping(clean): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_empty_mapping(clean, schema_parser): + parser = schema_parser parser.parse_schema( '/definitions/A', @@ -101,13 +94,8 @@ def test_empty_mapping(clean): assert list(foo_schema.variants.keys()) == ['A', 'B'] -def test_str_mapping(clean): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_str_mapping(clean, schema_parser): + parser = schema_parser parser.parse_schema( '/definitions/A', @@ -170,13 +158,8 @@ def test_str_mapping(clean): assert foo_schema.variants['bbb'].cpp_name == '::B' -def test_int_mapping(clean): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_int_mapping(clean, schema_parser): + parser = schema_parser parser.parse_schema( '/definitions/A', diff --git a/chaotic/tests/back/cpp/test_tr_ref.py b/chaotic/tests/back/cpp/test_tr_ref.py index ac14e1a1a02c..c0741d515350 100644 --- a/chaotic/tests/back/cpp/test_tr_ref.py +++ b/chaotic/tests/back/cpp/test_tr_ref.py @@ -4,18 +4,11 @@ from chaotic.back.cpp.types import CppRef from chaotic.back.cpp.types import CppStruct from chaotic.front import ref_resolver -from chaotic.front.parser import ParserConfig -from chaotic.front.parser import SchemaParser from chaotic.front.types import SchemaObject -def test_simple_ref(clean, cpp_name_func): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_simple_ref(clean, cpp_name_func, schema_parser): + parser = schema_parser parser.parse_schema( '/definitions/Type', diff --git a/chaotic/tests/back/cpp/test_tr_string.py b/chaotic/tests/back/cpp/test_tr_string.py index 28617124637d..de27623edfde 100644 --- a/chaotic/tests/back/cpp/test_tr_string.py +++ b/chaotic/tests/back/cpp/test_tr_string.py @@ -2,17 +2,14 @@ from chaotic.back.cpp import types as cpp_types -def test_simple(simple_gen): +def test_simple(simple_gen, cpp_primitive_type): types = simple_gen({'type': 'string'}) assert types == { - '::type': cpp_types.CppPrimitiveType( - raw_cpp_type=type_name.TypeName('std::string'), - user_cpp_type=None, - json_schema=None, - nullable=False, + '::type': cpp_primitive_type( validators=cpp_types.CppPrimitiveValidator( prefix='type', ), + raw_cpp_type_str='std::string', ), } diff --git a/chaotic/tests/back/cpp/test_types.py b/chaotic/tests/back/cpp/test_types.py index 8ec0c6757c3a..4cc8025ee8a4 100644 --- a/chaotic/tests/back/cpp/test_types.py +++ b/chaotic/tests/back/cpp/test_types.py @@ -7,13 +7,13 @@ def test_camel_to_snake_case_smoke(): assert types.camel_to_snake_case('unsigned') == 'unsigned' -def test_get_include_by_cpp_type_smoke(): - assert types.CppType.get_include_by_cpp_type('geometry::Distance') == [ +def test_get_includes_by_cpp_type_smoke(): + assert types.CppType.get_includes_by_cpp_type('geometry::Distance') == [ 'userver/chaotic/io/geometry/distance.hpp', ] - assert types.CppType.get_include_by_cpp_type('std::vector') == [ + assert types.CppType.get_includes_by_cpp_type('std::vector') == [ 'userver/chaotic/io/std/vector.hpp', ] - assert types.CppType.get_include_by_cpp_type( + assert types.CppType.get_includes_by_cpp_type( 'userver::utils::StrongTypedef', - ) == ['userver/chaotic/io/xxx.hpp'] + ) == ['userver/utils/strong_typedef.hpp', 'userver/chaotic/io/xxx.hpp'] diff --git a/chaotic/tests/compilers/test_dynamic_configs.py b/chaotic/tests/compilers/test_dynamic_configs.py index b22cb91d6dc2..249a7d3bf996 100644 --- a/chaotic/tests/compilers/test_dynamic_configs.py +++ b/chaotic/tests/compilers/test_dynamic_configs.py @@ -3,7 +3,6 @@ from typing import Any from chaotic import error -from chaotic.back.cpp import type_name from chaotic.back.cpp import types from chaotic.compilers import dynamic_config @@ -21,17 +20,14 @@ def parse_variable_content( return compiler.extract_variable_type() -def test_smoke(): +def test_smoke(cpp_primitive_type): var = parse_variable_content({'schema': {'type': 'integer'}, 'default': 1}) - expected = types.CppPrimitiveType( - raw_cpp_type=type_name.TypeName('int'), - nullable=False, - user_cpp_type=None, - json_schema=None, + expected = cpp_primitive_type( validators=types.CppPrimitiveValidator( namespace='::taxi_config::var', prefix='VariableTypeRaw', ), + raw_cpp_type_str='int', ) assert var.without_json_schema() == expected @@ -89,7 +85,10 @@ def test_strong_typedef_dependencies(): }) assert False except error.BaseError as exc: - assert 'Include file "userver/chaotic/io/xxx.hpp" not found' in exc.msg + assert ( + 'Include file "userver/utils/strong_typedef.hpp" not found' in exc.msg + or 'Include file "userver/chaotic/io/xxx.hpp" not found' in exc.msg + ) def test_default_isomorphic(): diff --git a/chaotic/tests/conftest.py b/chaotic/tests/conftest.py index 7544f7e704b8..5e69870315b6 100644 --- a/chaotic/tests/conftest.py +++ b/chaotic/tests/conftest.py @@ -1,6 +1,23 @@ +from typing import Any +from typing import Optional + import pytest +from chaotic.back.cpp import type_name +from chaotic.back.cpp.types import CppPrimitiveType +from chaotic.back.cpp.types import CppPrimitiveValidator from chaotic.front import parser +from chaotic.front.types import Schema + + +@pytest.fixture +def schema_parser(): + config = parser.ParserConfig(erase_prefix='') + return parser.SchemaParser( + config=config, + full_filepath='full', + full_vfilepath='vfull', + ) @pytest.fixture @@ -16,3 +33,31 @@ def func(input_: dict): return schema_parser.parsed_schemas() return func + + +@pytest.fixture +def cpp_primitive_type(): + """Factory fixture for creating CppPrimitiveType instances with common defaults.""" + + def create( + validators: CppPrimitiveValidator, + raw_cpp_type_str: str, + user_cpp_type: Optional[str] = None, + json_schema: Optional[Schema] = None, + nullable: bool = False, + default: Any = None, + ): + kwargs = { + 'raw_cpp_type': type_name.TypeName(raw_cpp_type_str), + 'user_cpp_type': user_cpp_type, + 'json_schema': json_schema, + 'nullable': nullable, + 'validators': validators, + } + + if default is not None: + kwargs['default'] = default + + return CppPrimitiveType(**kwargs) + + return create diff --git a/chaotic/tests/front/test_all_of.py b/chaotic/tests/front/test_all_of.py index e8feefddaee3..666a12da842c 100644 --- a/chaotic/tests/front/test_all_of.py +++ b/chaotic/tests/front/test_all_of.py @@ -1,15 +1,15 @@ +import pytest + from chaotic.front.parser import ParserError from chaotic.front.types import AllOf from chaotic.front.types import SchemaObject def test_of_none(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'allOf': []}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/allOf' - assert exc.msg == 'Empty allOf' + assert exc.value.infile_path == '/definitions/type/allOf' + assert exc.value.msg == 'Empty allOf' # stupid, but valid @@ -52,9 +52,7 @@ def test_2_empty(simple_parse): def test_non_object(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'allOf': [{'type': 'integer'}]}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/allOf/0' - assert exc.msg == 'Non-object type in allOf: integer' + assert exc.value.infile_path == '/definitions/type/allOf/0' + assert exc.value.msg == 'Non-object type in allOf: integer' diff --git a/chaotic/tests/front/test_array.py b/chaotic/tests/front/test_array.py index 3904f405e36c..693426f8e8ac 100644 --- a/chaotic/tests/front/test_array.py +++ b/chaotic/tests/front/test_array.py @@ -1,13 +1,13 @@ +import pytest + from chaotic.front.parser import ParserError def test_array_missing_items(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'array'}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/items' - assert exc.msg == '"items" is missing' + assert exc.value.infile_path == '/definitions/type/items' + assert exc.value.msg == '"items" is missing' def test_int_array(simple_parse): diff --git a/chaotic/tests/front/test_basic.py b/chaotic/tests/front/test_basic.py index 5b835b085d06..bb6c859e617d 100644 --- a/chaotic/tests/front/test_basic.py +++ b/chaotic/tests/front/test_basic.py @@ -1,45 +1,34 @@ +import pytest + from chaotic.front import ref_resolver -from chaotic.front.parser import ParserConfig from chaotic.front.parser import ParserError -from chaotic.front.parser import SchemaParser def test_generic_error(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'integer', 'unknown_field': '1'}) - assert False - except ParserError as exc: - assert exc.full_filepath == 'full' - assert exc.schema_type == 'jsonschema' + assert exc.value.full_filepath == 'full' + assert exc.value.schema_type == 'jsonschema' def test_unknown_field(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'integer', 'unknown_field': '1'}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/unknown_field' - assert exc.msg == ( - 'Unknown field: "unknown_field", known fields: ' - '["default", "enum", "exclusiveMaximum", "exclusiveMinimum", ' - '"format", "maximum", "minimum", "nullable", "type"]' - ) + assert exc.value.infile_path == '/definitions/type/unknown_field' + assert exc.value.msg == ( + 'Unknown field: "unknown_field", known fields: ' + '["default", "deprecated", "enum", "exclusiveMaximum", "exclusiveMinimum", ' + '"format", "maximum", "minimum", "nullable", "type"]' + ) -def test_duplicate_path(simple_parse): - try: - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_duplicate_path(schema_parser): + with pytest.raises(ParserError) as exc: + parser = schema_parser parser.parse_schema('/definitions/type', {'type': 'integer'}) parser.parse_schema('/definitions/type', {'type': 'number'}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type' - assert exc.msg == 'Duplicate path: vfull#/definitions/type' + assert exc.value.infile_path == '/definitions/type' + assert exc.value.msg == 'Duplicate path: vfull#/definitions/type' def test_x_unknown_field(simple_parse): @@ -51,41 +40,37 @@ def test_x_known_field(simple_parse): def test_wrong_field_type(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'intxxxx'}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/type' - assert exc.msg == 'Unknown type "intxxxx"' + assert exc.value.infile_path == '/definitions/type/type' + assert exc.value.msg == 'Unknown type "intxxxx"' def test_no_schema_type(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'enum': [1, 2, 3]}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type' - assert exc.msg == '"type" is missing' + assert exc.value.infile_path == '/definitions/type' + assert exc.value.msg == '"type" is missing' def test_schema_type_string(simple_parse): simple_parse({'type': 'string'}) -def test_ref(simple_parse): - try: - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_title(simple_parse): + simple_parse({'type': 'string', 'title': 'something'}) + + +def test_ref(schema_parser): + with pytest.raises(Exception) as exc_info: + parser = schema_parser parser.parse_schema( '/definitions/type1', {'$ref': '#/definitions/type'}, ) rr = ref_resolver.RefResolver() rr.sort_schemas(parser.parsed_schemas()) - assert False - except Exception as exc: # pylint: disable=broad-exception-caught - assert str(exc) == ('$ref to unknown type "vfull#/definitions/type", known refs:\n- vfull#/definitions/type1') + + assert str(exc_info.value) == ( + '$ref to unknown type "vfull#/definitions/type", known refs:\n- vfull#/definitions/type1' + ) diff --git a/chaotic/tests/front/test_number.py b/chaotic/tests/front/test_number.py index f1ed67d45738..a47e2d73b0a3 100644 --- a/chaotic/tests/front/test_number.py +++ b/chaotic/tests/front/test_number.py @@ -45,12 +45,10 @@ def test_number_minmax_exclusive(simple_parse): def test_number_extra_enum(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'number', 'enum': [1.0]}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/enum' - assert 'Unknown field: "enum"' in exc.msg + assert exc.value.infile_path == '/definitions/type/enum' + assert 'Unknown field: "enum"' in exc.value.msg def test_integer_min_max(simple_parse): @@ -75,12 +73,10 @@ def test_integer_minmax_exclusive(simple_parse): def test_integer_min_max_number(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'integer', 'minimum': 1.1}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/minimum' - assert exc.msg == 'field "minimum" has wrong type' + assert exc.value.infile_path == '/definitions/type/minimum' + assert exc.value.msg == 'field "minimum" has wrong type' def test_integer_enum(simple_parse): @@ -91,18 +87,14 @@ def test_integer_enum(simple_parse): def test_integer_enum_wrong_type(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'integer', 'enum': ['1']}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/enum' - assert exc.msg == 'field "enum" contains non-integers (1)' + assert exc.value.infile_path == '/definitions/type/enum' + assert exc.value.msg == 'field "enum" contains non-integers (1)' def test_integer_min_wrong_str(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'type': 'integer', 'minimum': '1'}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/minimum' - assert exc.msg == 'field "minimum" has wrong type' + assert exc.value.infile_path == '/definitions/type/minimum' + assert exc.value.msg == 'field "minimum" has wrong type' diff --git a/chaotic/tests/front/test_object.py b/chaotic/tests/front/test_object.py index 615980e1b83b..9777818d76d8 100644 --- a/chaotic/tests/front/test_object.py +++ b/chaotic/tests/front/test_object.py @@ -1,3 +1,5 @@ +import pytest + from chaotic.front.parser import ParserError from chaotic.front.types import Boolean from chaotic.front.types import Integer @@ -17,57 +19,49 @@ def test_very_empty(simple_parse): def test_unknown_required(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({ 'type': 'object', 'properties': {}, 'additionalProperties': False, 'required': ['unknown'], }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/required' - assert exc.msg == ('Field "unknown" is set in "required", but missing in "properties"') + assert exc.value.infile_path == '/definitions/type/required' + assert exc.value.msg == ('Field "unknown" is set in "required", but missing in "properties"') def test_unknown_fields(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({ 'type': 'object', 'unknown_field': 'x', 'properties': {}, 'additionalProperties': False, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/unknown_field' - assert 'Unknown field: "unknown_field"' in exc.msg + assert exc.value.infile_path == '/definitions/type/unknown_field' + assert 'Unknown field: "unknown_field"' in exc.value.msg def test_error_in_property(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({ 'type': 'object', 'properties': {'field': {'type': 'xxxx'}}, 'additionalProperties': False, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/properties/field/type' - assert exc.msg == 'Unknown type "xxxx"' + assert exc.value.infile_path == '/definitions/type/properties/field/type' + assert exc.value.msg == 'Unknown type "xxxx"' def test_error_in_extra(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({ 'type': 'object', 'properties': {}, 'additionalProperties': {'type': 'xxx'}, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/additionalProperties/type' - assert exc.msg == 'Unknown type "xxx"' + assert exc.value.infile_path == '/definitions/type/additionalProperties/type' + assert exc.value.msg == 'Unknown type "xxx"' def test_property_and_additional(simple_parse): diff --git a/chaotic/tests/front/test_one_of.py b/chaotic/tests/front/test_one_of.py index f803a44b594c..fbcf6fc67fc2 100644 --- a/chaotic/tests/front/test_one_of.py +++ b/chaotic/tests/front/test_one_of.py @@ -1,8 +1,6 @@ import pytest -from chaotic.front.parser import ParserConfig from chaotic.front.parser import ParserError -from chaotic.front.parser import SchemaParser from chaotic.front.types import Boolean from chaotic.front.types import DiscMapping from chaotic.front.types import Integer @@ -15,14 +13,9 @@ @pytest.fixture(name='parse_after_refs') -def _parse_after_refs(): +def _parse_after_refs(schema_parser): def func(input_: dict): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) + parser = schema_parser parser.parse_schema( '/definitions/type1', { @@ -108,12 +101,10 @@ def func(input_: dict): def test_of_none(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'oneOf': []}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/oneOf' - assert exc.msg == 'Empty oneOf' + assert exc.value.infile_path == '/definitions/type/oneOf' + assert exc.value.msg == 'Empty oneOf' # stupid, but valid @@ -145,8 +136,20 @@ def test_wo_discriminator_2(simple_parse): } +def test_wo_discriminator_nullable(simple_parse): + parsed = simple_parse({'oneOf': [{'type': 'integer'}], 'nullable': True}) + assert parsed.schemas['vfull#/definitions/type'].nullable + + +def test_wo_discriminator_nullable_wrong_type(simple_parse): + with pytest.raises(ParserError) as exc: + simple_parse({'oneOf': [{'type': 'integer'}], 'nullable': 1}) + assert exc.value.infile_path == '/definitions/type/oneOf/nullable' + assert exc.value.msg == 'field "nullable" has wrong type' + + def test_wd_no_ref_or_object(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({ 'oneOf': [ {'type': 'integer'}, @@ -158,14 +161,12 @@ def test_wd_no_ref_or_object(simple_parse): ], 'discriminator': {'propertyName': 'foo'}, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/oneOf/0' - assert exc.msg == 'Not a $ref in oneOf with discriminator' + assert exc.value.infile_path == '/definitions/type/oneOf/0' + assert exc.value.msg == 'Not a $ref in oneOf with discriminator' def test_wd_wrong_property(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({ 'oneOf': [ { @@ -176,14 +177,12 @@ def test_wd_wrong_property(simple_parse): ], 'discriminator': {'propertyName': 'foo'}, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/oneOf/0' - assert exc.msg == 'Not a $ref in oneOf with discriminator' + assert exc.value.infile_path == '/definitions/type/oneOf/0' + assert exc.value.msg == 'Not a $ref in oneOf with discriminator' def test_wd_wrong_property2(parse_after_refs): - try: + with pytest.raises(ParserError) as exc: parse_after_refs({ 'oneOf': [ {'$ref': '#/definitions/type1'}, @@ -192,22 +191,18 @@ def test_wd_wrong_property2(parse_after_refs): ], 'discriminator': {'propertyName': 'foo'}, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/oneOf/2' - assert exc.msg == 'No discriminator property "foo"' + assert exc.value.infile_path == '/definitions/type/oneOf/2' + assert exc.value.msg == 'No discriminator property "foo"' def test_wd_wrong_type(parse_after_refs): - try: + with pytest.raises(ParserError) as exc: parse_after_refs({ 'oneOf': [{'$ref': '#/definitions/type_int'}], 'discriminator': {'propertyName': 'foo'}, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/oneOf/0' - assert exc.msg == 'oneOf $ref to non-object' + assert exc.value.infile_path == '/definitions/type/oneOf/0' + assert exc.value.msg == 'oneOf $ref to non-object (Integer)' def test_wd_ok(parse_after_refs): @@ -311,7 +306,7 @@ def test_wd_ok_with_int_mapping(parse_after_refs): def test_wd_non_uniform_mapping(parse_after_refs): - try: + with pytest.raises(ParserError) as exc: parse_after_refs({ 'oneOf': [ {'$ref': '#/definitions/type3'}, @@ -325,13 +320,12 @@ def test_wd_non_uniform_mapping(parse_after_refs): }, }, }) - except ParserError as exc: - assert exc.infile_path == '/definitions/type/discriminator/mapping' - assert exc.msg.startswith('Not uniform mapping') + assert exc.value.infile_path == '/definitions/type/discriminator/mapping' + assert exc.value.msg.startswith('Not uniform mapping') def test_wd_ok_with_mapping_missing_ref(parse_after_refs): - try: + with pytest.raises(ParserError) as exc: parse_after_refs({ 'oneOf': [ {'$ref': '#/definitions/type1'}, @@ -342,14 +336,12 @@ def test_wd_ok_with_mapping_missing_ref(parse_after_refs): 'mapping': {'t2': '#/definitions/type2'}, }, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/discriminator/mapping' - assert exc.msg == 'Missing $ref in mapping: vfull#/definitions/type1' + assert exc.value.infile_path == '/definitions/type/discriminator/mapping' + assert exc.value.msg == 'Missing $ref in mapping: vfull#/definitions/type1' def test_wd_ok_with_mapping_invalid_ref(parse_after_refs): - try: + with pytest.raises(ParserError) as exc: parse_after_refs({ 'oneOf': [ {'$ref': '#/definitions/type1'}, @@ -364,14 +356,12 @@ def test_wd_ok_with_mapping_invalid_ref(parse_after_refs): }, }, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/discriminator/mapping' - assert exc.msg == ("$ref(s) outside of oneOf: ['vfull#/definitions/wrong']") + assert exc.value.infile_path == '/definitions/type/discriminator/mapping' + assert exc.value.msg == ("$ref(s) outside of oneOf: ['vfull#/definitions/wrong']") def test_wd_invalidtype_mapping_value(parse_after_refs): - try: + with pytest.raises(ParserError) as exc: parse_after_refs({ 'oneOf': [ {'$ref': '#/definitions/type1'}, @@ -382,14 +372,12 @@ def test_wd_invalidtype_mapping_value(parse_after_refs): 'mapping': {'t1': 1, 't2': '#/definitions/type2'}, }, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/discriminator/mapping/t1' - assert exc.msg == 'Not a string in mapping' + assert exc.value.infile_path == '/definitions/type/discriminator/mapping/t1' + assert exc.value.msg == 'Not a string in mapping' def test_wd_extra_field(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({ 'oneOf': [ { @@ -400,7 +388,17 @@ def test_wd_extra_field(simple_parse): ], 'discriminator': {'foo': 1, 'propertyName': 'foo'}, }) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type/discriminator/foo' - assert exc.msg == ('Unknown field: "foo", known fields: ["mapping", "propertyName"]') + assert exc.value.infile_path == '/definitions/type/discriminator/foo' + assert exc.value.msg == ('Unknown field: "foo", known fields: ["mapping", "propertyName"]') + + +def test_wd_nullable(parse_after_refs): + schema = parse_after_refs({ + 'oneOf': [ + {'$ref': '#/definitions/type1'}, + {'$ref': '#/definitions/type2'}, + ], + 'discriminator': {'propertyName': 'foo'}, + 'nullable': True, + }) + assert schema['vfull#/definitions/type'].nullable diff --git a/chaotic/tests/front/test_ref.py b/chaotic/tests/front/test_ref.py index 1ab19cadc95e..5a763b77a2af 100644 --- a/chaotic/tests/front/test_ref.py +++ b/chaotic/tests/front/test_ref.py @@ -1,5 +1,7 @@ import collections +import pytest + from chaotic.front import ref_resolver from chaotic.front import types from chaotic.front.parser import ParserConfig @@ -10,13 +12,8 @@ from chaotic.front.types import Ref -def test_ref_ok(): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_ref_ok(schema_parser): + parser = schema_parser parser.parse_schema('/definitions/type1', {'type': 'integer'}) parser.parse_schema('/definitions/type2', {'$ref': '#/definitions/type1'}) @@ -32,13 +29,8 @@ def test_ref_ok(): } -def test_ref_from_items_ok(): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_ref_from_items_ok(schema_parser): + parser = schema_parser parser.parse_schema('/definitions/type1', {'type': 'integer'}) parser.parse_schema( @@ -59,48 +51,37 @@ def test_ref_from_items_ok(): } -def test_ref_invalid(): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', +def test_ref_invalid(schema_parser): + parser = schema_parser + + parser.parse_schema('/definitions/type1', {'type': 'integer'}) + parser.parse_schema( + '/definitions/type2', + {'$ref': '#/definitions/other_type'}, ) + rr = ref_resolver.RefResolver() - try: - parser.parse_schema('/definitions/type1', {'type': 'integer'}) - parser.parse_schema( - '/definitions/type2', - {'$ref': '#/definitions/other_type'}, - ) - rr = ref_resolver.RefResolver() + with pytest.raises(Exception) as exc: rr.sort_schemas(parser.parsed_schemas()) - assert False - except Exception as exc: # pylint: disable=broad-exception-caught - assert str(exc) == ( - '$ref to unknown type "vfull#/definitions/other_type", ' - 'known refs:\n- vfull#/definitions/type1\n' - '- vfull#/definitions/type2' - ) + + assert str(exc.value) == ( + '$ref to unknown type "vfull#/definitions/other_type", ' + 'known refs:\n- vfull#/definitions/type1\n' + '- vfull#/definitions/type2' + ) def test_extra_fields(simple_parse): - try: + with pytest.raises(ParserError) as exc: simple_parse({'$ref': '123', 'field': 1}) - assert False - except ParserError as exc: - assert exc.infile_path == '/definitions/type' - assert exc.msg == "Unknown field(s) ['field']" + assert exc.value.infile_path == '/definitions/type' + assert exc.value.msg == "Unknown field(s) ['field']" -def test_sibling_file(): +def test_sibling_file(schema_parser): config = ParserConfig(erase_prefix='') schemas = [] - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) + parser = schema_parser parser.parse_schema('/definitions/type1', {'type': 'integer'}) schemas.append(parser.parsed_schemas()) @@ -137,13 +118,8 @@ def test_sibling_file(): } -def test_forward_reference(): - config = ParserConfig(erase_prefix='') - parser = SchemaParser( - config=config, - full_filepath='full', - full_vfilepath='vfull', - ) +def test_forward_reference(schema_parser): + parser = schema_parser parser.parse_schema('/definitions/type1', {'$ref': '#/definitions/type2'}) parser.parse_schema('/definitions/type2', {'type': 'integer'}) parser.parse_schema('/definitions/type3', {'$ref': '#/definitions/type4'}) @@ -184,20 +160,38 @@ def test_forward_reference(): }) -def test_cycle(): +def test_cycle(schema_parser): + parser = schema_parser + parser.parse_schema('/definitions/type1', {'$ref': '#/definitions/type2'}) + parser.parse_schema('/definitions/type2', {'$ref': '#/definitions/type1'}) + + rr = ref_resolver.RefResolver() + + with pytest.raises(ref_resolver.ResolverError) as exc: + rr.sort_schemas(parser.parsed_schemas()) + + assert str(exc.value) == '$ref cycle: vfull#/definitions/type1, vfull#/definitions/type2' + + +def test_self_ref(schema_parser): + parser = schema_parser + parser.parse_schema('/definitions/type1', {'$ref': '#/definitions/type1'}) + + rr = ref_resolver.RefResolver() + + with pytest.raises(ref_resolver.ResolverError) as exc: + rr.sort_schemas(parser.parsed_schemas()) + + assert str(exc.value) == '$ref cycle: vfull#/definitions/type1' + + +def test_no_fragment(): config = ParserConfig(erase_prefix='') parser = SchemaParser( config=config, full_filepath='full', full_vfilepath='vfull', ) - parser.parse_schema('/definitions/type1', {'$ref': '#/definitions/type2'}) - parser.parse_schema('/definitions/type2', {'$ref': '#/definitions/type1'}) - - rr = ref_resolver.RefResolver() - try: - rr.sort_schemas(parser.parsed_schemas()) - except ref_resolver.ResolverError as exc: - assert str(exc) == '$ref cycle: vfull#/definitions/type1, vfull#/definitions/type2' - else: - assert False + with pytest.raises(ParserError) as exc_info: + parser.parse_schema('/definitions/type1', {'$ref': '/definitions/type2'}) + assert exc_info.value.msg == 'Error in $ref (/definitions/type2): there should be exactly one "#" inside' diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index ac2096b7fb45..0b609b571561 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -16,6 +16,7 @@ userver_module( # HACK: common source between unittest and dbtest targets. "${CMAKE_CURRENT_SOURCE_DIR}/src/storages/tests/utils_test.cpp" DBTEST_DATABASES clickhouse + DEPENDS core ) target_compile_options(${PROJECT_NAME} PUBLIC "-Wno-error=pedantic") diff --git a/cmake/ChaoticGen.cmake b/cmake/ChaoticGen.cmake index 1f7b2088efe9..3119cc15fce4 100644 --- a/cmake/ChaoticGen.cmake +++ b/cmake/ChaoticGen.cmake @@ -131,7 +131,6 @@ function(userver_target_generate_chaotic TARGET) list(APPEND CHAOTIC_ARGS "-o" "${PARSE_OUTPUT_DIR}/${PARSE_OUTPUT_PREFIX}") list(APPEND CHAOTIC_ARGS "--relative-to" "${PARSE_RELATIVE_TO}") - list(APPEND CHAOTIC_ARGS "--clang-format" "${CLANG_FORMAT}") _userver_initialize_codegen_flag() @@ -150,6 +149,7 @@ function(userver_target_generate_chaotic TARGET) OUTPUT ${SCHEMAS} COMMAND ${CMAKE_COMMAND} -E env "USERVER_PYTHON=${USERVER_CHAOTIC_PYTHON_BINARY}" "${CHAOTIC_BIN}" ${CHAOTIC_EXTRA_ARGS} ${CHAOTIC_ARGS} ${PARSE_SCHEMAS} + --clang-format "${CLANG_FORMAT}" DEPENDS ${PARSE_SCHEMAS} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" VERBATIM ${CODEGEN} @@ -198,23 +198,23 @@ function(userver_target_generate_openapi_client TARGET) endif() set(SCHEMAS - "${PARSE_OUTPUT_DIR}/include/client/${PARSE_NAME}/client.hpp" - "${PARSE_OUTPUT_DIR}/include/client/${PARSE_NAME}/client_impl.hpp" - "${PARSE_OUTPUT_DIR}/include/client/${PARSE_NAME}/component.hpp" - "${PARSE_OUTPUT_DIR}/include/client/${PARSE_NAME}/requests.hpp" - "${PARSE_OUTPUT_DIR}/include/client/${PARSE_NAME}/responses.hpp" - "${PARSE_OUTPUT_DIR}/include/client/${PARSE_NAME}/exceptions.hpp" - "${PARSE_OUTPUT_DIR}/src/client/${PARSE_NAME}/client.cpp" - "${PARSE_OUTPUT_DIR}/src/client/${PARSE_NAME}/client_impl.cpp" - "${PARSE_OUTPUT_DIR}/src/client/${PARSE_NAME}/component.cpp" - "${PARSE_OUTPUT_DIR}/src/client/${PARSE_NAME}/requests.cpp" - "${PARSE_OUTPUT_DIR}/src/client/${PARSE_NAME}/responses.cpp" - "${PARSE_OUTPUT_DIR}/src/client/${PARSE_NAME}/exceptions.cpp" + "${PARSE_OUTPUT_DIR}/include/clients/${PARSE_NAME}/client.hpp" + "${PARSE_OUTPUT_DIR}/include/clients/${PARSE_NAME}/client_impl.hpp" + "${PARSE_OUTPUT_DIR}/include/clients/${PARSE_NAME}/component.hpp" + "${PARSE_OUTPUT_DIR}/include/clients/${PARSE_NAME}/requests.hpp" + "${PARSE_OUTPUT_DIR}/include/clients/${PARSE_NAME}/responses.hpp" + "${PARSE_OUTPUT_DIR}/include/clients/${PARSE_NAME}/exceptions.hpp" + "${PARSE_OUTPUT_DIR}/src/clients/${PARSE_NAME}/client.cpp" + "${PARSE_OUTPUT_DIR}/src/clients/${PARSE_NAME}/client_impl.cpp" + "${PARSE_OUTPUT_DIR}/src/clients/${PARSE_NAME}/component.cpp" + "${PARSE_OUTPUT_DIR}/src/clients/${PARSE_NAME}/requests.cpp" + "${PARSE_OUTPUT_DIR}/src/clients/${PARSE_NAME}/responses.cpp" + "${PARSE_OUTPUT_DIR}/src/clients/${PARSE_NAME}/exceptions.cpp" ) foreach(SCHEMA ${PARSE_SCHEMAS}) string(REGEX REPLACE "^.*/([^/]*)\\.([^.]*)\$" "\\1" SCHEMA "${SCHEMA}") - set(SCHEMAS ${SCHEMAS} "${PARSE_OUTPUT_DIR}/include/client/${PARSE_NAME}/${SCHEMA}.hpp" - "${PARSE_OUTPUT_DIR}/src/client/${PARSE_NAME}/${SCHEMA}.cpp" + set(SCHEMAS ${SCHEMAS} "${PARSE_OUTPUT_DIR}/include/clients/${PARSE_NAME}/${SCHEMA}.hpp" + "${PARSE_OUTPUT_DIR}/src/clients/${PARSE_NAME}/${SCHEMA}.cpp" ) endforeach() diff --git a/cmake/DownloadUsingCPM.cmake b/cmake/DownloadUsingCPM.cmake index a754d0ac0be1..4f16f389076d 100644 --- a/cmake/DownloadUsingCPM.cmake +++ b/cmake/DownloadUsingCPM.cmake @@ -73,3 +73,10 @@ function(mark_targets_as_system directory) endforeach() endforeach() endfunction() + +function(_userver_print_cpm_packages) + message(STATUS "Dependencies from CPM:") + foreach(PACKAGE ${CPM_PACKAGES}) + message(STATUS "- ${PACKAGE}") + endforeach() +endfunction() diff --git a/cmake/GetUserverVersion.cmake b/cmake/GetUserverVersion.cmake index 2826b3355477..f4e3d5edf733 100644 --- a/cmake/GetUserverVersion.cmake +++ b/cmake/GetUserverVersion.cmake @@ -21,8 +21,9 @@ else() message(STATUS "Git not found") endif() -set(USERVER_MAJOR_VERSION 2) -set(USERVER_MINOR_VERSION 12-rc) +file(READ ${USERVER_ROOT_DIR}/version.txt VERSION) +string(REGEX MATCH ^[0-9]+ USERVER_MAJOR_VERSION "${VERSION}") +string(REGEX MATCH [-0-9a-z]+$ USERVER_MINOR_VERSION "${VERSION}") set(USERVER_VERSION "${USERVER_MAJOR_VERSION}.${USERVER_MINOR_VERSION}") string(REPLACE "-" "_" USERVER_VERSION_STR "${USERVER_VERSION}") diff --git a/cmake/ModuleHelpers.cmake b/cmake/ModuleHelpers.cmake index 740097b8e8c8..e702d1e42aca 100644 --- a/cmake/ModuleHelpers.cmake +++ b/cmake/ModuleHelpers.cmake @@ -3,20 +3,46 @@ include_guard(GLOBAL) cmake_policy(SET CMP0054 NEW) macro(_userver_module_begin) - set(options) + set(options + CPM_DOWNLOAD_ONLY + ) set(oneValueArgs # Target name, also used for package name by default NAME VERSION ) set(multiValueArgs DEBIAN_NAMES FORMULA_NAMES RPM_NAMES PACMAN_NAMES PKG_NAMES # For version detection of manually installed packages and unknown package managers. - PKG_CONFIG_NAMES + PKG_CONFIG_NAMES + # For CPM options + CPM_NAME + CPM_VERSION + CPM_GITHUB_REPOSITORY + CPM_URL + CPM_OPTIONS + CPM_SOURCE_SUBDIR + CPM_GIT_TAG ) cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") set(name "${ARG_NAME}") + string(TOUPPER "${ARG_CPM_NAME}" ARG_CPM_NAME) + string(REPLACE "-" "_" ARG_CPM_NAME "${ARG_CPM_NAME}") + + if(ARG_CPM_NAME) + option( + USERVER_DOWNLOAD_PACKAGE_${ARG_CPM_NAME} + "Download and setup ${ARG_CPM_NAME} if no library of matching version was found" + ${USERVER_DOWNLOAD_PACKAGES} + ) + option( + USERVER_FORCE_DOWNLOAD_${ARG_CPM_NAME} + "Download ${ARG_CPM_NAME} even if there is an installed system package" + ${USERVER_FORCE_DOWNLOAD_PACKAGES} + ) + endif() + if(ARG_VERSION) if(NOT ${name}_FIND_VERSION OR "${${name}_FIND_VERSION}" VERSION_LESS "${ARG_VERSION}") set("${name}_FIND_VERSION" "${ARG_VERSION}") @@ -78,6 +104,12 @@ endmacro() macro(_userver_module_find_part) # Also uses ARGs left over from _userver_find_module_begin + # TODO: return() doesn't work inside of macro + # if(USERVER_FORCE_DOWNLOAD_${ARG_CPM_NAME}) + # message(STATUS "Skipping ${ARG_CPM_NAME} system package search due to USERVER_FORCE_DOWNLOAD_${ARG_CPM_NAME}=TRUE") + # return() + # endif() + set(options) set(oneValueArgs PART_TYPE) set(multiValueArgs NAMES PATHS PATH_SUFFIXES) @@ -261,12 +293,25 @@ macro(_userver_module_end) list(APPEND required_vars "${programs_variable}") endif() if(required_vars) - find_package_handle_standard_args( - "${current_package_name}" - REQUIRED_VARS ${required_vars} - FAIL_MESSAGE "${FULL_ERROR_MESSAGE}" - ) - mark_as_advanced(${required_vars}) + foreach(_CURRENT_VAR ${required_vars}) + if(NOT ${_CURRENT_VAR}) + set(NEED_CPM TRUE) + if(USERVER_DOWNLOAD_PACKAGE_${ARG_CPM_NAME}) + set(${_CURRENT_VAR}) + endif() + endif() + endforeach() + + if(NEED_CPM AND USERVER_DOWNLOAD_PACKAGE_${ARG_CPM_NAME}) + _userver_cpm_addpackage("${current_package_name}") + else() + find_package_handle_standard_args( + "${current_package_name}" + REQUIRED_VARS ${required_vars} + FAIL_MESSAGE "${FULL_ERROR_MESSAGE}" + ) + mark_as_advanced(${required_vars}) + endif() else() # Forward to another CMake module, add nice error messages if missing. set(wrapped_package_name "${current_package_name}") @@ -323,6 +368,29 @@ macro(_userver_module_end) endif() endmacro() +macro(_userver_cpm_addpackage name) + include(DownloadUsingCPM) + + set(EXTRA_ARGS) + if(ARG_CPM_DOWNLOAD_ONLY) + set(EXTRA_ARGS ${EXTRA_ARGS} DOWNLOAD_ONLY) + endif() + cpmaddpackage( + NAME ${name} + VERSION ${ARG_CPM_VERSION} + GITHUB_REPOSITORY ${ARG_CPM_GITHUB_REPOSITORY} + URL ${ARG_CPM_URL} + OPTIONS ${ARG_CPM_OPTIONS} + SOURCE_SUBDIR ${ARG_CPM_SOURCE_SUBDIR} + GIT_TAG ${ARG_CPM_GIT_TAG} + ${EXTRA_ARGS} + ) + if(NOT ARG_CPM_DOWNLOAD_ONLY) + mark_targets_as_system("${${name}_SOURCE_DIR}") + endif() + set(${name}_FOUND 1) +endmacro() + function(_userver_macos_set_default_dir variable command_args) set(default_value "") if(CMAKE_SYSTEM_NAME MATCHES "Darwin" AND NOT DEFINED ${variable}) diff --git a/cmake/PrepareInstall.cmake b/cmake/PrepareInstall.cmake index 828e3dcd921d..3c5b3326de5c 100644 --- a/cmake/PrepareInstall.cmake +++ b/cmake/PrepareInstall.cmake @@ -60,6 +60,7 @@ function(_userver_export_targets) CONFIGURATIONS RELEASE NAMESPACE userver:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver/release + COMPONENT universal ) install( EXPORT userver-targets_d @@ -67,6 +68,7 @@ function(_userver_export_targets) CONFIGURATIONS DEBUG NAMESPACE userver:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver/debug + COMPONENT universal ) endfunction() @@ -74,6 +76,7 @@ function(_userver_directory_install) if(NOT USERVER_INSTALL) return() endif() + set(option) set(oneValueArgs COMPONENT DESTINATION PATTERN) set(multiValueArgs FILES DIRECTORY PROGRAMS) cmake_parse_arguments(ARG "${option}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") @@ -145,3 +148,64 @@ function(_userver_make_install_config) DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/userver" ) endfunction() + +function(_userver_install_component) + if(NOT USERVER_INSTALL) + return() + endif() + + set(oneValueArgs MODULE) + set(multiValueArgs DEPENDS) + cmake_parse_arguments(ARG "${option}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + string(TOUPPER "${ARG_MODULE}" MODULE_UPPER) + if(CPACK_COMPONENTS_GROUPING STREQUAL ONE_PER_GROUP) + if(NOT CPACK_DEBIAN_${MODULE_UPPER}_PACKAGE_DEPENDS) + message(FATAL_ERROR "File with per-component dependencies is missing (component ${ARG_MODULE}). Either use CPACK_COMPONENTS_GROUPING=ALL_COMPONENTS_IN_ONE to build a single all-in-one package, or create dependency file ${USERVER_ROOT_DIR}/scripts/docs/en/deps/${DEPENDENCIES_FILESTEM}/${ARG_MODULE}.") + endif() + endif() + + execute_process( + COMMAND cat "${USERVER_ROOT_DIR}/scripts/docs/en/deps/${DEPENDENCIES_FILESTEM}/${ARG_MODULE}" + COMMAND tr "\n" " " + COMMAND sed "s/ \\(.\\)/, \\1/g" + OUTPUT_VARIABLE MODULE_DEPENDS + ) + file(APPEND "${CMAKE_BINARY_DIR}/cpack.variables.inc" " + set(CPACK_DEBIAN_${MODULE_UPPER}_PACKAGE_NAME libuserver-${ARG_MODULE}-dev) + set(CPACK_DEBIAN_${MODULE_UPPER}_PACKAGE_CONFLICTS libuserver-all-dev) + set(CPACK_COMPONENT_${MODULE_UPPER}_DEPENDS ${ARG_DEPENDS}) + set(CPACK_DEBIAN_${MODULE_UPPER}_PACKAGE_DEPENDS \"${MODULE_DEPENDS}\") + ") + + file(APPEND "${CMAKE_BINARY_DIR}/cpack.inc" " + cpack_add_component_group(${ARG_MODULE} EXPANDED) + cpack_add_component(${ARG_MODULE} GROUP ${ARG_MODULE} INSTALL_TYPES Full) + ") +endfunction() + +function(_userver_prepare_components) + file(REMOVE "${CMAKE_BINARY_DIR}/cpack.inc") + file(REMOVE "${CMAKE_BINARY_DIR}/cpack.variables.inc") + + # DEB dependencies: + execute_process(COMMAND lsb_release -cs OUTPUT_VARIABLE OS_CODENAME) + if(OS_CODENAME MATCHES "^bookworm") + set(DEPENDENCIES_FILESTEM "debian-12") + elseif(OS_CODENAME MATCHES "^bullseye") + set(DEPENDENCIES_FILESTEM "debian-11") + elseif(OS_CODENAME MATCHES "^noble") + set(DEPENDENCIES_FILESTEM "ubuntu-24.04") + elseif(OS_CODENAME MATCHES "^jammy") + set(DEPENDENCIES_FILESTEM "ubuntu-22.04") + elseif(OS_CODENAME MATCHES "^impish") + set(DEPENDENCIES_FILESTEM "ubuntu-21.04") + elseif(OS_CODENAME MATCHES "^focal") + set(DEPENDENCIES_FILESTEM "ubuntu-20.04") + elseif(OS_CODENAME MATCHES "^bionic") + set(DEPENDENCIES_FILESTEM "ubuntu-18.04") + endif() + set(DEPENDENCIES_FILESTEM ${DEPENDENCIES_FILESTEM} CACHE INTERNAL "") +endfunction() + +_userver_prepare_components() diff --git a/cmake/SetupAbseil.cmake b/cmake/SetupAbseil.cmake index fb669c056f75..596beddee6dd 100644 --- a/cmake/SetupAbseil.cmake +++ b/cmake/SetupAbseil.cmake @@ -31,6 +31,9 @@ cpmaddpackage( GITHUB_REPOSITORY abseil/abseil-cpp SYSTEM + PATCHES + abseil_pr_1707.patch + abseil_pr_1739.patch OPTIONS "ABSL_PROPAGATE_CXX_STD ON" "ABSL_ENABLE_INSTALL ON" diff --git a/cmake/SetupBoost.cmake b/cmake/SetupBoost.cmake new file mode 100644 index 000000000000..b5af70ac0447 --- /dev/null +++ b/cmake/SetupBoost.cmake @@ -0,0 +1,73 @@ +include_guard(GLOBAL) + +option(USERVER_DOWNLOAD_PACKAGE_BOOST "Download and setup Boost if no library of matching version was found" + ${USERVER_DOWNLOAD_PACKAGES} +) +option(USERVER_FORCE_DOWNLOAD_BOOST "Download Boost even if there is an installed system package" + ${USERVER_FORCE_DOWNLOAD_PACKAGES} +) + +set(BOOST_VERSION 1.89.0) +set(BOOST_INCLUDE_LIBRARIES_FIND_PACKAGE + atomic + program_options + filesystem + regex + locale + iostreams + context + coroutine +) +set(BOOST_INCLUDE_LIBRARIES + ${BOOST_INCLUDE_LIBRARIES_FIND_PACKAGE} + coroutine2 + stacktrace + uuid + lockfree + endian + assert + predef +) +string(REGEX REPLACE ";" "\\\\\\\\;" BOOST_INCLUDE_LIBRARIES_LIST "${BOOST_INCLUDE_LIBRARIES}") + +if(NOT USERVER_FORCE_DOWNLOAD_BOOST AND NOT BOOST_CPM) + if(USERVER_DOWNLOAD_PACKAGE_BOOST) + set(MAYBE_REQUIRED) + else() + set(MAYBE_REQUIRED REQUIRED) + endif() + find_package( + Boost ${MAYBE_REQUIRED} CONFIG + COMPONENTS ${BOOST_INCLUDE_LIBRARIES_FIND_PACKAGE} stacktrace_basic + OPTIONAL_COMPONENTS stacktrace_backtrace stacktrace_windbg coroutine2 config assert + ) + + if(Boost_FOUND) + return() + endif() +endif() + +include(DownloadUsingCPM) +set(BOOST_CPM TRUE CACHE BOOL "") + +cpmaddpackage( + NAME Boost + VERSION ${BOOST_VERSION} + URL https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-cmake.tar.xz + URL_HASH SHA256=67acec02d0d118b5de9eb441f5fb707b3a1cdd884be00ca24b9a73c995511f74 + OPTIONS + "BOOST_ENABLE_CMAKE ON" + "BOOST_INCLUDE_LIBRARIES ${BOOST_INCLUDE_LIBRARIES_LIST}" + "BOOST_SKIP_INSTALL_RULES ON" + "BUILD_SHARED_LIBS OFF" + "BOOST_RUNTIME_LINK static" + "BUILD_TESTING OFF" + "BOOST_LOCKFREE_BUILD_TESTS OFF" + EXCLUDE_FROM_ALL +) + +# set version variable as find_package() does +set(Boost_VERSION_STRING ${BOOST_VERSION} CACHE STRING "") + +# We have fresh version of boost, DWCAS should work +set(USERVER_IMPL_DWCAS_CHECKED TRUE CACHE INTERNAL "TRUE iff checked that DWCAS works") diff --git a/cmake/SetupCURL.cmake b/cmake/SetupCURL.cmake index 0eb5f2b57fcf..c640c35c88d4 100644 --- a/cmake/SetupCURL.cmake +++ b/cmake/SetupCURL.cmake @@ -61,6 +61,8 @@ cpmaddpackage( "BUILD_CURL_EXE OFF" "BUILD_SHARED_LIBS OFF" "CURL_DISABLE_TESTS ON" + "CURL_DISABLE_LDAP ON" + "HAVE_DLOPEN TRUE" ${CURL_LTO_OPTION} ) diff --git a/cmake/SetupGTest.cmake b/cmake/SetupGTest.cmake index c92275400232..3988026f626a 100644 --- a/cmake/SetupGTest.cmake +++ b/cmake/SetupGTest.cmake @@ -5,8 +5,11 @@ endif() option(USERVER_DOWNLOAD_PACKAGE_GTEST "Download and setup gtest if no gtest of matching version was found" ${USERVER_DOWNLOAD_PACKAGES} ) +option(USERVER_FORCE_DOWNLOAD_GTEST "Download gtest even if there is an installed system package" + ${USERVER_FORCE_DOWNLOAD_PACKAGES} +) -if(NOT USERVER_FORCE_DOWNLOAD_PACKAGES) +if(NOT USERVER_FORCE_DOWNLOAD_GTEST) find_package(GTest QUIET) if(NOT GTest_FOUND AND NOT USERVER_DOWNLOAD_PACKAGE_GTEST) message( diff --git a/cmake/SetupGrpc.cmake b/cmake/SetupGrpc.cmake index ec0f0afea6d3..c826a13200f3 100644 --- a/cmake/SetupGrpc.cmake +++ b/cmake/SetupGrpc.cmake @@ -73,6 +73,7 @@ cpmaddpackage( GITHUB_REPOSITORY grpc/grpc SYSTEM + PATCHES grpc_pr_36805.patch OPTIONS "BUILD_SHARED_LIBS OFF" "CARES_BUILD_TOOLS OFF" diff --git a/cmake/SetupOpenssl.cmake b/cmake/SetupOpenssl.cmake new file mode 100644 index 000000000000..08b6b1b6c77f --- /dev/null +++ b/cmake/SetupOpenssl.cmake @@ -0,0 +1,72 @@ +include_guard(GLOBAL) + +option(USERVER_DOWNLOAD_PACKAGE_OPENSSL "Download and setup OpenSSL if no library of matching version was found" + ${USERVER_DOWNLOAD_PACKAGES} +) +option(USERVER_FORCE_DOWNLOAD_OPENSSL "Download OpenSSL even if there is an installed system package" + ${USERVER_FORCE_DOWNLOAD_PACKAGES} +) + +if(NOT USERVER_FORCE_DOWNLOAD_OPENSSL AND NOT OpenSSL_CPM) + if(USERVER_DOWNLOAD_PACKAGE_OPENSSL) + find_package(OpenSSL) + else() + find_package(OpenSSL REQUIRED) + endif() + + if(OpenSSL_FOUND) + return() + endif() +endif() +set(OpenSSL_CPM TRUE CACHE BOOL "") + +include(DownloadUsingCPM) + +set(OPENSSL_INSTALL_DIR ${CMAKE_BINARY_DIR}/openssl) +execute_process(COMMAND mkdir -p ${OPENSSL_INSTALL_DIR}/usr/local/include) + +# Flags are copied from Ubuntu's debian/rules +set(CONFIGURE_FLAGS no-idea no-mdc2 no-rc5 no-zlib no-ssl3 enable-unit-test no-ssl3-method enable-rfc3779 enable-cms no-capieng) +set(OPENSSL_VERSION 3.5.2) + +cpmaddpackage( + NAME OpenSSL + URL https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz + URL_HASH SHA512=db2c7a88bea432f96d867a98af15f850f371d4136c657338de93cb88a39a3578c025b5df7310e195a02fc715ad5a2422a319a44f0247c6a7e2ba8b36aad77651 +) + +# We use custom target to be able to set build dependency +# for *external* libcurl from CPM +add_custom_target( + OpenSSL + test -e ${OPENSSL_INSTALL_DIR}/.installed || CFLAGS=${CMAKE_C_FLAGS} ./config --libdir=/usr/local/lib ${CONFIGURE_FLAGS} + COMMAND + test -e ${OPENSSL_INSTALL_DIR}/.installed || make -j8 + COMMAND + test -e ${OPENSSL_INSTALL_DIR}/.installed || make DESTDIR=${OPENSSL_INSTALL_DIR} install_sw + COMMAND + touch ${OPENSSL_INSTALL_DIR}/.installed + WORKING_DIRECTORY ${OpenSSL_SOURCE_DIR} + COMMENT "Compiling OpenSSL library" +) + + +add_library(Crypto STATIC IMPORTED GLOBAL) +add_dependencies(Crypto OpenSSL) +set_property(TARGET Crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_INSTALL_DIR}/usr/local/lib/libcrypto.a) +target_include_directories(Crypto INTERFACE ${OPENSSL_INSTALL_DIR}/usr/local/include) + + +add_library(SSL STATIC IMPORTED GLOBAL) +add_dependencies(SSL OpenSSL) +set_property(TARGET SSL PROPERTY IMPORTED_LOCATION ${OPENSSL_INSTALL_DIR}/usr/local/lib/libssl.a) +target_include_directories(SSL INTERFACE ${OPENSSL_INSTALL_DIR}/usr/local/include) + +add_library(OpenSSL::Crypto ALIAS Crypto) +add_library(OpenSSL::SSL ALIAS SSL) + +# Light emulation of find_package(OpenSSL) for libcurl +set(OpenSSL_FOUND TRUE CACHE BOOL "" FORCE) +set(OPENSSL_FOUND TRUE CACHE BOOL "" FORCE) +set(OPENSSL_CRYPTO_LIBRARY OpenSSL::Crypto CACHE STRING "" FORCE) +set(OPENSSL_INCLUDE_DIR ${OPENSSL_INSTALL_DIR}/usr/local/include CACHE FILEPATH "" FORCE) diff --git a/cmake/SetupProtobuf.cmake b/cmake/SetupProtobuf.cmake index 2987e2d31b0a..85346f6d7d52 100644 --- a/cmake/SetupProtobuf.cmake +++ b/cmake/SetupProtobuf.cmake @@ -4,7 +4,12 @@ option(USERVER_FORCE_DOWNLOAD_PROTOBUF "Download Protobuf even if there is an in ) function(_userver_set_protobuf_version_category) - if(Protobuf_VERSION VERSION_GREATER_EQUAL 5.26.0 + if(Protobuf_VERSION VERSION_GREATER_EQUAL 6.30.0 + AND Protobuf_VERSION VERSION_LESS 7.0.0 + OR Protobuf_VERSION VERSION_GREATER_EQUAL 30.0.0 + ) + set_property(GLOBAL PROPERTY userver_protobuf_version_category 6) + elseif(Protobuf_VERSION VERSION_GREATER_EQUAL 5.26.0 AND Protobuf_VERSION VERSION_LESS 6.0.0 OR Protobuf_VERSION VERSION_GREATER_EQUAL 26.0.0 ) diff --git a/cmake/UserverGenerateDynamicConfigsDocs.cmake b/cmake/UserverGenerateDynamicConfigsDocs.cmake index c80a0303c308..3cbd3d3c4522 100644 --- a/cmake/UserverGenerateDynamicConfigsDocs.cmake +++ b/cmake/UserverGenerateDynamicConfigsDocs.cmake @@ -4,7 +4,8 @@ function(_userver_add_target_gen_dynamic_configs_docs) add_custom_target( userver-gen-dynamic-configs-docs COMMENT "Generate dynamic_configs .md docs" - COMMAND + COMMAND + ${USERVER_PYTHON_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/docs/dynamic_config_yaml_to_md.py -o ${CMAKE_CURRENT_BINARY_DIR}/docs-dynamic-configs ${YAML_FILENAMES} diff --git a/cmake/UserverModule.cmake b/cmake/UserverModule.cmake index 4b537c95da5e..86067c334f14 100644 --- a/cmake/UserverModule.cmake +++ b/cmake/UserverModule.cmake @@ -23,6 +23,7 @@ function(userver_module MODULE) UBENCH_LINK_LIBRARIES UBENCH_DATABASES UBENCH_ENV + DEPENDS ) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) if(ARG_UNPARSED_ARGUMENTS) @@ -89,6 +90,8 @@ function(userver_module MODULE) DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/userver" ) endif() + + _userver_install_component(MODULE ${MODULE} DEPENDS ${ARG_DEPENDS}) endif() # 1. userver-${MODULE}-unittest diff --git a/cmake/UserverPack.cmake b/cmake/UserverPack.cmake index 2b702528d61d..af281aa9419b 100644 --- a/cmake/UserverPack.cmake +++ b/cmake/UserverPack.cmake @@ -6,6 +6,13 @@ set(CPACK_PACKAGE_DESCRIPTION services and utilities." ) +option(USERVER_INSTALL_MULTIPACKAGE "Whether create per-component packages" OFF) +if(USERVER_INSTALL_MULTIPACKAGE) + set(CPACK_COMPONENTS_GROUPING ONE_PER_GROUP) +else() + set(CPACK_COMPONENTS_GROUPING ALL_COMPONENTS_IN_ONE) +endif() + set(CPACK_PACKAGE_NAME "libuserver-all-dev") set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_PACKAGE_VERSION "${USERVER_VERSION}") @@ -26,27 +33,9 @@ set(CPACK_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS WORLD_EXECUTE ) -# DEB dependencies: -execute_process(COMMAND lsb_release -cs OUTPUT_VARIABLE OS_CODENAME) -if(OS_CODENAME MATCHES "^bookworm") - set(DEPENDENCIES_FILE "debian-12.md") -elseif(OS_CODENAME MATCHES "^bullseye") - set(DEPENDENCIES_FILE "debian-11.md") -elseif(OS_CODENAME MATCHES "^noble") - set(DEPENDENCIES_FILE "ubuntu-24.04.md") -elseif(OS_CODENAME MATCHES "^jammy") - set(DEPENDENCIES_FILE "ubuntu-22.04.md") -elseif(OS_CODENAME MATCHES "^impish") - set(DEPENDENCIES_FILE "ubuntu-21.04.md") -elseif(OS_CODENAME MATCHES "^focal") - set(DEPENDENCIES_FILE "ubuntu-20.04.md") -elseif(OS_CODENAME MATCHES "^bionic") - set(DEPENDENCIES_FILE "ubuntu-18.04.md") -endif() - -if(DEPENDENCIES_FILE) +if(DEPENDENCIES_FILESTEM) execute_process( - COMMAND cat "${USERVER_ROOT_DIR}/scripts/docs/en/deps/${DEPENDENCIES_FILE}" + COMMAND cat "${USERVER_ROOT_DIR}/scripts/docs/en/deps/${DEPENDENCIES_FILESTEM}.md" COMMAND tr "\n" " " COMMAND sed "s/ \\(.\\)/, \\1/g" OUTPUT_VARIABLE CPACK_DEBIAN_PACKAGE_DEPENDS @@ -55,5 +44,16 @@ else() set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6") endif() +if(CPACK_COMPONENTS_GROUPING STREQUAL ONE_PER_GROUP) + set(CPACK_DEB_COMPONENT_INSTALL ON) + set(CPACK_DEBIAN_ENABLE_COMPONENT_DEPENDS ON) +endif() + +# Must go just before CPack +include("${CMAKE_BINARY_DIR}/cpack.variables.inc") + # CPack setup is ready. Including it: include(CPack) + +# Must go after all modules +include("${CMAKE_BINARY_DIR}/cpack.inc") diff --git a/cmake/UserverRequireDWCAS.cmake b/cmake/UserverRequireDWCAS.cmake index f280b0ec8926..12d502a119b4 100644 --- a/cmake/UserverRequireDWCAS.cmake +++ b/cmake/UserverRequireDWCAS.cmake @@ -38,6 +38,8 @@ function(userver_target_require_dwcas target visibility) # boost::atomic::value() since Boost 1.74.0. if("${Boost_VERSION_STRING}" VERSION_GREATER_EQUAL "${BOOST_DWCAS_MIN_VERSION}") message(STATUS "DWCAS: Using boost::atomic") + list(APPEND TEST_LIBRARIES "Boost::atomic") + get_target_property(BOOST_ATOMIC_INCLUDE_DIR Boost::atomic INTERFACE_INCLUDE_DIRECTORIES) else() message(WARNING "DWCAS: Using std::atomic") target_compile_definitions(${target} ${visibility} USERVER_USE_STD_DWCAS=1) @@ -92,8 +94,9 @@ function(userver_target_require_dwcas target visibility) try_run( RUN_RESULT COMPILE_RESULT "${CMAKE_CURRENT_BINARY_DIR}/require_dwcas" "${USERVER_ROOT_DIR}/cmake/UserverRequireDWCAS.cpp" - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIRS}" - COMPILE_DEFINITIONS ${TEST_DEFINITIONS} LINK_LIBRARIES ${TEST_LIBRARIES} + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${BOOST_ATOMIC_INCLUDE_DIR};${Boost_INCLUDE_DIRS}" + COMPILE_DEFINITIONS ${TEST_DEFINITIONS} + LINK_LIBRARIES ${TEST_LIBRARIES} COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT ) diff --git a/cmake/UserverSetupEnvironment.cmake b/cmake/UserverSetupEnvironment.cmake index 7f1d52c4edbe..351a34627001 100644 --- a/cmake/UserverSetupEnvironment.cmake +++ b/cmake/UserverSetupEnvironment.cmake @@ -77,6 +77,12 @@ function(_userver_setup_environment_impl) PARENT_SCOPE ) + if (NOT DEFINED CMAKE_CXX_SCAN_FOR_MODULES) + # For now, we don't use C++ modules, so we can save ourselves the overhead + # of scanning every source file + set(CMAKE_CXX_SCAN_FOR_MODULES OFF) + endif() + add_compile_options("-pipe" "-g" "-fPIC") add_compile_definitions("PIC=1") diff --git a/cmake/UserverSql.cmake b/cmake/UserverSql.cmake index fcf8e0bd13e4..00c5e7d36fdd 100644 --- a/cmake/UserverSql.cmake +++ b/cmake/UserverSql.cmake @@ -20,7 +20,7 @@ _userver_prepare_sql() function(userver_add_sql_library TARGET) set(OPTIONS) - set(ONE_VALUE_ARGS OUTPUT_DIR NAMESPACE QUERY_LOG_MODE) + set(ONE_VALUE_ARGS SOURCE_DIR OUTPUT_DIR NAMESPACE QUERY_LOG_MODE) set(MULTI_VALUE_ARGS SQL_FILES) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) if(NOT ARG_NAMESPACE) @@ -30,12 +30,19 @@ function(userver_add_sql_library TARGET) set(ARG_QUERY_LOG_MODE "full") endif() set(FILENAME "sql_queries") + if(NOT ARG_SOURCE_DIR) + set(ARG_SOURCE_DIR ".") + endif() + if(NOT IS_ABSOLUTE "${ARG_SOURCE_DIR}") + set(ARG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/${ARG_SOURCE_DIR}") + endif() set(SQL_FILES) foreach(WILDCARD ${ARG_SQL_FILES}) - file(GLOB FILES ${WILDCARD}) + file(GLOB_RECURSE FILES RELATIVE ${ARG_SOURCE_DIR} "${ARG_SOURCE_DIR}/${WILDCARD}") list(APPEND SQL_FILES ${FILES}) endforeach() + list(TRANSFORM SQL_FILES PREPEND "${ARG_SOURCE_DIR}/") get_property(USERVER_SQL_PYTHON_BINARY GLOBAL PROPERTY userver_sql_python_binary) get_property(USERVER_SQL_SCRIPTS_PATH GLOBAL PROPERTY userver_scripts_sql) @@ -50,7 +57,8 @@ function(userver_add_sql_library TARGET) OUTPUT ${output_files} COMMAND ${USERVER_SQL_PYTHON_BINARY} ${USERVER_SQL_SCRIPTS_PATH}/generator.py --namespace ${ARG_NAMESPACE} - --output-dir ${ARG_OUTPUT_DIR} --query-log-mode ${ARG_QUERY_LOG_MODE} --testsuite-output-dir + --source-dir ${ARG_SOURCE_DIR} --output-dir ${ARG_OUTPUT_DIR} + --query-log-mode ${ARG_QUERY_LOG_MODE} --testsuite-output-dir ${TESTSUITE_OUTPUT_DIR} ${SQL_FILES} ${CODEGEN} DEPENDS ${SQL_FILES} ) diff --git a/cmake/abseil_pr_1707.patch b/cmake/abseil_pr_1707.patch new file mode 100644 index 000000000000..31f44ade0dfd --- /dev/null +++ b/cmake/abseil_pr_1707.patch @@ -0,0 +1,33 @@ +commit 6dee153242d7becebe026a9bed52f4114441719d +Author: Soo-Hwan Na +Date: Wed Jul 10 09:20:07 2024 -0700 + + PR #1707: Fixup absl_random compile breakage in Apple ARM64 targets + + Imported from GitHub PR https://github.com/abseil/abseil-cpp/pull/1707 + + Switched to append a full string of "-Xarch_x86_64 -maes" instead of " -Xarch_x86_64" "-maes", for example. Now cmake correctly appends -Xarch_x86_64 to each x64 specific compile option, removing the error caused in recent clang releases: + + clang++: error: unsupported option '-msse4.1' for target 'arm64-apple-darwin23.5.0' + + Merge 83d17537ee70158d627681a0f0c15f15f30ef838 into f46495ea96f68fc3f6c394f099b2992743f6ff7f + + Merging this change closes #1707 + + COPYBARA_INTEGRATE_REVIEW=https://github.com/abseil/abseil-cpp/pull/1707 from Royna2544:patch-1 83d17537ee70158d627681a0f0c15f15f30ef838 + PiperOrigin-RevId: 651046496 + Change-Id: Ifdb3848febeead4fb562a2d9f0fdca2e0aea185d + +diff --git a/absl/copts/AbseilConfigureCopts.cmake b/absl/copts/AbseilConfigureCopts.cmake +index 1afb9610..e0007ef8 100644 +--- a/absl/copts/AbseilConfigureCopts.cmake ++++ b/absl/copts/AbseilConfigureCopts.cmake +@@ -42,7 +42,7 @@ if(APPLE AND CMAKE_CXX_COMPILER_ID MATCHES [[Clang]]) + string(TOUPPER "${_arch}" _arch_uppercase) + string(REPLACE "X86_64" "X64" _arch_uppercase ${_arch_uppercase}) + foreach(_flag IN LISTS ABSL_RANDOM_HWAES_${_arch_uppercase}_FLAGS) +- list(APPEND ABSL_RANDOM_RANDEN_COPTS "-Xarch_${_arch}" "${_flag}") ++ list(APPEND ABSL_RANDOM_RANDEN_COPTS "-Xarch_${_arch} ${_flag}") + endforeach() + endforeach() + # If a compiler happens to deal with an argument for a currently unused diff --git a/cmake/abseil_pr_1739.patch b/cmake/abseil_pr_1739.patch new file mode 100644 index 000000000000..ca0c47dcfb7b --- /dev/null +++ b/cmake/abseil_pr_1739.patch @@ -0,0 +1,38 @@ +From faf1b03a591f06933da02976119da5743f428e4f Mon Sep 17 00:00:00 2001 +From: Christopher Fore +Date: Mon, 5 Aug 2024 10:48:19 -0400 +Subject: [PATCH] container/internal: Explicitly include +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +GCC 15 will no longer include by default, resulting in build +failures in projects that do not explicitly include it. + +Error: +absl/container/internal/container_memory.h:66:27: error: ‘uintptr_t’ does not name a type + 66 | assert(reinterpret_cast(p) % Alignment == 0 && + | ^~~~~~~~~ +absl/container/internal/container_memory.h:31:1: note: ‘uintptr_t’ is defined in header ‘’; this is probably fixable by adding ‘#include ’ + 30 | #include "absl/utility/utility.h" + +++ |+#include + 31 | + +See-also: https://gcc.gnu.org/pipermail/gcc-cvs/2024-August/407124.html +Signed-off-by: Christopher Fore +--- + absl/container/internal/container_memory.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/absl/container/internal/container_memory.h b/absl/container/internal/container_memory.h +index ba8e08a2d22..e7031797018 100644 +--- a/absl/container/internal/container_memory.h ++++ b/absl/container/internal/container_memory.h +@@ -17,6 +17,7 @@ + + #include + #include ++#include + #include + #include + #include diff --git a/cmake/embedded_config.cmake b/cmake/embedded_config.cmake index e226a44d20d4..727626def2af 100644 --- a/cmake/embedded_config.cmake +++ b/cmake/embedded_config.cmake @@ -1,6 +1,6 @@ cmake_policy(SET CMP0053 NEW) -set(NAMESPACE userver) +set(NAMESPACE USERVER_NAMESPACE) set(FILE_IN ${CMAKE_CURRENT_BINARY_DIR}/embedded.h.in) set(TEMPLATE " @@ -35,6 +35,7 @@ APPLE_PREFIX \"@NAME@_end:\\n\" \".byte 0\\n\" APPLE_PREFIX \"@NAME@_size:\\n\" \".int \" APPLE_PREFIX \"@NAME@_end - \" APPLE_PREFIX \"@NAME@_begin\\n\" +\".previous\\n\" ); extern \"C\" const char @NAME@_begin[]; @@ -43,7 +44,7 @@ extern \"C\" const int @NAME@_size; __attribute__((constructor)) void @NAME@_call() { - utils::RegisterResource(\"@NAME@\", std::string_view{@NAME@_begin, static_cast(@NAME@_size)}); + ${NAMESPACE}::utils::RegisterResource(\"@NAME@\", std::string_view{@NAME@_begin, static_cast(@NAME@_size)}); } " ) diff --git a/cmake/grpc_pr_36805.patch b/cmake/grpc_pr_36805.patch new file mode 100644 index 000000000000..9a50fabef48e --- /dev/null +++ b/cmake/grpc_pr_36805.patch @@ -0,0 +1,30 @@ +commit e55f69cedd0ef7344e0bcb64b5ec9205e6aa4f04 +Author: Amy Huang +Date: Wed Jun 5 09:41:28 2024 -0700 + + [Fix] new clang -Wmissing-template-arg-list-after-template-kw warning (#36805) + + Clang now requires a template argument list after the use of the template keyword. Edit this + instance to remove the template keyword since there are no template arguments. + + See https://github.com/llvm/llvm-project/pull/80801. + + Closes #36805 + + COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36805 from amykhuang:master 6f385be2a0f014d1a27fb32e427938c4ced57f83 + PiperOrigin-RevId: 640554705 + +diff --git a/src/core/lib/promise/detail/basic_seq.h b/src/core/lib/promise/detail/basic_seq.h +index 663609d41..5f4199e5d 100644 +--- a/src/core/lib/promise/detail/basic_seq.h ++++ b/src/core/lib/promise/detail/basic_seq.h +@@ -99,8 +99,7 @@ class BasicSeqIter { + } + cur_ = next; + state_.~State(); +- Construct(&state_, +- Traits::template CallSeqFactory(f_, *cur_, std::move(arg))); ++ Construct(&state_, Traits::CallSeqFactory(f_, *cur_, std::move(arg))); + return PollNonEmpty(); + }); + } diff --git a/cmake/modules/FindPythonDev.cmake b/cmake/modules/FindPythonDev.cmake index 28c5922e11e8..77babfbf125c 100644 --- a/cmake/modules/FindPythonDev.cmake +++ b/cmake/modules/FindPythonDev.cmake @@ -11,19 +11,14 @@ _userver_module_begin( python ) +# minimum version is bound by testsuite _userver_module_find_include( NAMES Python.h PATHS /usr/include/python3m - /usr/include/python3.6m - /usr/include/python3.7m - /usr/include/python3.8m /usr/include/python3.9m /usr/include/python3 - /usr/include/python3.6 - /usr/include/python3.7 - /usr/include/python3.8 /usr/include/python3.9 /usr/include/python3.10 /usr/include/python3.11 diff --git a/cmake/modules/Findc-ares.cmake b/cmake/modules/Findc-ares.cmake index 22cb379ee89a..d077dccfba44 100644 --- a/cmake/modules/Findc-ares.cmake +++ b/cmake/modules/Findc-ares.cmake @@ -13,6 +13,16 @@ _userver_module_begin( c-ares PKG_CONFIG_NAMES libcares + + CPM_NAME c-ares + CPM_GITHUB_REPOSITORY c-ares/c-ares + CPM_VERSION 1.34.5 + CPM_OPTIONS + "CARES_STATIC ON" + "CARES_SHARED OFF" + "CARES_INSTALL OFF" + "CARES_BUILD_TOOLS OFF" + "CARES_STATIC_PIC ON" ) _userver_module_find_include(NAMES ares.h) diff --git a/cmake/modules/Findlibev.cmake b/cmake/modules/Findlibev.cmake index 68022a78f1f0..f030286fcfe9 100644 --- a/cmake/modules/Findlibev.cmake +++ b/cmake/modules/Findlibev.cmake @@ -9,6 +9,10 @@ _userver_module_begin( libev-devel PACMAN_NAMES libev + + CPM_NAME libev + CPM_URL http://dist.schmorp.de/libev/libev-4.33.tar.gz + CPM_DOWNLOAD_ONLY ) _userver_module_find_include(NAMES ev.h libev/ev.h) @@ -17,6 +21,52 @@ _userver_module_find_library(NAMES ev) _userver_module_end() +function(_userver_execute_process) + execute_process( + ${ARGV} + RESULT_VARIABLE RET + ) + if(NOT ("${RET}" EQUAL 0)) + message(FATAL_ERROR "Command failed with return code ${RET} (${ARGV})") + endif() +endfunction() + if(NOT TARGET libev::libev) - add_library(libev::libev ALIAS libev) + if(TARGET libev) + add_library(libev::libev ALIAS libev) + elseif(libev_ADDED) + # nghttp2 doesn't use find_package(), but calls find_path() and find_library() + # so we have to provide libev.a at the _configure_ time, not at build time, =( + if(NOT EXISTS ${libev_BINARY_DIR}/.built) + _userver_execute_process( + COMMAND ${CMAKE_COMMAND} -E copy_directory ${libev_SOURCE_DIR} ${libev_BINARY_DIR} + ) + _userver_execute_process( + COMMAND ./configure + WORKING_DIRECTORY ${libev_BINARY_DIR} + ) + _userver_execute_process( + COMMAND make + WORKING_DIRECTORY ${libev_BINARY_DIR} + ) + _userver_execute_process( + COMMAND rm -rf ${libev_BINARY_DIR}/.libs/libev.so + ) + _userver_execute_process( + COMMAND touch ${libev_BINARY_DIR}/.built + ) + endif() + + add_library(libev STATIC IMPORTED) + target_include_directories(libev INTERFACE ${libev_BINARY_DIR}) + set_target_properties(libev PROPERTIES IMPORTED_LOCATION ${libev_BINARY_DIR}/.libs/libev.a) + + # For nghttp2 installed from CPM + list(APPEND CMAKE_INCLUDE_PATH ${libev_BINARY_DIR}) + list(APPEND CMAKE_LIBRARY_PATH ${libev_BINARY_DIR}/.libs) + + add_library(libev::libev ALIAS libev) + else() + message(FATAL_ERROR "libev cmake target not found, don't know how to link") + endif() endif() diff --git a/cmake/modules/Findlibmariadb.cmake b/cmake/modules/Findlibmariadb.cmake index f513785d9963..840526fe40cc 100644 --- a/cmake/modules/Findlibmariadb.cmake +++ b/cmake/modules/Findlibmariadb.cmake @@ -7,6 +7,8 @@ _userver_module_begin( libmariadb-dev FORMULA_NAMES mariadb + PACMAN_NAMES + mariadb-libs PKG_CONFIG_NAMES mariadb ) diff --git a/cmake/modules/Findlibnghttp2.cmake b/cmake/modules/Findlibnghttp2.cmake index 968f850663bf..03811d16fb3f 100644 --- a/cmake/modules/Findlibnghttp2.cmake +++ b/cmake/modules/Findlibnghttp2.cmake @@ -7,6 +7,16 @@ _userver_module_begin( nghttp2 PACMAN_NAMES libnghttp2 + + CPM_NAME libnghttp2 + CPM_GITHUB_REPOSITORY nghttp2/nghttp2 + CPM_VERSION 1.66.0 + CPM_GIT_TAG v1.66.0 + CPM_OPTIONS + "BUILD_STATIC_LIBS ON" + "BUILD_SHARED_LIBS OFF" + "ENABLE_APP OFF" + "ENABLE_EXAMPLES OFF" ) _userver_module_find_include(NAMES nghttp2/nghttp2.h) @@ -16,5 +26,11 @@ _userver_module_find_library(NAMES nghttp2) _userver_module_end() if(NOT TARGET libnghttp2::nghttp2) - add_library(libnghttp2::nghttp2 ALIAS libnghttp2) + if(TARGET libnghttp2) + add_library(libnghttp2::nghttp2 ALIAS libnghttp2) + elseif(TARGET nghttp2_static) + add_library(libnghttp2::nghttp2 ALIAS nghttp2_static) + else() + message(FATAL_ERROR "libnghttp2{,_static} cmake target not found, don't know how to link") + endif() endif() diff --git a/cmake/modules/Findyaml-cpp.cmake b/cmake/modules/Findyaml-cpp.cmake index 042f615f3196..3e10058d8c40 100644 --- a/cmake/modules/Findyaml-cpp.cmake +++ b/cmake/modules/Findyaml-cpp.cmake @@ -7,6 +7,10 @@ _userver_module_begin( yaml-cpp PACMAN_NAMES yaml-cpp + + CPM_NAME yaml-cpp + CPM_GITHUB_REPOSITORY jbeder/yaml-cpp + CPM_GIT_TAG yaml-cpp-0.7.0 ) _userver_module_find_include(NAMES yaml-cpp/yaml.h yaml-cpp/node.h PATH_SUFFIXES include) diff --git a/cmake/modules/Findzstd.cmake b/cmake/modules/Findzstd.cmake index 29b5865ab2c2..13ef2650f5e5 100644 --- a/cmake/modules/Findzstd.cmake +++ b/cmake/modules/Findzstd.cmake @@ -9,6 +9,8 @@ _userver_module_begin( libzstd-dev PACMAN_NAMES zstd + + # TODO: CPM ) _userver_module_find_include(NAMES zdict.h zstd.h zstd_errors.h PATH_SUFFIXES include) @@ -18,7 +20,11 @@ _userver_module_find_library(NAMES zstd PATH_SUFFIXES lib) _userver_module_end() if(NOT TARGET zstd::zstd) - add_library(zstd::zstd ALIAS zstd) + if(TARGET libzstd_static) + add_library(zstd::zstd ALIAS libzstd_static) + else() + add_library(zstd::zstd ALIAS zstd) + endif() endif() if(NOT TARGET ZSTD::ZSTD) add_library(ZSTD::ZSTD ALIAS zstd) diff --git a/conanfile.py b/conanfile.py index 5c084f9db151..d0ba854da730 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,7 +1,6 @@ # pylint: disable=no-member import os import platform -import re from conan import ConanFile from conan.errors import ConanInvalidConfiguration @@ -12,6 +11,7 @@ from conan.tools.files import copy from conan.tools.files import load from conan.tools.scm import Git +from conan.tools.system import package_manager required_conan_version = '>=2.8.0' # pylint: disable=invalid-name @@ -104,13 +104,11 @@ def set_version(self): self, os.path.join( os.path.dirname(os.path.realpath(__file__)), - 'cmake/GetUserverVersion.cmake', + 'version.txt', ), ) - major_version = re.search(r'set\(USERVER_MAJOR_VERSION (.*)\)', content).group(1).strip() - minor_version = re.search(r'set\(USERVER_MINOR_VERSION (.*)\)', content).group(1).strip() - self.version = f'{major_version}.{minor_version}' # pylint: disable=attribute-defined-outside-init + self.version = content.strip() # pylint: disable=attribute-defined-outside-init def layout(self): cmake_layout(self) @@ -152,7 +150,9 @@ def requirements(self): ) self.requires('googleapis/cci.20230501') if self.options.with_postgresql: - self.requires('libpq/14.5') + # `run=True` required to find `pg_config` binary during `psycopg2` python module build + # without system package. We use system package. + self.requires('libpq/14.9') if self.options.with_mongodb or self.options.with_kafka: self.requires('cyrus-sasl/2.1.28') if self.options.with_mongodb: @@ -249,6 +249,12 @@ def generate(self): CMakeDeps(self).generate() def build(self): + # pg_config is required to build psycopg2 from source without system package. + # However, this approach fails on later stage, when venv for tests is built. + # libpq = self.dependencies["libpq"] + # if libpq: + # os.environ["PATH"] = os.environ["PATH"] + ":" + libpq.package_folder+ "/bin" + cmake = CMake(self) cmake.configure() cmake.build() @@ -261,3 +267,12 @@ def package_info(self): # https://docs.conan.io/2/examples/tools/cmake/cmake_toolchain/use_package_config_cmake.html self.cpp_info.set_property('cmake_find_mode', 'none') self.cpp_info.builddirs.append(os.path.join('lib', 'cmake', 'userver')) + + def system_requirements(self): + if self.options.with_postgresql: + # pg_config is required to build psycopg2 python module from source at + # testsuite venv creation during functional testing of user code. + package_manager.Apt(self).install(['libpq-dev']) + package_manager.Yum(self).install(['libpq-devel']) + package_manager.PacMan(self).install(['libpq-dev']) + package_manager.Zypper(self).install(['libpq-devel']) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 8cb558eb936d..659b6194f1db 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -37,16 +37,13 @@ file(GLOB_RECURSE INTERNAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/internal/*.cpp list(REMOVE_ITEM SOURCES ${INTERNAL_SOURCES}) -find_package(Boost REQUIRED CONFIG COMPONENTS program_options filesystem locale regex iostreams) find_package_required(ZLIB "zlib1g-dev") find_package(Iconv REQUIRED) -_userver_macos_set_default_dir(OPENSSL_ROOT_DIR "brew;--prefix;openssl") -find_package_required(OpenSSL "libssl-dev") -find_package(libnghttp2 REQUIRED) find_package(libev REQUIRED) +find_package(libnghttp2 REQUIRED) if(USERVER_CONAN) find_package(c-ares REQUIRED) @@ -98,15 +95,26 @@ endif() target_link_libraries( ${PROJECT_NAME} - PUBLIC userver-universal Boost::locale CURL::libcurl - PRIVATE Boost::filesystem - Boost::program_options - Boost::iostreams - Iconv::Iconv - OpenSSL::Crypto - OpenSSL::SSL - ZLIB::ZLIB + PUBLIC + userver-universal + Boost::locale + CURL::libcurl + PRIVATE + Boost::filesystem + Boost::program_options + Boost::iostreams + Iconv::Iconv + OpenSSL::SSL + OpenSSL::Crypto + ZLIB::ZLIB ) +if(TARGET Boost::endian) + # Boost from CPM + target_link_libraries( + ${PROJECT_NAME} + PUBLIC userver-universal Boost::endian Boost::lockfree + ) +endif() add_subdirectory(${USERVER_THIRD_PARTY_DIRS}/llhttp llhttp) @@ -130,17 +138,24 @@ if(CMAKE_SYSTEM_NAME MATCHES "Darwin" AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64" endif() endif() -option(USERVER_FEATURE_UBOOST_CORO "Use vendored boost context instead of a system one" - "${USERVER_UBOOST_CORO_DEFAULT}" -) +if("${Boost_VERSION_STRING}" VERSION_LESS "1.88.0") + option(USERVER_FEATURE_UBOOST_CORO "Use vendored boost context instead of a system one" + "${USERVER_UBOOST_CORO_DEFAULT}" + ) +else() + message(STATUS "Found a modern Boost version (${Boost_VERSION_STRING}), not using vendored Boost::context and Boost::coroutine2") + set(USERVER_FEATURE_UBOOST_CORO OFF) +endif() if(USERVER_FEATURE_UBOOST_CORO) add_subdirectory(${USERVER_THIRD_PARTY_DIRS}/uboost_coro uboost_coro_build) target_link_libraries(${PROJECT_NAME} PRIVATE userver-uboost-coro) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/uboost_coro/include) else() - find_package(Boost REQUIRED CONFIG COMPONENTS context coroutine) target_link_libraries(${PROJECT_NAME} PRIVATE Boost::context) + if(NOT USERVER_CONAN AND (TARGET Boost::coroutine2)) + target_link_libraries(${PROJECT_NAME} PRIVATE Boost::coroutine2) + endif() target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/sys_coro/include) endif() @@ -319,3 +334,5 @@ _userver_directory_install( DIRECTORY "${USERVER_ROOT_DIR}/testsuite/include_tests/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver/testsuite/include_tests/ ) + +_userver_install_component(MODULE core DEPENDS chaotic) diff --git a/core/dynamic_configs/EGRESS_HTTP_PROXY_ENABLED.yaml b/core/dynamic_configs/EGRESS_HTTP_PROXY_ENABLED.yaml new file mode 100644 index 000000000000..35ff566e5c41 --- /dev/null +++ b/core/dynamic_configs/EGRESS_HTTP_PROXY_ENABLED.yaml @@ -0,0 +1,4 @@ +default: false +description: '' +schema: + type: boolean diff --git a/core/dynamic_configs/EGRESS_NO_PROXY_TARGETS.yaml b/core/dynamic_configs/EGRESS_NO_PROXY_TARGETS.yaml new file mode 100644 index 000000000000..24e07b0c0514 --- /dev/null +++ b/core/dynamic_configs/EGRESS_NO_PROXY_TARGETS.yaml @@ -0,0 +1,15 @@ +default: + targets: [] +description: '' +schema: + type: object + additionalProperties: false + required: + - targets + properties: + targets: + type: array + items: + type: string + pattern: '^([^\/]+)(:\d+)?$' + x-taxi-cpp-type: std::unordered_set diff --git a/core/functional_tests/CMakeLists.txt b/core/functional_tests/CMakeLists.txt index 7b06961cb8d6..fca7a32ff14d 100644 --- a/core/functional_tests/CMakeLists.txt +++ b/core/functional_tests/CMakeLists.txt @@ -29,6 +29,9 @@ add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-static-service) add_subdirectory(tracing) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-tracing) +add_subdirectory(trx_tracker) +add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-trx-tracker) + add_subdirectory(uctl) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-uctl) diff --git a/core/functional_tests/metrics/tests/static/metrics_values.txt b/core/functional_tests/metrics/tests/static/metrics_values.txt index 694908a4c240..2b11b4aa5eb9 100644 --- a/core/functional_tests/metrics/tests/static/metrics_values.txt +++ b/core/functional_tests/metrics/tests/static/metrics_values.txt @@ -4,7 +4,7 @@ alerts.config_parse_error: GAUGE 0 alerts.dynamic_debug_invalid_location: GAUGE 0 alerts.log_reopening_error: GAUGE 0 -# Info on cahces +# Info on caches cache.any.documents.parse_failures.v2: cache_name=dynamic-config-client-updater RATE 0 cache.any.documents.parse_failures.v2: cache_name=sample-cache RATE 0 cache.any.documents.parse_failures: cache_name=dynamic-config-client-updater GAUGE 0 @@ -114,7 +114,13 @@ engine.coro-pool.stack-usage.is-monitor-active: GAUGE 0 engine.coro-pool.stack-usage.max-usage-percent: GAUGE 0 engine.ev-threads.cpu-load-percent: ev_thread_name=event-worker_0 GAUGE 0 engine.ev-threads.cpu-load-percent: ev_thread_name=event-worker_1 GAUGE 0 + +# Duration of time the service is running engine.load-ms: GAUGE 0 + +# Service startup took this amount of time +engine.pre-load-ms: GAUGE 0 + engine.task-processors-load-percent: task_processor=fs-task-processor, thread=0 GAUGE 0 engine.task-processors-load-percent: task_processor=fs-task-processor, thread=1 GAUGE 0 engine.task-processors-load-percent: task_processor=main-task-processor, thread=0 GAUGE 0 @@ -188,6 +194,10 @@ engine.task-processors.worker-threads: task_processor=fs-task-processor GAUGE 0 engine.task-processors.worker-threads: task_processor=main-task-processor GAUGE 0 engine.task-processors.worker-threads: task_processor=monitor-task-processor GAUGE 0 +# How many times heavy operations (like http calls) were called with open DB transaction +# See userver/utils/trx_tracker.hpp +engine.heavy-operations-in-transactions: RATE 0 + # How long the engine is running. It does not account components state (i.e. `components::State::IsAnyComponentInFatalState()`), # the engine could be fine, the components could be in components::ComponentHealth::kFatal state. engine.uptime-seconds: GAUGE 0 diff --git a/core/functional_tests/trx_tracker/CMakeLists.txt b/core/functional_tests/trx_tracker/CMakeLists.txt new file mode 100644 index 000000000000..d01c6df66c24 --- /dev/null +++ b/core/functional_tests/trx_tracker/CMakeLists.txt @@ -0,0 +1,8 @@ +project(userver-core-tests-trx-tracker CXX) + +file(GLOB_RECURSE SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*pp") +add_executable(${PROJECT_NAME} ${SOURCES} "main.cpp") +target_include_directories(${PROJECT_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") +target_link_libraries(${PROJECT_NAME} userver::core) + +userver_chaos_testsuite_add() diff --git a/core/functional_tests/trx_tracker/main.cpp b/core/functional_tests/trx_tracker/main.cpp new file mode 100644 index 000000000000..9d946e30ed59 --- /dev/null +++ b/core/functional_tests/trx_tracker/main.cpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/handler.hpp" + +int main(int argc, char* argv[]) { + const auto component_list = components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append() + .Append() + .Append(); + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/core/functional_tests/trx_tracker/src/handler.hpp b/core/functional_tests/trx_tracker/src/handler.hpp new file mode 100644 index 000000000000..c4f2018f6e19 --- /dev/null +++ b/core/functional_tests/trx_tracker/src/handler.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include + +namespace handlers { + +class Handler final : public server::handlers::HttpHandlerBase { +public: + static constexpr std::string_view kName = "handler"; + + Handler(const components::ComponentConfig& config, const components::ComponentContext& context) + : HttpHandlerBase(config, context) {} + + std::string HandleRequestThrow(const server::http::HttpRequest& /*request*/, server::request::RequestContext&) + const override { + utils::trx_tracker::TransactionLock lock; + lock.Lock(); + // Tespoint shouldn't actually trigger heavy operation + // in transaction check despite being implemented + // using http request + TESTPOINT("tp", formats::json::Value{}); + // Other heavy operation in transaction check + // should actually trigger + utils::trx_tracker::CheckNoTransactions(); + return ""; + } +}; + +} // namespace handlers diff --git a/core/functional_tests/trx_tracker/static_config.yaml b/core/functional_tests/trx_tracker/static_config.yaml new file mode 100644 index 000000000000..ac3d0a481d07 --- /dev/null +++ b/core/functional_tests/trx_tracker/static_config.yaml @@ -0,0 +1,53 @@ +components_manager: + task_processors: + main-task-processor: + worker_threads: 4 + + fs-task-processor: + worker_threads: 2 + + monitor-task-processor: + thread_name: mon-worker + worker_threads: 2 + + default_task_processor: main-task-processor + + components: + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: debug + overflow_behavior: discard + + server: + listener: + port: 8080 + task_processor: main-task-processor + listener-monitor: + port: 8081 + task_processor: main-task-processor + + handler: + path: /handler + method: GET + task_processor: main-task-processor + + handler-server-monitor: + path: /service/monitor + method: GET + task_processor: monitor-task-processor + + http-client: + fs-task-processor: fs-task-processor + user-agent: $server-name + user-agent#fallback: 'userver-based-service 1.0' + testsuite-support: + tests-control: + path: /tests/{action} + method: POST + task_processor: main-task-processor + testpoint-timeout: 10s + testpoint-url: $mockserver/testpoint + throttling_enabled: false diff --git a/core/functional_tests/trx_tracker/tests/conftest.py b/core/functional_tests/trx_tracker/tests/conftest.py new file mode 100644 index 000000000000..699ab947e59b --- /dev/null +++ b/core/functional_tests/trx_tracker/tests/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ['pytest_userver.plugins.core'] diff --git a/core/functional_tests/trx_tracker/tests/test_ignore_testpoint.py b/core/functional_tests/trx_tracker/tests/test_ignore_testpoint.py new file mode 100644 index 000000000000..8ce9dd87d4ee --- /dev/null +++ b/core/functional_tests/trx_tracker/tests/test_ignore_testpoint.py @@ -0,0 +1,18 @@ +HEAVY_OPERAIONS_IN_TRANSACTIONS = 'engine.heavy-operations-in-transactions' + + +async def test_trx_tracker_ignore_testpoint(service_client, monitor_client, testpoint): + @testpoint('tp') + def tp(data): + pass + + response = await service_client.get('/handler') + assert response.status == 200 + + # Check that testpoint was called + assert tp.times_called == 1 + + # Check that only manual heavy operation in + # transaction check was triggered + metrics = await monitor_client.metrics(prefix=HEAVY_OPERAIONS_IN_TRANSACTIONS) + assert metrics.value_at(path=HEAVY_OPERAIONS_IN_TRANSACTIONS) == 1 diff --git a/core/functional_tests/websocket/tests/test_websocket.py b/core/functional_tests/websocket/tests/test_websocket.py index 004a95af7c76..56f0eb2ca449 100644 --- a/core/functional_tests/websocket/tests/test_websocket.py +++ b/core/functional_tests/websocket/tests/test_websocket.py @@ -16,6 +16,19 @@ async def test_echo(websocket_client): assert response == 'hello' +async def test_echo_with_continuation(websocket_client): + async with websocket_client.get('chat') as chat: + # Send first fragment (not final, text frame) + await chat.write_frame(fin=False, opcode=0x1, data=b'First') + # Send intermediate fragment (not final, continuation frame) + await chat.write_frame(fin=False, opcode=0x0, data=b' second') + # Send last fragment (final, continuation frame) + await chat.write_frame(fin=True, opcode=0x0, data=b' third') + + response = await chat.recv() + assert response == 'First second third' + + async def test_close_by_server(websocket_client): async with websocket_client.get('chat') as chat: await chat.send('close') @@ -142,3 +155,28 @@ async def test_ping_pong_close(websocket_client): break assert connection_closed_by_ping + + +async def test_upgrade_header_with_tab_then_reconnect(service_port): + reader, writer = await asyncio.open_connection('localhost', service_port) + + # request with tab in Upgrade header + bad_request = ( + 'GET /chat HTTP/1.1\r\n' + 'Sec-WebSocket-Version: 13\r\n' + 'Sec-WebSocket-Key: fQU/VaAZ3+lpmSjWKevurQ==\r\n' + 'Connection: Upgrade\r\n' + 'Upgrade:\twebsocket\r\n' + '\r\n' + ) + + writer.write(bad_request.encode('ascii')) + await writer.drain() + writer.close() + await writer.wait_closed() + + # 2. Check new connection can be established + async with websockets.connect(f'ws://localhost:{service_port}/chat') as chat: + await chat.send('ping') + resp = await chat.recv() + assert resp == 'ping' diff --git a/core/include/userver/cache/expirable_lru_cache.hpp b/core/include/userver/cache/expirable_lru_cache.hpp index 464466551743..5f23f0ccd7df 100644 --- a/core/include/userver/cache/expirable_lru_cache.hpp +++ b/core/include/userver/cache/expirable_lru_cache.hpp @@ -121,6 +121,8 @@ class ExpirableLruCache final { */ std::optional GetOptionalNoUpdate(const Key& key); + std::optional> GetOptionalNoUpdateWithLastUpdateTime(const Key& key); + void Put(const Key& key, const Value& value); void Put(const Key& key, Value&& value); @@ -286,21 +288,28 @@ std::optional ExpirableLruCache::GetOptionalUnex } template -std::optional ExpirableLruCache::GetOptionalNoUpdate(const Key& key) { - auto now = utils::datetime::SteadyNow(); - auto old_value = lru_.Get(key); - +std::optional> +ExpirableLruCache::GetOptionalNoUpdateWithLastUpdateTime(const Key& key) { + const auto now = utils::datetime::SteadyNow(); + const auto old_value = lru_.Get(key); if (old_value) { if (!IsExpired(old_value->update_time, now)) { impl::CacheHit(stats_); - - return old_value->value; + return old_value; } else { impl::CacheStale(stats_); } } impl::CacheMiss(stats_); + return std::nullopt; +} +template +std::optional ExpirableLruCache::GetOptionalNoUpdate(const Key& key) { + auto value_with_update_time = GetOptionalNoUpdateWithLastUpdateTime(key); + if (value_with_update_time.has_value()) { + return value_with_update_time->value; + } return std::nullopt; } diff --git a/core/include/userver/clients/http/client.hpp b/core/include/userver/clients/http/client.hpp index eefe720fcdd0..032334c09566 100644 --- a/core/include/userver/clients/http/client.hpp +++ b/core/include/userver/clients/http/client.hpp @@ -62,7 +62,11 @@ class DestinationStatistics; /// @snippet clients/http/client_test.cpp Sample HTTP Client usage class Client final { public: - Client(ClientSettings settings, engine::TaskProcessor& fs_task_processor, impl::PluginPipeline&& plugin_pipeline); + Client( + ClientSettings settings, + engine::TaskProcessor& fs_task_processor, + std::vector> plugins + ); ~Client(); @@ -172,7 +176,7 @@ class Client final { clients::dns::Resolver* resolver_{nullptr}; utils::NotNull tracing_manager_; - impl::PluginPipeline plugin_pipeline_; + std::vector> plugins_; }; } // namespace clients::http diff --git a/core/include/userver/clients/http/plugin.hpp b/core/include/userver/clients/http/plugin.hpp index 7d0f908af1e8..72369f656cbc 100644 --- a/core/include/userver/clients/http/plugin.hpp +++ b/core/include/userver/clients/http/plugin.hpp @@ -91,7 +91,7 @@ namespace impl { class PluginPipeline final { public: - PluginPipeline(const std::vector>& plugins); + explicit PluginPipeline(const std::vector>& plugins); void HookPerformRequest(RequestState& request); @@ -104,7 +104,7 @@ class PluginPipeline final { bool HookOnRetry(RequestState& request); private: - const std::vector> plugins_; + utils::NotNull>*> plugins_; }; } // namespace impl diff --git a/core/include/userver/clients/http/request.hpp b/core/include/userver/clients/http/request.hpp index 4dc922a243e2..687956445035 100644 --- a/core/include/userver/clients/http/request.hpp +++ b/core/include/userver/clients/http/request.hpp @@ -89,7 +89,7 @@ class Request final { RequestStats&& req_stats, const std::shared_ptr& dest_stats, clients::dns::Resolver* resolver, - impl::PluginPipeline& plugin_pipeline, + const std::vector>& plugins, const tracing::TracingManagerBase& tracing_manager ); /// @endcond @@ -250,6 +250,9 @@ class Request final { return *this; } + /// Override list of plugins from @ref components::HttpClient for specific request + Request& SetPluginsList(const std::vector>& plugins) &; + /// Override log URL. Useful for "there's a secret in the query". /// @warning The query might be logged by other intermediate HTTP agents /// (nginx, L7 balancer, etc.). diff --git a/core/include/userver/clients/http/response_future.hpp b/core/include/userver/clients/http/response_future.hpp index 49015207f1a4..13d12d4ecf77 100644 --- a/core/include/userver/clients/http/response_future.hpp +++ b/core/include/userver/clients/http/response_future.hpp @@ -13,6 +13,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -44,10 +45,10 @@ class ResponseFuture final { /// @brief Stops the current task execution until the request finishes /// @throws clients::http::CancelException if the current task is being cancelled /// @returns std::future_status::ready or std::future_status::timeout - std::future_status Wait(); + std::future_status Wait(utils::impl::SourceLocation location = utils::impl::SourceLocation::Current()); /// @brief Wait for the response and return it - std::shared_ptr Get(); + std::shared_ptr Get(utils::impl::SourceLocation location = utils::impl::SourceLocation::Current()); void SetCancellationPolicy(CancellationPolicy cp); diff --git a/core/include/userver/components/manager_controller_component.hpp b/core/include/userver/components/manager_controller_component.hpp index 5a5458b5fb54..5a3bf391fa0d 100644 --- a/core/include/userver/components/manager_controller_component.hpp +++ b/core/include/userver/components/manager_controller_component.hpp @@ -49,6 +49,7 @@ class Manager; /// preheat_stacktrace_collector | whether to collect a dummy stacktrace at server start up (usable to avoid loading debug info at random point at runtime) | true /// userver_experiments.*NAME* | whether to enable certain userver experiments; these are gradually enabled by userver team, for internal use only | false /// graceful_shutdown_interval | at shutdown, first hang for this duration with /ping 5xx to give the balancer a chance to redirect new requests to other hosts | 0s +/// enable_trx_tracker | Enable checking of heavy operations (like http calls) while having active database transactions. | true /// /// ## Static task_processor options: /// Name | Description | Default value diff --git a/core/include/userver/concurrent/queue.hpp b/core/include/userver/concurrent/queue.hpp index 2d7b4d820893..6ae1853dda92 100644 --- a/core/include/userver/concurrent/queue.hpp +++ b/core/include/userver/concurrent/queue.hpp @@ -278,14 +278,12 @@ class GenericQueue final : public std::enable_shared_from_this [[nodiscard]] bool Push(Token& token, T&& value, engine::Deadline deadline) { const std::size_t value_size = QueuePolicy::GetElementSize(value); - UASSERT(value_size > 0); return producer_side_.Push(token, std::move(value), deadline, value_size); } template [[nodiscard]] bool PushNoblock(Token& token, T&& value) { const std::size_t value_size = QueuePolicy::GetElementSize(value); - UASSERT(value_size > 0); return producer_side_.PushNoblock(token, std::move(value), value_size); } diff --git a/core/include/userver/congestion_control/component.hpp b/core/include/userver/congestion_control/component.hpp index b1a62f00a4c9..dae0aa80b3b3 100644 --- a/core/include/userver/congestion_control/component.hpp +++ b/core/include/userver/congestion_control/component.hpp @@ -14,6 +14,8 @@ USERVER_NAMESPACE_BEGIN namespace congestion_control { +class Controller; + // clang-format off /// @ingroup userver_components @@ -52,6 +54,7 @@ class Component final : public components::ComponentBase { server::congestion_control::Limiter& GetServerLimiter(); server::congestion_control::Sensor& GetServerSensor(); + const congestion_control::Controller& GetServerController() const; private: void OnConfigUpdate(const dynamic_config::Snapshot& cfg); diff --git a/core/include/userver/dump/aggregates.hpp b/core/include/userver/dump/aggregates.hpp index c22c4b86a89e..bfe2652be03f 100644 --- a/core/include/userver/dump/aggregates.hpp +++ b/core/include/userver/dump/aggregates.hpp @@ -33,7 +33,7 @@ constexpr bool AreAllDumpable(std::index_sequence) { template constexpr bool IsDumpableAggregate() { - if constexpr (std::is_aggregate_v && !meta::kIsDetected) { + if constexpr (std::is_aggregate_v && !meta::IsDetected) { constexpr auto kSize = boost::pfr::tuple_size_v; static_assert( AreAllDumpable(std::make_index_sequence{}), diff --git a/core/include/userver/dump/meta_containers.hpp b/core/include/userver/dump/meta_containers.hpp index 0c4b64dc6231..ba0c08f9ebba 100644 --- a/core/include/userver/dump/meta_containers.hpp +++ b/core/include/userver/dump/meta_containers.hpp @@ -10,6 +10,7 @@ #include #include +#include USERVER_NAMESPACE_BEGIN @@ -53,7 +54,7 @@ using InsertResult = decltype(dump::Insert(std::declval(), std::declval inline constexpr bool kIsContainer = meta::kIsRange && std::is_default_constructible_v && meta::kIsSizable && - meta::kIsDetected; + meta::IsDetected; } // namespace dump diff --git a/core/include/userver/dynamic_config/source.hpp b/core/include/userver/dynamic_config/source.hpp index 5106bbd23f6b..dde5e393a1a3 100644 --- a/core/include/userver/dynamic_config/source.hpp +++ b/core/include/userver/dynamic_config/source.hpp @@ -174,7 +174,7 @@ class Source final { /// the first time immediately invokes the function with the current config /// snapshot (this invocation will be executed synchronously). /// - /// @note Сallbacks occur only if one of the passed config is changed. This is + /// @note Callbacks occur only if one of the passed config is changed. This is /// true under any components::DynamicConfigClientUpdater options. /// /// @warning To use this function, configs must have the `operator==`. diff --git a/core/include/userver/engine/io/tls_wrapper.hpp b/core/include/userver/engine/io/tls_wrapper.hpp index f07d7163e4f8..6757b647f14d 100644 --- a/core/include/userver/engine/io/tls_wrapper.hpp +++ b/core/include/userver/engine/io/tls_wrapper.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -40,13 +41,7 @@ class [[nodiscard]] TlsWrapper final : public RwBase { ); /// Starts a TLS server on an opened socket - static TlsWrapper StartTlsServer( - Socket&& socket, - const crypto::CertificatesChain& cert_chain, - const crypto::PrivateKey& key, - Deadline deadline, - const std::vector& extra_cert_authorities = {} - ); + static TlsWrapper StartTlsServer(Socket&& socket, const crypto::SslCtx& ctx, Deadline deadline); ~TlsWrapper() override; @@ -72,6 +67,14 @@ class [[nodiscard]] TlsWrapper final : public RwBase { /// received any more, received bytes count otherwise. [[nodiscard]] size_t RecvSome(void* buf, size_t len, Deadline deadline); + /// @brief Receives up to len bytes from the socket + /// @returns + /// - nullopt on data absence + /// - optional{0} if socket is closed by peer. + /// - optional{data_bytes_available} otherwise, + /// 1 <= data_bytes_available <= len + [[nodiscard]] std::optional RecvNoblock(void* buf, size_t len); + /// @brief Receives exactly len bytes from the socket. /// @note Can return less than len if socket is closed by peer. [[nodiscard]] size_t RecvAll(void* buf, size_t len, Deadline deadline); @@ -85,6 +88,14 @@ class [[nodiscard]] TlsWrapper final : public RwBase { /// socket extraction if interrupted. [[nodiscard]] Socket StopTls(Deadline deadline); + /// @brief Receives up to len bytes from the stream + /// @returns + /// - nullopt on data absence + /// - optional{0} if socket is closed by peer. + /// - optional{data_bytes_available} otherwise, + /// 1 <= data_bytes_available <= len + [[nodiscard]] std::optional ReadNoblock(void* buf, size_t len) override { return RecvNoblock(buf, len); } + /// @brief Receives at least one byte from the socket. /// @returns 0 if connection is closed on one side and no data could be /// received any more, received bytes count otherwise. diff --git a/core/include/userver/engine/task/task_with_result.hpp b/core/include/userver/engine/task/task_with_result.hpp index 7cf2c79d00a8..5eee39c1cf11 100644 --- a/core/include/userver/engine/task/task_with_result.hpp +++ b/core/include/userver/engine/task/task_with_result.hpp @@ -54,11 +54,11 @@ class [[nodiscard]] TaskWithResult : public Task { EnsureValid(); Wait(); + const utils::FastScopeGuard invalidate([this]() noexcept { Invalidate(); }); if (GetState() == State::kCancelled) { throw TaskCancelledException(CancellationReason()); } - const utils::FastScopeGuard invalidate([this]() noexcept { Invalidate(); }); return utils::impl::CastWrappedCall(GetPayload()).Retrieve(); } diff --git a/core/include/userver/fs/temp_file.hpp b/core/include/userver/fs/temp_file.hpp index 4b73ca2b14ed..6032bf8ae565 100644 --- a/core/include/userver/fs/temp_file.hpp +++ b/core/include/userver/fs/temp_file.hpp @@ -40,7 +40,7 @@ class TempFile final { TempFile() = delete; TempFile(TempFile&& other) noexcept = default; TempFile& operator=(TempFile&& other) noexcept = default; - ~TempFile(); + ~TempFile() noexcept; /// Take ownership of an existing file static TempFile Adopt(std::string path, engine::TaskProcessor& fs_task_processor); diff --git a/core/include/userver/logging/component.hpp b/core/include/userver/logging/component.hpp index 48b63d8e72f3..1296e9c0808f 100644 --- a/core/include/userver/logging/component.hpp +++ b/core/include/userver/logging/component.hpp @@ -54,9 +54,9 @@ namespace components { /// /// ### Logs output /// You can specify where logs are written in the `file_path` option: -/// - Use file_path: '@stdout' to write your logs to standard output stream; -/// - Use file_path: '@stderr' to write your logs to standard error stream; -/// - Use file_path: '@null' to suppress sending of logs; +/// - Use file_path: '\@stdout' to write your logs to standard output stream; +/// - Use file_path: '\@stderr' to write your logs to standard error stream; +/// - Use file_path: '\@null' to suppress sending of logs; /// - Use file_path: /absolute/path/to/log/file.log to write your logs to file. Use USR1 signal or @ref server::handlers::OnLogRotate to reopen files after log rotation; /// - Use file_path: 'unix:/absolute/path/to/logs.sock' to write your logs to unix socket. Socket must be created before the service starts and closed by listener after service is shut down. /// diff --git a/core/include/userver/logging/logger.hpp b/core/include/userver/logging/logger.hpp index b46eca0a07d9..36061518e69c 100644 --- a/core/include/userver/logging/logger.hpp +++ b/core/include/userver/logging/logger.hpp @@ -44,13 +44,13 @@ LoggerPtr MakeStdoutLogger(const std::string& name, Format format, Level level = /// @see components::Logging LoggerPtr MakeFileLogger(const std::string& name, const std::string& path, Format format, Level level = Level::kInfo); -namespace impl::default_ { +namespace impl { bool DoShouldLog(Level) noexcept; void PrependCommonTags(TagWriter writer, Level logger_level); -} // namespace impl::default_ +} // namespace impl } // namespace logging diff --git a/core/include/userver/server/component.hpp b/core/include/userver/server/component.hpp index 4abddae83899..4823dbf03e72 100644 --- a/core/include/userver/server/component.hpp +++ b/core/include/userver/server/component.hpp @@ -38,7 +38,7 @@ namespace components { /// ---- | ----------- | ------------- /// logger_access | set to logger name from components::Logging component to write access logs into it; do not set to avoid writing access logs | - /// logger_access_tskv | set to logger name from components::Logging component to write access logs in TSKV format into it; do not set to avoid writing access logs | - -/// max_response_size_in_flight | set it to the size of response in bytes and the component will drop bigger responses from handlers that allow throttling | - +/// max_response_size_in_flight | drop incomming requests if the handler allows throttling and the size of waiting for send responses in bytes is greater than this value | - /// server-name | value to send in HTTP Server header | value from utils::GetUserverIdentifier() /// listener | (*required*) *see below* | - /// listener-monitor | *see below* | - diff --git a/core/include/userver/server/handlers/auth/digest/auth_checker_base.hpp b/core/include/userver/server/handlers/auth/digest/auth_checker_base.hpp index b7f326ca0422..63339a88237c 100644 --- a/core/include/userver/server/handlers/auth/digest/auth_checker_base.hpp +++ b/core/include/userver/server/handlers/auth/digest/auth_checker_base.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -45,9 +46,11 @@ class Hasher final { /// algorithm. std::string GetHash(std::string_view data) const; + /// @overload + std::string GetHash(std::initializer_list data) const; + private: - using HashAlgorithm = std::function; - HashAlgorithm hash_algorithm_; + const HashAlgTypes hash_algorithm_; const SecdistConfig& secdist_config_; }; diff --git a/core/include/userver/server/handlers/exceptions.hpp b/core/include/userver/server/handlers/exceptions.hpp index 0abfde4700ed..d9b9cf1cac97 100644 --- a/core/include/userver/server/handlers/exceptions.hpp +++ b/core/include/userver/server/handlers/exceptions.hpp @@ -107,13 +107,13 @@ template using HasInternalMessage = decltype(std::declval().GetInternalMessage()); template -inline constexpr bool kHasInternalMessage = meta::kIsDetected; +inline constexpr bool kHasInternalMessage = meta::IsDetected; template using HasExternalBody = decltype(std::declval().GetExternalBody()); template -inline constexpr bool kHasExternalBody = meta::kIsDetected; +inline constexpr bool kHasExternalBody = meta::IsDetected; template inline constexpr bool kIsMessageBuilder = kHasExternalBody; @@ -121,7 +121,7 @@ inline constexpr bool kIsMessageBuilder = kHasExternalBody; template struct MessageExtractor { static_assert( - meta::kIsDetected, + meta::IsDetected, "Please use your message builder to build external body for " "your error. See server::handlers::CustomHandlerException " "for more info" @@ -134,7 +134,7 @@ struct MessageExtractor { } std::string GetServiceCode() const { - if constexpr (meta::kIsDetected) { + if constexpr (meta::IsDetected) { return builder.GetServiceCode(); } else { return std::string{}; @@ -171,17 +171,17 @@ struct CustomHandlerExceptionData final { formats::json::Value details; private: - void Apply(HandlerErrorCode handler_code_) { handler_code = handler_code_; } + void Apply(HandlerErrorCode l_handler_code) { handler_code = l_handler_code; } - void Apply(ServiceErrorCode service_code_) { service_code = std::move(service_code_.body); } + void Apply(ServiceErrorCode l_service_code) { service_code = std::move(l_service_code.body); } - void Apply(InternalMessage internal_message_) { internal_message = std::move(internal_message_.body); } + void Apply(InternalMessage l_internal_message) { internal_message = std::move(l_internal_message.body); } - void Apply(ExternalBody external_body_) { external_body = std::move(external_body_.body); } + void Apply(ExternalBody l_external_body) { external_body = std::move(l_external_body.body); } - void Apply(ExtraHeaders headers_) { headers = std::move(headers_.headers); } + void Apply(ExtraHeaders l_headers) { headers = std::move(l_headers.headers); } - void Apply(formats::json::Value details_) { details = std::move(details_); } + void Apply(formats::json::Value l_details) { details = std::move(l_details); } template void Apply(MessageBuilder&& builder) { diff --git a/core/include/userver/server/http/http_request.hpp b/core/include/userver/server/http/http_request.hpp index 9cd50ac8a8e0..df636433c151 100644 --- a/core/include/userver/server/http/http_request.hpp +++ b/core/include/userver/server/http/http_request.hpp @@ -62,10 +62,15 @@ class HttpRequest final { /// @return Minor version of HTTP. For example, for HTTP 1.0 it returns 0 int GetHttpMinor() const; - /// @return Request URL + /// @brief Get HTTP request target as provided by the client (see + /// https://www.rfc-editor.org/rfc/rfc7230#section-5.3). May contain the whole URL, but usually it consists of path + /// and query string. const std::string& GetUrl() const; - /// @return Request path + /// @brief Get the path part of HTTP request URL. + /// + /// Unlike @ref server::handlers::HandlerConfig::path, path parameters are not replaced with placeholders, this is + /// the original highly cardinal path. const std::string& GetRequestPath() const; /// @cond diff --git a/core/include/userver/server/request/response_base.hpp b/core/include/userver/server/request/response_base.hpp index ad2bacf0be69..3c438c64baca 100644 --- a/core/include/userver/server/request/response_base.hpp +++ b/core/include/userver/server/request/response_base.hpp @@ -52,23 +52,23 @@ namespace server::request { class ResponseDataAccounter final { public: - void StartRequest(size_t size, std::chrono::steady_clock::time_point create_time); + void StartRequest(std::size_t size, std::chrono::steady_clock::time_point create_time); - void StopRequest(size_t size, std::chrono::steady_clock::time_point create_time); + void StopRequest(std::size_t size, std::chrono::steady_clock::time_point create_time); - size_t GetCurrentLevel() const { return current_; } + std::size_t GetPendingResponsesSizeInBytes() const { return pending_responses_size_in_bytes_; } - size_t GetMaxLevel() const { return max_; } + std::size_t GetMaxPendingResponsesSizeInBytes() const { return max_pending_responses_size_in_bytes_; } - void SetMaxLevel(size_t size) { max_ = size; } + void SetMaxPendingResponsesSizeInBytes(size_t size) { max_pending_responses_size_in_bytes_ = size; } std::chrono::milliseconds GetAvgRequestTime() const; private: - std::atomic current_{0}; - std::atomic max_{std::numeric_limits::max()}; - concurrent::StripedCounter count_; - concurrent::StripedCounter time_sum_; + std::atomic pending_responses_size_in_bytes_{0}; + std::atomic max_pending_responses_size_in_bytes_{std::numeric_limits::max()}; + concurrent::StripedCounter pending_responses_count_{}; + concurrent::StripedCounter time_sum_{}; }; // TODO: merge with HttpResponse diff --git a/core/include/userver/server/server.hpp b/core/include/userver/server/server.hpp index 347e22ea00bc..8d3e48640383 100644 --- a/core/include/userver/server/server.hpp +++ b/core/include/userver/server/server.hpp @@ -68,7 +68,7 @@ class Server final : public congestion_control::Limitee, public congestion_contr std::uint64_t GetTotalRequests() const override; private: - std::unique_ptr pimpl; + std::unique_ptr pimpl_; }; } // namespace server diff --git a/core/include/userver/storages/query.hpp b/core/include/userver/storages/query.hpp index adac7e46db82..5ef52ddbb295 100644 --- a/core/include/userver/storages/query.hpp +++ b/core/include/userver/storages/query.hpp @@ -51,7 +51,7 @@ class Query { enum class LogMode : unsigned char { kFull, ///< Output name and optionally statement - kNameOnly, ///< Ouput only name + kNameOnly, ///< Output only name }; Query() = default; @@ -85,12 +85,12 @@ class Query { private: struct DynamicStrings { - std::string statement_; - std::optional name_; + std::string statement; + std::optional name; }; struct StaticStrings { - utils::StringLiteral statement_; - std::optional name_; + utils::StringLiteral statement; + std::optional name; }; struct NameViewVisitor; diff --git a/core/include/userver/tracing/span.hpp b/core/include/userver/tracing/span.hpp index d34368fad53f..33d4c3ee9e5e 100644 --- a/core/include/userver/tracing/span.hpp +++ b/core/include/userver/tracing/span.hpp @@ -296,9 +296,9 @@ class Span final { static OptionalDeleter DoNotDelete() noexcept; private: - explicit OptionalDeleter(bool do_delete) : do_delete(do_delete) {} + explicit OptionalDeleter(bool do_delete) : do_delete_(do_delete) {} - const bool do_delete; + const bool do_delete_; }; friend class SpanBuilder; diff --git a/core/include/userver/utils/datetime.hpp b/core/include/userver/utils/datetime.hpp index cc0f69a0d63e..13a9f7901253 100644 --- a/core/include/userver/utils/datetime.hpp +++ b/core/include/userver/utils/datetime.hpp @@ -93,6 +93,15 @@ cctz::civil_second Localize(const std::chrono::system_clock::time_point& tp, con /// @snippet utils/datetime_test.cpp Localize example std::time_t Unlocalize(const cctz::civil_second& local_tp, const std::string& timezone); +/// @brief Retrieves a time zone by name +/// +/// Returns the corresponding time zone if the given timezone name is valid. +/// Returns std::nullopt if no matching time zone is found. +/// +/// @param timezone Time zone name (e.g. "UTC", "Europe/Moscow") +/// @return std::optional containing the time zone or std::nullopt +std::optional GetOptionalTimezone(const std::string& timezone); + } // namespace utils::datetime USERVER_NAMESPACE_END diff --git a/core/include/userver/utils/hedged_request.hpp b/core/include/userver/utils/hedged_request.hpp index 51e46261e030..e0bc57ca1eda 100644 --- a/core/include/userver/utils/hedged_request.hpp +++ b/core/include/userver/utils/hedged_request.hpp @@ -96,12 +96,12 @@ struct PlanEntry { PlanEntry(TimePoint timepoint, std::size_t request_index, std::size_t attempt_id, Action action) : timepoint(timepoint), request_index(request_index), attempt_id(attempt_id), action(action) {} - bool operator<(const PlanEntry& other) const noexcept { return tie() < other.tie(); } - bool operator>(const PlanEntry& other) const noexcept { return tie() > other.tie(); } - bool operator==(const PlanEntry& other) const noexcept { return tie() == other.tie(); } - bool operator<=(const PlanEntry& other) const noexcept { return tie() <= other.tie(); } - bool operator>=(const PlanEntry& other) const noexcept { return tie() >= other.tie(); } - bool operator!=(const PlanEntry& other) const noexcept { return tie() != other.tie(); } + bool operator<(const PlanEntry& other) const noexcept { return Tie() < other.Tie(); } + bool operator>(const PlanEntry& other) const noexcept { return Tie() > other.Tie(); } + bool operator==(const PlanEntry& other) const noexcept { return Tie() == other.Tie(); } + bool operator<=(const PlanEntry& other) const noexcept { return Tie() <= other.Tie(); } + bool operator>=(const PlanEntry& other) const noexcept { return Tie() >= other.Tie(); } + bool operator!=(const PlanEntry& other) const noexcept { return Tie() != other.Tie(); } TimePoint timepoint; std::size_t request_index{0}; @@ -109,7 +109,7 @@ struct PlanEntry { Action action; private: - std::tuple tie() const noexcept { + std::tuple Tie() const noexcept { return std::tie(timepoint, request_index, attempt_id, action); } }; @@ -144,7 +144,7 @@ struct Context { using ReplyType = typename RequestTraits::ReplyType; Context(std::vector inputs, HedgingSettings settings) - : inputs_(std::move(inputs)), settings(std::move(settings)) { + : inputs_(std::move(inputs)), settings_(std::move(settings)) { const std::size_t size = this->inputs_.size(); request_states_.resize(size); } @@ -155,8 +155,8 @@ struct Context { for (std::size_t request_id = 0; request_id < request_count; ++request_id) { plan_.emplace(start_time, request_id, 0, Action::StartTry); } - plan_.emplace(start_time + settings.timeout_all, 0, 0, Action::Stop); - subrequests_.reserve(settings.max_attempts * request_count); + plan_.emplace(start_time + settings_.timeout_all, 0, 0, Action::Stop); + subrequests_.reserve(settings_.max_attempts * request_count); } std::optional NextEventTime() const { @@ -185,7 +185,7 @@ struct Context { } } - const HedgingSettings& GetSettings() const { return settings; } + const HedgingSettings& GetSettings() const { return settings_; } size_t GetRequestsCount() const { return inputs_.size(); } @@ -232,7 +232,7 @@ struct Context { return; } - if (attempts_made >= settings.max_attempts) { + if (attempts_made >= settings_.max_attempts) { return; } auto& strategy = inputs_[request_index]; @@ -248,14 +248,14 @@ struct Context { request_state.subrequest_indices.push_back(idx); input_by_subrequests_[idx] = request_index; attempts_made++; - plan_.emplace(now + settings.hedging_delay, request_index, attempts_made, Action::StartTry); + plan_.emplace(now + settings_.hedging_delay, request_index, attempts_made, Action::StartTry); } /// Called on getting error in request with @param request_idx void OnRetriableReply(std::size_t request_idx, std::chrono::milliseconds retry_delay, TimePoint now) { const auto& request_state = request_states_[request_idx]; if (request_state.finished) return; - if (request_state.attempts_made >= settings.max_attempts) return; + if (request_state.attempts_made >= settings_.max_attempts) return; plan_.emplace(now + retry_delay, request_idx, request_state.attempts_made, Action::StartTry); } @@ -266,7 +266,7 @@ struct Context { private: /// user provided request strategies bulk std::vector inputs_; - HedgingSettings settings; + HedgingSettings settings_; /// Our plan of what we will do at what time std::priority_queue, std::greater<>> plan_{}; @@ -297,8 +297,8 @@ struct HedgedRequestBulkFuture { engine::impl::ContextAccessor* TryGetContextAccessor() { return task_.TryGetContextAccessor(); } private: - template - friend auto HedgeRequestsBulkAsync(std::vector inputs, HedgingSettings settings); + template + friend auto HedgeRequestsBulkAsync(std::vector inputs, HedgingSettings settings); using Task = engine::TaskWithResult>>; HedgedRequestBulkFuture(Task&& task) : task_(std::move(task)) {} Task task_; @@ -324,8 +324,8 @@ struct HedgedRequestFuture { engine::impl::ContextAccessor* TryGetContextAccessor() { return task_.TryGetContextAccessor(); } private: - template - friend auto HedgeRequestAsync(RequestStrategy_ input, HedgingSettings settings); + template + friend auto HedgeRequestAsync(TRequestStrategy input, HedgingSettings settings); using Task = engine::TaskWithResult>; HedgedRequestFuture(Task&& task) : task_(std::move(task)) {} Task task_; diff --git a/core/include/userver/utils/periodic_task.hpp b/core/include/userver/utils/periodic_task.hpp index 22ba3147ae93..31a4dd0209e5 100644 --- a/core/include/userver/utils/periodic_task.hpp +++ b/core/include/userver/utils/periodic_task.hpp @@ -6,22 +6,19 @@ #include #include #include -#include #include -#include -#include -#include +#include #include +#include #include -// TODO remove extra include -#include -#include -#include - USERVER_NAMESPACE_BEGIN +namespace engine { +class TaskProcessor; +} // namespace engine + namespace testsuite { class PeriodicTaskControl; } // namespace testsuite @@ -204,36 +201,10 @@ class PeriodicTask final { Settings GetCurrentSettings() const; private: - enum class SuspendState { kRunning, kSuspended }; - - void DoStart(); - - void Run(); - - bool Step(); - - bool StepDebug(bool preserve_span); - - bool DoStep(); - - std::chrono::milliseconds MutatePeriod(std::chrono::milliseconds period); - - std::string_view GetName() const noexcept; - - std::string name_; - std::atomic is_name_set_{false}; - Callback callback_; - engine::TaskWithResult task_; - rcu::Variable settings_; - engine::SingleConsumerEvent changed_event_; - std::atomic should_force_step_{false}; - std::optional mutate_period_random_; - - // For kNow only - engine::Mutex step_mutex_; - std::atomic suspend_state_; - - std::optional registration_holder_; + class Impl; + constexpr static std::size_t kSize = 432; + constexpr static std::size_t kAlignment = 16; + utils::FastPimpl impl_; }; } // namespace utils diff --git a/core/include/userver/utils/statistics/busy.hpp b/core/include/userver/utils/statistics/busy.hpp index 6533f1dca933..5845f60931c4 100644 --- a/core/include/userver/utils/statistics/busy.hpp +++ b/core/include/userver/utils/statistics/busy.hpp @@ -41,7 +41,7 @@ class BusyStorage final { Duration GetNotCommittedLoad() const noexcept; struct Impl; - std::unique_ptr pimpl; + std::unique_ptr pimpl_; }; /// @brief A RAII-style guard to account code block execution time in diff --git a/core/include/userver/utils/statistics/metric_tag_impl.hpp b/core/include/userver/utils/statistics/metric_tag_impl.hpp index fc0cc4d0279e..48dbedd0ab03 100644 --- a/core/include/userver/utils/statistics/metric_tag_impl.hpp +++ b/core/include/userver/utils/statistics/metric_tag_impl.hpp @@ -63,7 +63,7 @@ class MetricWrapperBase { template class MetricWrapper final : public MetricWrapperBase { static_assert( - meta::kIsDetected || kHasWriterSupport, + meta::IsDetected || kHasWriterSupport, "Provide a `void DumpMetric(utils::statistics::Writer&, const Metric&)`" "function in the namespace of `Metric`." ); @@ -94,7 +94,7 @@ class MetricWrapper final : public MetricWrapperBase { bool HasWriterSupport() const noexcept override { return kHasWriterSupport; } void Reset() override { - if constexpr (meta::kIsDetected) { + if constexpr (meta::IsDetected) { ResetMetric(data_); } } diff --git a/core/include/userver/utils/statistics/percentile_format_json.hpp b/core/include/userver/utils/statistics/percentile_format_json.hpp index 1a2183891a0c..1c95566948dd 100644 --- a/core/include/userver/utils/statistics/percentile_format_json.hpp +++ b/core/include/userver/utils/statistics/percentile_format_json.hpp @@ -24,7 +24,7 @@ std::string GetPercentileFieldName(double perc); template formats::json::ValueBuilder PercentileToJson(const T& perc, std::initializer_list percents) { static_assert( - meta::kIsDetected, + meta::IsDetected, "T must specify T::GetPercentile(double) returning " "json-serializable value" ); diff --git a/core/include/userver/utils/trx_tracker.hpp b/core/include/userver/utils/trx_tracker.hpp new file mode 100644 index 000000000000..24b07eacaeeb --- /dev/null +++ b/core/include/userver/utils/trx_tracker.hpp @@ -0,0 +1,131 @@ +#pragma once + +/// @file +/// @brief Tracking for heavy operations while having active transactions. + +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +/// @brief Tracking for heavy operations while having active transactions. +/// +/// Some operations, like http requests, are heavy and can take +/// too long during an incident. If they are called during an active +/// database transaction, connection will be held for longer and +/// connection pool will be exhausted. Transaction tracker prevents this +/// by holding counter of active transactions in TaskLocalVariable +/// and checking for active transactions in heavy operations. +/// +/// ## Example usage: +/// +/// @snippet utils/trx_tracker_test.cpp Sample TransactionTracker usage +namespace utils::trx_tracker { + +namespace impl { + +/// @brief Global enabler for transaction tracker. +class GlobalEnabler final { +public: + explicit GlobalEnabler(bool enable = true); + ~GlobalEnabler(); + + GlobalEnabler(const GlobalEnabler&) = delete; + GlobalEnabler& operator=(const GlobalEnabler&) = delete; +}; + +/// @brief Check if transaction tracker is enabled. +bool IsEnabled() noexcept; + +/// @brief Unique ID for every task. +/// +/// Sometimes transactions start and end in different coroutines. +/// To prevent transaction from incrementing and decrementing different +/// transaction counters, TransactionLock stores TaskId on Lock and +/// checks that stored TaskId is the same as current TaskId in Unlock. +class TaskId final { +public: + TaskId(); + + bool operator==(const TaskId& other) const; + bool operator!=(const TaskId& other) const; + +private: + std::thread::id created_thread_id_; + std::uint64_t thread_local_counter_; +}; + +} // namespace impl + +/// @brief Class for incrementing and decrementing transaction counter. +class TransactionLock final { +public: + TransactionLock() = default; + TransactionLock(const TransactionLock&) = delete; + TransactionLock(TransactionLock&&) noexcept; + TransactionLock operator=(const TransactionLock&) = delete; + TransactionLock& operator=(TransactionLock&&) noexcept; + + /// @brief Decrement transaction counter on destruction. + ~TransactionLock(); + + /// @brief Manually increment transaction counter. + void Lock() noexcept; + + /// @brief Manually decrement transaction counter. + void Unlock() noexcept; + +private: + std::optional task_id_; +}; + +/// @brief Check for active transactions. +void CheckNoTransactions(utils::impl::SourceLocation location = utils::impl::SourceLocation::Current()); + +/// @overload +void CheckNoTransactions(std::string_view location); + +/// @brief Disable check for active transactions. +/// +/// To consciously call a heavy operation in active transaction, +/// check can be disabled by creating an instance of this class. +/// Checks will be disabled until every instance either has +/// Reenable() method called or is destroyed. +class CheckDisabler final { +public: + /// @brief Disable check for active transactions. + explicit CheckDisabler(); + + /// @brief Reenable check for active transactions on destruction. + ~CheckDisabler(); + + CheckDisabler(const CheckDisabler&) = delete; + CheckDisabler(CheckDisabler&&) = delete; + CheckDisabler operator=(const CheckDisabler&) = delete; + CheckDisabler operator=(CheckDisabler&&) = delete; + + /// @brief Manually reenable check for active transactions. + void Reenable() noexcept; + +private: + bool reenabled_ = false; +}; + +/// @brief Statistics for transaction tracker. +struct TransactionTrackerStatistics final { + /// @brief How many times check for active transactions was triggered. + utils::statistics::Rate triggers{0}; +}; + +/// @brief Get statistics for transaction tracker. +TransactionTrackerStatistics GetStatistics() noexcept; + +/// @brief Reset statistics for transaction tracker. +void ResetStatistics(); + +} // namespace utils::trx_tracker + +USERVER_NAMESPACE_END diff --git a/core/library.yaml b/core/library.yaml index fb1445ea4fa9..9aab5047ddae 100644 --- a/core/library.yaml +++ b/core/library.yaml @@ -8,6 +8,8 @@ description: userver core stuff configs: names: - BAGGAGE_SETTINGS + - EGRESS_HTTP_PROXY_ENABLED + - EGRESS_NO_PROXY_TARGETS - HTTP_CLIENT_CONNECTION_POOL_SIZE - HTTP_CLIENT_CONNECT_THROTTLE - USERVER_BAGGAGE_ENABLED diff --git a/core/src/clients/dns/resolver.cpp b/core/src/clients/dns/resolver.cpp index 6a43d8bb1659..0ab57d1d5e6d 100644 --- a/core/src/clients/dns/resolver.cpp +++ b/core/src/clients/dns/resolver.cpp @@ -419,6 +419,8 @@ AddrVector Resolver::Resolve(const std::string& name, engine::Deadline deadline) } UINVARIANT(false, "Unexpected cache result status"); + // never reaches + return LocalhostAddrs(); } const Resolver::LookupSourceCounters& Resolver::GetLookupSourceCounters() const { diff --git a/core/src/clients/http/client.cpp b/core/src/clients/http/client.cpp index 701562f366b6..5447bedfe091 100644 --- a/core/src/clients/http/client.cpp +++ b/core/src/clients/http/client.cpp @@ -49,7 +49,7 @@ const tracing::TracingManagerBase* GetTracingManager(const ClientSettings& setti Client::Client( ClientSettings settings, engine::TaskProcessor& fs_task_processor, - impl::PluginPipeline&& plugin_pipeline + std::vector> plugins ) : deadline_propagation_config_(settings.deadline_propagation), cancellation_policy_(settings.cancellation_policy), @@ -59,7 +59,7 @@ Client::Client( user_agent_(utils::GetUserverIdentifier()), connect_rate_limiter_(std::make_shared()), tracing_manager_(GetTracingManager(settings)), - plugin_pipeline_(std::move(plugin_pipeline)) { + plugins_(std::move(plugins)) { const auto io_threads = settings.io_threads; const auto& thread_name_prefix = settings.thread_name_prefix; @@ -125,7 +125,7 @@ Request Client::CreateRequest() { statistics_[idx].CreateRequestStats(), destination_statistics_, resolver_, - plugin_pipeline_, + plugins_, *tracing_manager_.GetBase()}; } else { auto i = utils::RandRange(multis_.size()); @@ -140,7 +140,7 @@ Request Client::CreateRequest() { statistics_[i].CreateRequestStats(), destination_statistics_, resolver_, - plugin_pipeline_, + plugins_, *tracing_manager_.GetBase()}; } catch (engine::WaitInterruptedException&) { throw clients::http::CancelException("wait interrupted", {}, ErrorKind::kCancel); diff --git a/core/src/clients/http/client_crl_test.cpp b/core/src/clients/http/client_crl_test.cpp index 2efaecbdc819..226888730d16 100644 --- a/core/src/clients/http/client_crl_test.cpp +++ b/core/src/clients/http/client_crl_test.cpp @@ -309,19 +309,18 @@ EmOKfeOntrWGKRoDws82ckOkpBkZ0/9gsl8g18u+jFCcSUfmXH7FtGg= -----END X509 CRL-----)"; struct TlsServer { - TlsServer() : port_(tcp_listener_.socket.Getsockname().Port()) {} + TlsServer() : port(tcp_listener.socket.Getsockname().Port()) {} void ReceiveAndShutdown(std::initializer_list cas = {}) { auto deadline = engine::Deadline::FromDuration(utest::kMaxTestWaitTime); - auto socket = tcp_listener_.socket.Accept(deadline); + auto socket = tcp_listener.socket.Accept(deadline); - auto tls_server = engine::io::TlsWrapper::StartTlsServer( - std::move(socket), + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( crypto::LoadCertificatesChainFromString(kServerCertificate), crypto::PrivateKey::LoadFromString(kRevokedServerPrivateKey), - deadline, cas ); + auto tls_server = engine::io::TlsWrapper::StartTlsServer(std::move(socket), ssl_ctx, deadline); std::array data{}; const auto size = tls_server.RecvSome(data.data(), data.size(), deadline); @@ -340,8 +339,8 @@ struct TlsServer { EXPECT_TRUE(socket.IsValid()); } - internal::net::TcpListener tcp_listener_; - int port_; + internal::net::TcpListener tcp_listener; + int port; }; auto InterceptCrlDistribution() { @@ -371,7 +370,7 @@ UTEST(HttpClient, HttpsWithNoCrl) { auto http_client_ptr = utest::CreateHttpClient(); TlsServer tls_server; - const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port_); + const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port); auto response_future = http_client_ptr->CreateRequest() .post(ssl_url) @@ -400,7 +399,7 @@ UTEST(HttpClient, HttpsWithCrl) { auto http_client_ptr = utest::CreateHttpClient(); TlsServer tls_server; - const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port_); + const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port); auto response_future = http_client_ptr->CreateRequest() .post(ssl_url) @@ -428,7 +427,7 @@ UTEST(HttpClient, HttpsWithCrlNoVerify) { auto http_client_ptr = utest::CreateHttpClient(); TlsServer tls_server; - const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port_); + const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port); auto response_future = http_client_ptr->CreateRequest() .post(ssl_url) @@ -454,7 +453,7 @@ UTEST(HttpClient, HttpsWithNoServerCa) { auto http_client_ptr = utest::CreateHttpClient(); TlsServer tls_server; - const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port_); + const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port); auto response_future = http_client_ptr->CreateRequest() .post(ssl_url) @@ -478,7 +477,7 @@ UTEST(HttpClient, HttpsWithNoClientCa) { auto http_client_ptr = utest::CreateHttpClient(); TlsServer tls_server; - const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port_); + const auto ssl_url = fmt::format("https://[::1]:{}", tls_server.port); auto response_future = http_client_ptr->CreateRequest() .post(ssl_url) diff --git a/core/src/clients/http/client_test.cpp b/core/src/clients/http/client_test.cpp index 3791166de90c..1d564659c2f2 100644 --- a/core/src/clients/http/client_test.cpp +++ b/core/src/clients/http/client_test.cpp @@ -132,7 +132,7 @@ using HttpResponse = utest::SimpleServer::Response; using HttpRequest = utest::SimpleServer::Request; using HttpCallback = utest::SimpleServer::OnRequest; -std::optional process_100(const HttpRequest& request) { +std::optional Process100(const HttpRequest& request) { const bool requires_continue = (request.find("Expect: 100-continue") != std::string::npos); if (requires_continue || request.empty()) { @@ -151,7 +151,7 @@ struct EchoCallback { HttpResponse operator()(const HttpRequest& request) const { LOG_INFO() << "HTTP Server receive: " << request; - const auto cont = process_100(request); + const auto cont = Process100(request); if (cont) { return *cont; } @@ -184,7 +184,7 @@ struct ValidatingSharedCallback { HttpResponse operator()(const HttpRequest& request) const { LOG_INFO() << "HTTP Server receive: " << request; - const auto cont = process_100(request); + const auto cont = Process100(request); EXPECT_FALSE(!!cont) << "This callback does not work with CONTINUE"; @@ -245,7 +245,7 @@ struct AuthCallback { } }; -HttpResponse put_validate_callback(const HttpRequest& request) { +HttpResponse PutValidateCallback(const HttpRequest& request) { LOG_INFO() << "HTTP Server receive: " << request; EXPECT_NE(request.find("PUT"), std::string::npos) << "PUT request has no PUT in headers: " << request; @@ -256,7 +256,7 @@ HttpResponse put_validate_callback(const HttpRequest& request) { HttpResponse::kWriteAndClose}; } -HttpResponse sleep_callback_base(const HttpRequest& request, std::chrono::milliseconds sleep_for) { +HttpResponse SleepCallbackBase(const HttpRequest& request, std::chrono::milliseconds sleep_for) { LOG_INFO() << "HTTP Server receive: " << request; engine::InterruptibleSleepFor(sleep_for); @@ -268,18 +268,14 @@ HttpResponse sleep_callback_base(const HttpRequest& request, std::chrono::millis HttpResponse::kWriteAndClose}; } -HttpResponse sleep_callback(const HttpRequest& request) { - return sleep_callback_base(request, utest::kMaxTestWaitTime); -} +HttpResponse SleepCallback(const HttpRequest& request) { return SleepCallbackBase(request, utest::kMaxTestWaitTime); } -HttpResponse sleep_callback_1s(const HttpRequest& request) { - return sleep_callback_base(request, std::chrono::seconds(1)); -} +HttpResponse SleepCallback1s(const HttpRequest& request) { return SleepCallbackBase(request, std::chrono::seconds(1)); } -HttpResponse huge_data_callback(const HttpRequest& request) { +HttpResponse HugeDataCallback(const HttpRequest& request) { LOG_INFO() << "HTTP Server receive: " << request; - const auto cont = process_100(request); + const auto cont = Process100(request); if (cont) { return *cont; } @@ -314,7 +310,7 @@ std::string AssertHeader(const HttpRequest& request, std::string_view header) { return TryGetHeader(request, header); } -HttpResponse header_validate_callback(const HttpRequest& request) { +HttpResponse HeaderValidateCallback(const HttpRequest& request) { LOG_INFO() << "HTTP Server receive: " << request; AssertHeader(request, kTestHeader); return { @@ -323,7 +319,7 @@ HttpResponse header_validate_callback(const HttpRequest& request) { HttpResponse::kWriteAndClose}; } -HttpResponse user_agent_validate_callback(const HttpRequest& request) { +HttpResponse UserAgentValidateCallback(const HttpRequest& request) { LOG_INFO() << "HTTP Server receive: " << request; auto header_value = AssertHeader(request, http::headers::kUserAgent); @@ -335,7 +331,7 @@ HttpResponse user_agent_validate_callback(const HttpRequest& request) { HttpResponse::kWriteAndClose}; } -HttpResponse no_user_agent_validate_callback(const HttpRequest& request) { +HttpResponse NoUserAgentValidateCallback(const HttpRequest& request) { LOG_INFO() << "HTTP Server receive: " << request; auto header_value = TryGetHeader(request, http::headers::kUserAgent); EXPECT_EQ(header_value, utils::GetUserverIdentifier()) << "In request: " << request; @@ -507,6 +503,8 @@ std::string DifferentUrlsRetryStreamResponseBody( } UINVARIANT(false, "No alive servers"); + // Never reaches + return {}; } } // namespace sample @@ -597,7 +595,7 @@ UTEST(HttpClient, PostEcho) { UTEST(HttpClient, StatsOnTimeout) { const int kRetries = 5; - const utest::SimpleServer http_server{&sleep_callback}; + const utest::SimpleServer http_server{&SleepCallback}; auto http_client_ptr = utest::CreateHttpClient(); auto request = http_client_ptr->CreateRequest() @@ -675,7 +673,7 @@ UTEST(HttpClient, CancelRetries) { if (server_requests > kMinRetries) { enough_retries_event.Send(); } - return sleep_callback_1s(request); + return SleepCallback1s(request); }; const utest::SimpleServer http_server{callback}; @@ -733,7 +731,7 @@ UTEST(HttpClient, CancelRetries) { } UTEST(HttpClient, PostShutdownWithPendingRequest) { - const utest::SimpleServer http_server{&sleep_callback}; + const utest::SimpleServer http_server{&SleepCallback}; auto http_client_ptr = utest::CreateHttpClient(); for (unsigned i = 0; i < kRepetitions; ++i) @@ -751,7 +749,7 @@ UTEST(HttpClient, PostShutdownWithPendingRequestHuge) { // The test produces too much logs otherwise auto log_level_scope = LogLevelScope(logging::Level::kError); - const utest::SimpleServer http_server{&sleep_callback}; + const utest::SimpleServer http_server{&SleepCallback}; auto http_client_ptr = utest::CreateHttpClient(); std::string request = kTestData; @@ -785,7 +783,7 @@ UTEST(HttpClient, PutEcho) { } UTEST(HttpClient, PutValidateHeader) { - const utest::SimpleServer http_server{&put_validate_callback}; + const utest::SimpleServer http_server{&PutValidateCallback}; auto http_client_ptr = utest::CreateHttpClient(); auto request = http_client_ptr->CreateRequest() @@ -800,7 +798,7 @@ UTEST(HttpClient, PutValidateHeader) { } UTEST(HttpClient, PutShutdownWithPendingRequest) { - const utest::SimpleServer http_server{&sleep_callback}; + const utest::SimpleServer http_server{&SleepCallback}; auto http_client_ptr = utest::CreateHttpClient(); for (unsigned i = 0; i < kRepetitions; ++i) @@ -818,7 +816,7 @@ UTEST(HttpClient, PutShutdownWithPendingRequestHuge) { // The test produces too much logs otherwise auto log_level_scope = LogLevelScope(logging::Level::kError); - const utest::SimpleServer http_server{&sleep_callback}; + const utest::SimpleServer http_server{&SleepCallback}; auto http_client_ptr = utest::CreateHttpClient(); std::string request = kTestData; @@ -838,7 +836,7 @@ UTEST(HttpClient, PutShutdownWithPendingRequestHuge) { } UTEST(HttpClient, PutShutdownWithHugeResponse) { - const utest::SimpleServer http_server{&huge_data_callback}; + const utest::SimpleServer http_server{&HugeDataCallback}; auto http_client_ptr = utest::CreateHttpClient(); for (unsigned i = 0; i < kRepetitions; ++i) @@ -972,7 +970,7 @@ UTEST(HttpClient, MethodsMixReuseRequestData) { } UTEST(HttpClient, Headers) { - const utest::SimpleServer http_server{&header_validate_callback}; + const utest::SimpleServer http_server{&HeaderValidateCallback}; auto http_client_ptr = utest::CreateHttpClient(); clients::http::Headers headers; @@ -994,8 +992,8 @@ UTEST(HttpClient, Headers) { } UTEST(HttpClient, HeadersUserAgent) { - const utest::SimpleServer http_server{&user_agent_validate_callback}; - const utest::SimpleServer http_server_no_ua{&no_user_agent_validate_callback}; + const utest::SimpleServer http_server{&UserAgentValidateCallback}; + const utest::SimpleServer http_server_no_ua{&NoUserAgentValidateCallback}; auto http_client_ptr = utest::CreateHttpClient(); auto request = http_client_ptr->CreateRequest() @@ -1239,7 +1237,7 @@ UTEST(HttpClient, Retry) { UTEST(HttpClient, TinyTimeout) { auto http_client_ptr = utest::CreateHttpClient(); - const utest::SimpleServer http_server{sleep_callback_1s}; + const utest::SimpleServer http_server{SleepCallback1s}; for (unsigned i = 0; i < kRepetitions; ++i) { auto response_future = http_client_ptr->CreateRequest() @@ -1325,7 +1323,7 @@ UTEST(HttpClient, RequestReuseBasic) { UTEST(HttpClient, RequestReuseSample) { const EchoCallback shared_echo_callback{}; const utest::SimpleServer http_server{shared_echo_callback, utest::SimpleServer::kTcpIpV6}; - const utest::SimpleServer http_sleep_server{sleep_callback_1s}; + const utest::SimpleServer http_sleep_server{SleepCallback1s}; std::string data = "Some long long request"; for (unsigned i = 0; i < kFewRepetitions; ++i) { @@ -1368,7 +1366,7 @@ UTEST(HttpClient, RequestReuseSample) { UTEST(HttpClient, DISABLED_RequestReuseSampleStream) { const EchoCallback shared_echo_callback{}; const utest::SimpleServer http_server{shared_echo_callback, utest::SimpleServer::kTcpIpV6}; - const utest::SimpleServer http_sleep_server{sleep_callback_1s}; + const utest::SimpleServer http_sleep_server{SleepCallback1s}; std::string data = "Some long long request"; for (unsigned i = 0; i < kFewRepetitions; ++i) { @@ -1394,7 +1392,7 @@ UTEST(HttpClient, DISABLED_RequestReuseSampleStream) { UTEST(HttpClient, RequestReuseDifferentUrlAndTimeout) { const EchoCallback shared_echo_callback; const utest::SimpleServer http_echo_server{shared_echo_callback, utest::SimpleServer::kTcpIpV6}; - const utest::SimpleServer http_sleep_server{sleep_callback_1s}; + const utest::SimpleServer http_sleep_server{SleepCallback1s}; auto http_client_ptr = utest::CreateHttpClient(); diff --git a/core/src/clients/http/client_wait_test.cpp b/core/src/clients/http/client_wait_test.cpp index 419e3169077d..7edf88311e02 100644 --- a/core/src/clients/http/client_wait_test.cpp +++ b/core/src/clients/http/client_wait_test.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/core/src/clients/http/destination_statistics_test.cpp b/core/src/clients/http/destination_statistics_test.cpp index 5dae97b2370d..464fdd1556a4 100644 --- a/core/src/clients/http/destination_statistics_test.cpp +++ b/core/src/clients/http/destination_statistics_test.cpp @@ -1,7 +1,9 @@ #include #include + #include +#include #include #include diff --git a/core/src/clients/http/form_test.cpp b/core/src/clients/http/form_test.cpp index d440dc8e79e8..b568b1fde97f 100644 --- a/core/src/clients/http/form_test.cpp +++ b/core/src/clients/http/form_test.cpp @@ -47,7 +47,7 @@ bool ReceivedFull(const HttpRequest& request) { return request.find(end_boundary) != std::string::npos; } -void validate_filesend( +void ValidateFilesend( const HttpRequest& request, std::string key, std::string filename, @@ -82,22 +82,22 @@ void validate_filesend( EXPECT_LT(filename_pos, test_data_pos) << "Nested filename appears after test data: " << request; } -HttpResponse validating_callback1(const HttpRequest& request) { +HttpResponse ValidatingCallback1(const HttpRequest& request) { if (!ReceivedFull(request)) { return {{}, HttpResponse::kTryReadMore}; } - validate_filesend(request, kKey, kFileNameTxt, kImageJpeg, kTestData); + ValidateFilesend(request, kKey, kFileNameTxt, kImageJpeg, kTestData); return {kOkCloseResponse, HttpResponse::kWriteAndClose}; } -HttpResponse validating_callback2(const HttpRequest& request) { +HttpResponse ValidatingCallback2(const HttpRequest& request) { if (!ReceivedFull(request)) { return {{}, HttpResponse::kTryReadMore}; } - validate_filesend(request, kKey, kFileNameTxt, kImageJpeg, kTestData); - validate_filesend(request, kKey2, kFileName2Bmp, kImageBmp, kOtherTestData); + ValidateFilesend(request, kKey, kFileNameTxt, kImageJpeg, kTestData); + ValidateFilesend(request, kKey2, kFileName2Bmp, kImageBmp, kOtherTestData); return {kOkCloseResponse, HttpResponse::kWriteAndClose}; } @@ -105,7 +105,7 @@ HttpResponse validating_callback2(const HttpRequest& request) { } // namespace UTEST(CurlFormTest, MultipartFileWithContentType) { - const utest::SimpleServer http_server{&validating_callback1}; + const utest::SimpleServer http_server{&ValidatingCallback1}; auto http_client_ptr = utest::CreateHttpClient(); clients::http::Form form; @@ -123,7 +123,7 @@ UTEST(CurlFormTest, MultipartFileWithContentType) { } UTEST(CurlFormTest, FilesWithContentType) { - const utest::SimpleServer http_server{&validating_callback2}; + const utest::SimpleServer http_server{&ValidatingCallback2}; auto http_client_ptr = utest::CreateHttpClient(); clients::http::Form form; @@ -143,7 +143,7 @@ UTEST(CurlFormTest, FilesWithContentType) { } UTEST(CurlFormTest, FormMovable) { - const utest::SimpleServer http_server{&validating_callback2}; + const utest::SimpleServer http_server{&ValidatingCallback2}; auto http_client_ptr = utest::CreateHttpClient(); diff --git a/core/src/clients/http/plugin.cpp b/core/src/clients/http/plugin.cpp index a50e8a001d06..0923400006b6 100644 --- a/core/src/clients/http/plugin.cpp +++ b/core/src/clients/http/plugin.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -38,12 +40,12 @@ const std::string& Plugin::GetName() const { return name_; } namespace impl { -PluginPipeline::PluginPipeline(const std::vector>& plugins) : plugins_(plugins) {} +PluginPipeline::PluginPipeline(const std::vector>& plugins) : plugins_(&plugins) {} void PluginPipeline::HookCreateSpan(RequestState& request_state, tracing::Span& span) { PluginRequest req(request_state); - for (const auto& plugin : plugins_) { + for (const auto& plugin : *plugins_) { plugin->HookCreateSpan(req, span); } } @@ -52,8 +54,7 @@ void PluginPipeline::HookOnCompleted(RequestState& request_state, Response& resp PluginRequest req(request_state); // NOLINTNEXTLINE(modernize-loop-convert) - for (auto it = plugins_.rbegin(); it != plugins_.rend(); ++it) { - const auto& plugin = *it; + for (const auto& plugin : *plugins_ | boost::adaptors::reversed) { plugin->HookOnCompleted(req, response); } } @@ -62,8 +63,7 @@ void PluginPipeline::HookOnError(RequestState& request_state, std::error_code ec PluginRequest req(request_state); // NOLINTNEXTLINE(modernize-loop-convert) - for (auto it = plugins_.rbegin(); it != plugins_.rend(); ++it) { - const auto& plugin = *it; + for (const auto& plugin : *plugins_ | boost::adaptors::reversed) { plugin->HookOnError(req, ec); } } @@ -71,7 +71,7 @@ void PluginPipeline::HookOnError(RequestState& request_state, std::error_code ec bool PluginPipeline::HookOnRetry(RequestState& request_state) { PluginRequest req(request_state); - for (const auto& plugin : plugins_) { + for (const auto& plugin : *plugins_) { if (!plugin->HookOnRetry(req)) return false; } return true; @@ -80,7 +80,7 @@ bool PluginPipeline::HookOnRetry(RequestState& request_state) { void PluginPipeline::HookPerformRequest(RequestState& request_state) { PluginRequest req(request_state); - for (const auto& plugin : plugins_) { + for (const auto& plugin : *plugins_) { plugin->HookPerformRequest(req); } } diff --git a/core/src/clients/http/request.cpp b/core/src/clients/http/request.cpp index a08be63dd4b4..d3daa9267876 100644 --- a/core/src/clients/http/request.cpp +++ b/core/src/clients/http/request.cpp @@ -207,17 +207,11 @@ Request::Request( RequestStats&& req_stats, const std::shared_ptr& dest_stats, clients::dns::Resolver* resolver, - impl::PluginPipeline& plugin_pipeline, + const std::vector>& plugins, const tracing::TracingManagerBase& tracing_manager ) - : pimpl_(std::make_shared( - std::move(wrapper), - std::move(req_stats), - dest_stats, - resolver, - plugin_pipeline, - tracing_manager - )) { + : pimpl_(std::make_shared< + RequestState>(std::move(wrapper), std::move(req_stats), dest_stats, resolver, plugins, tracing_manager)) { LOG_TRACE() << "Request::Request()"; // default behavior follow redirects and verify ssl pimpl_->follow_redirects(true); @@ -242,7 +236,7 @@ StreamedResponse Request::async_perform_stream_body( } std::shared_ptr Request::perform(utils::impl::SourceLocation location) { - return async_perform(location).Get(); + return async_perform(location).Get(location); } Request& Request::url(std::string url) & { @@ -531,6 +525,11 @@ Request Request::delete_method(std::string url, std::string data) && { return std::move(this->delete_method(std::move(url), std::move(data))); } +Request& Request::SetPluginsList(const std::vector>& plugins) & { + pimpl_->SetPluginsList(plugins); + return *this; +} + Request& Request::SetLoggedUrl(std::string url) & { pimpl_->SetLoggedUrl(std::move(url)); return *this; diff --git a/core/src/clients/http/request_state.cpp b/core/src/clients/http/request_state.cpp index c8370235be7d..d06329c9d7cd 100644 --- a/core/src/clients/http/request_state.cpp +++ b/core/src/clients/http/request_state.cpp @@ -100,7 +100,7 @@ bool IsSetCookie(std::string_view key) { // Not a strict check, but OK for non-header line check bool IsHttpStatusLineStart(const char* ptr, size_t size) { return (size > 5 && memcmp(ptr, "HTTP/", 5) == 0); } -char* rfind_not_space(char* ptr, size_t size) { +char* RfindNotSpace(char* ptr, size_t size) { for (char* p = ptr + size - 1; p >= ptr; --p) { const char c = *p; if (c == '\n' || c == '\r' || c == ' ' || c == '\t') continue; @@ -215,7 +215,7 @@ RequestState::RequestState( RequestStats&& req_stats, const std::shared_ptr& dest_stats, clients::dns::Resolver* resolver, - impl::PluginPipeline& plugin_pipeline, + const std::vector>& plugins, const tracing::TracingManagerBase& tracing_manager ) : easy_(std::move(wrapper)), @@ -227,13 +227,13 @@ RequestState::RequestState( is_cancelled_(false), errorbuffer_(), resolver_{resolver}, - plugin_pipeline_{plugin_pipeline} { + plugin_pipeline_(plugins) { // Libcurl calls sigaction(2) way too frequently unless this option is used. easy().set_no_signal(true); easy().set_error_buffer(errorbuffer_.data()); // define header function - easy().set_header_function(&RequestState::on_header); + easy().set_header_function(&RequestState::OnHeader); easy().set_header_data(this); // set autodecoding @@ -278,7 +278,7 @@ void RequestState::ca(crypto::Certificate cert) { } else { // Legacy non-portable way, broken since 7.87.0 ca_ = std::move(cert); - easy().set_ssl_ctx_function(&RequestState::on_certificate_request); + easy().set_ssl_ctx_function(&RequestState::OnCertificateRequest); easy().set_ssl_ctx_data(this); } } @@ -326,7 +326,7 @@ void RequestState::client_key_cert(crypto::PrivateKey pkey, crypto::Certificate cert_id.resize(kCertIdLength, '='); easy().set_egd_socket(cert_id); - easy().set_ssl_ctx_function(&RequestState::on_certificate_request); + easy().set_ssl_ctx_function(&RequestState::OnCertificateRequest); easy().set_ssl_ctx_data(this); } } @@ -401,14 +401,14 @@ void RequestState::SetDeadlinePropagationConfig(const DeadlinePropagationConfig& deadline_propagation_config_ = deadline_propagation_config; } -size_t RequestState::on_header(void* ptr, size_t size, size_t nmemb, void* userdata) { +size_t RequestState::OnHeader(void* ptr, size_t size, size_t nmemb, void* userdata) { auto* self = static_cast(userdata); const std::size_t data_size = size * nmemb; - if (self) self->parse_header(static_cast(ptr), data_size); + if (self) self->ParseHeader(static_cast(ptr), data_size); return data_size; } -curl::native::CURLcode RequestState::on_certificate_request(void* /*curl*/, void* sslctx, void* userdata) noexcept { +curl::native::CURLcode RequestState::OnCertificateRequest(void* /*curl*/, void* sslctx, void* userdata) noexcept { auto* ssl = static_cast(sslctx); auto* self = static_cast(userdata); @@ -441,7 +441,7 @@ curl::native::CURLcode RequestState::on_certificate_request(void* /*curl*/, void return curl::native::CURLcode::CURLE_OK; } -void RequestState::on_completed(std::shared_ptr holder, std::error_code err) { +void RequestState::OnCompleted(std::shared_ptr holder, std::error_code err) { UASSERT(holder); UASSERT(holder->span_storage_); auto& span = holder->span_storage_->Get(); @@ -493,7 +493,7 @@ void RequestState::on_completed(std::shared_ptr holder, std::error const utils::Overloaded visitor{ [&holder, &err](FullBufferedData& buffered_data) { { [[maybe_unused]] const auto cleanup = holder->response_move(); } - auto promise = std::move(buffered_data.promise_); + auto promise = std::move(buffered_data.promise); // The task will wake up and may reuse RequestState. promise.set_exception(holder->PrepareException(err)); }, @@ -516,7 +516,7 @@ void RequestState::on_completed(std::shared_ptr holder, std::error const utils::Overloaded visitor{ [&holder](FullBufferedData& buffered_data) { - auto promise = std::move(buffered_data.promise_); + auto promise = std::move(buffered_data.promise); // The task will wake up and may reuse RequestState. promise.set_value(holder->response_move()); }, @@ -530,7 +530,7 @@ void RequestState::on_completed(std::shared_ptr holder, std::error // it is unsafe to touch any content of holder after this point! } -void RequestState::on_retry(std::shared_ptr holder, std::error_code err) { +void RequestState::OnRetry(std::shared_ptr holder, std::error_code err) { UASSERT(holder); UASSERT(holder->span_storage_); LOG_TRACE() << "RequestImpl::on_retry" << tracing::impl::LogSpanAsLastNoCurrent{holder->span_storage_->Get()}; @@ -550,7 +550,7 @@ void RequestState::on_retry(std::shared_ptr holder, std::error_cod if (not_need_retry) { // finish if no need to retry - RequestState::on_completed(std::move(holder), err); + RequestState::OnCompleted(std::move(holder), err); } else { // calculate backoff before retry const auto eb_power = std::clamp(holder->retry_.current - 1, 0, kEBMaxPower); @@ -559,7 +559,7 @@ void RequestState::on_retry(std::shared_ptr holder, std::error_cod holder->UpdateTimeoutFromDeadline(backoff); if (holder->remote_timeout_ <= std::chrono::milliseconds::zero()) { holder->deadline_expired_ = true; - RequestState::on_completed(std::move(holder), err); + RequestState::OnCompleted(std::move(holder), err); return; } @@ -574,19 +574,19 @@ void RequestState::on_retry(std::shared_ptr holder, std::error_cod // call on_retry_timer on timer auto& holder_ref = *holder; holder_ref.retry_.timer->SingleshotAsync(backoff, [holder = std::move(holder)](std::error_code err) { - holder->on_retry_timer(err); + holder->OnRetryTimer(err); }); } } -void RequestState::on_retry_timer(std::error_code err) { +void RequestState::OnRetryTimer(std::error_code err) { // if there is no error with timer call perform, otherwise finish if (!err) - perform_request([holder = shared_from_this()](std::error_code err) mutable { - RequestState::on_retry(std::move(holder), err); + PerformRequest([holder = shared_from_this()](std::error_code err) mutable { + RequestState::OnRetry(std::move(holder), err); }); else - on_completed(shared_from_this(), err); + OnCompleted(shared_from_this(), err); } void RequestState::ParseSingleCookie(const char* ptr, size_t size) { @@ -598,11 +598,11 @@ void RequestState::ParseSingleCookie(const char* ptr, size_t size) { } } -void RequestState::parse_header(char* ptr, size_t size) try { +void RequestState::ParseHeader(char* ptr, size_t size) try { /* It is a fast path in curl's thread (io thread). Creation of tmp * std::string, boost::trim_right_if(), etc. is too expensive. */ - auto* end = rfind_not_space(ptr, size); + auto* end = RfindNotSpace(ptr, size); if (ptr == end) { const auto status_code = static_cast(easy().get_response_code()); response()->SetStatusCode(status_code); @@ -644,6 +644,10 @@ void RequestState::parse_header(char* ptr, size_t size) try { LOG_ERROR() << "Failed to parse header: " << e.what(); } +void RequestState::SetPluginsList(const std::vector>& plugins) { + plugin_pipeline_ = impl::PluginPipeline(plugins); +} + void RequestState::SetLoggedUrl(std::string url) { log_url_ = std::move(url); } const std::string& RequestState::GetLoggedOriginalUrl() const noexcept { @@ -669,11 +673,11 @@ engine::Future> RequestState::async_perform(utils::imp // set place for response body easy().set_sink(&response_->sink_string()); - auto future = std::get_if(&data_)->promise_.get_future(); + auto future = std::get_if(&data_)->promise.get_future(); if (UpdateTimeoutFromDeadlineAndCheck()) { - perform_request([holder = shared_from_this()](std::error_code err) mutable { - RequestState::on_retry(std::move(holder), err); + PerformRequest([holder = shared_from_this()](std::error_code err) mutable { + RequestState::OnRetry(std::move(holder), err); }); } @@ -698,15 +702,15 @@ RequestState::async_perform_stream(const std::shared_ptr& queue, utils::i auto future = std::get_if(&data_)->headers_promise.get_future(); if (UpdateTimeoutFromDeadlineAndCheck()) { - perform_request([holder = shared_from_this()](std::error_code err) mutable { - RequestState::on_completed(std::move(holder), err); + PerformRequest([holder = shared_from_this()](std::error_code err) mutable { + RequestState::OnCompleted(std::move(holder), err); }); } return future; } -void RequestState::perform_request(curl::easy::handler_type handler) { +void RequestState::PerformRequest(curl::easy::handler_type handler) { UASSERT_MSG(!cert_ || pkey_, "Setting certificate is useless without setting private key"); UASSERT(response_); @@ -727,12 +731,12 @@ void RequestState::perform_request(curl::easy::handler_type handler) { // TODO: should retry - TAXICOMMON-4932 auto* buffered_data = std::get_if(&data_); if (buffered_data) { - buffered_data->promise_.set_exception(std::current_exception()); + buffered_data->promise.set_exception(std::current_exception()); } } catch (const BaseException& ex) { auto* buffered_data = std::get_if(&data_); if (buffered_data) { - buffered_data->promise_.set_exception(std::current_exception()); + buffered_data->promise.set_exception(std::current_exception()); } } }) @@ -804,7 +808,7 @@ void RequestState::HandleDeadlineAlreadyPassed() { const utils::Overloaded visitor{ [&exc](FullBufferedData& buffered_data) { - auto promise = std::move(buffered_data.promise_); + auto promise = std::move(buffered_data.promise); // The task will wake up and may reuse RequestState. promise.set_exception(std::move(exc)); }, diff --git a/core/src/clients/http/request_state.hpp b/core/src/clients/http/request_state.hpp index 1b9c4a97ec3c..264f254e6220 100644 --- a/core/src/clients/http/request_state.hpp +++ b/core/src/clients/http/request_state.hpp @@ -47,7 +47,7 @@ class RequestState : public std::enable_shared_from_this { RequestStats&& req_stats, const std::shared_ptr& dest_stats, clients::dns::Resolver* resolver, - impl::PluginPipeline& plugin_pipeline, + const std::vector>& plugins, const tracing::TracingManagerBase& tracing_manager ); ~RequestState(); @@ -134,6 +134,7 @@ class RequestState : public std::enable_shared_from_this { std::shared_ptr response() const { return response_; } std::shared_ptr response_move() { return std::move(response_); } + void SetPluginsList(const std::vector>& plugins); void SetLoggedUrl(std::string url); void SetEasyTimeout(std::chrono::milliseconds timeout); @@ -143,22 +144,22 @@ class RequestState : public std::enable_shared_from_this { private: /// final callback that calls user callback and set value in promise - static void on_completed(std::shared_ptr, std::error_code err); + static void OnCompleted(std::shared_ptr, std::error_code err); /// retry callback - static void on_retry(std::shared_ptr, std::error_code err); + static void OnRetry(std::shared_ptr, std::error_code err); /// header function curl callback - static size_t on_header(void* ptr, size_t size, size_t nmemb, void* userdata); + static size_t OnHeader(void* ptr, size_t size, size_t nmemb, void* userdata); /// certificate function curl callback - static curl::native::CURLcode on_certificate_request(void* curl, void* sslctx, void* userdata) noexcept; + static curl::native::CURLcode OnCertificateRequest(void* curl, void* sslctx, void* userdata) noexcept; /// parse one header - void parse_header(char* ptr, size_t size); + void ParseHeader(char* ptr, size_t size); void ParseSingleCookie(const char* ptr, size_t size); /// simply run perform_request if there is now errors from timer - void on_retry_timer(std::error_code err); + void OnRetryTimer(std::error_code err); /// run curl async_request, called once per attempt - void perform_request(curl::easy::handler_type handler); + void PerformRequest(curl::easy::handler_type handler); void UpdateTimeoutFromDeadline(std::chrono::milliseconds backoff); [[nodiscard]] bool UpdateTimeoutFromDeadlineAndCheck(std::chrono::milliseconds backoff = {}); @@ -237,7 +238,7 @@ class RequestState : public std::enable_shared_from_this { clients::dns::Resolver* resolver_{nullptr}; std::string proxy_url_; - impl::PluginPipeline& plugin_pipeline_; + impl::PluginPipeline plugin_pipeline_; struct StreamData { StreamData(Queue::Producer&& queue_producer) : queue_producer(std::move(queue_producer)) {} @@ -248,7 +249,7 @@ class RequestState : public std::enable_shared_from_this { }; struct FullBufferedData { - engine::Promise> promise_; + engine::Promise> promise; }; std::variant data_; diff --git a/core/src/clients/http/response_future.cpp b/core/src/clients/http/response_future.cpp index ddefd9aa05b8..61a98a3a4a30 100644 --- a/core/src/clients/http/response_future.cpp +++ b/core/src/clients/http/response_future.cpp @@ -5,6 +5,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -94,7 +95,9 @@ void ResponseFuture::Detach() { request_state_.reset(); } -std::future_status ResponseFuture::Wait() { +std::future_status ResponseFuture::Wait(utils::impl::SourceLocation location) { + utils::trx_tracker::CheckNoTransactions(location); + switch (future_.wait_until(deadline_)) { case engine::FutureStatus::kCancelled: { const auto stats = request_state_->easy().get_local_stats(); @@ -122,8 +125,8 @@ std::future_status ResponseFuture::Wait() { UINVARIANT(false, "Invalid engine::FutureStatus"); } -std::shared_ptr ResponseFuture::Get() { - const auto future_status = Wait(); +std::shared_ptr ResponseFuture::Get(utils::impl::SourceLocation location) { + const auto future_status = Wait(location); if (future_status == std::future_status::ready) { if (request_state_->IsDeadlineExpired()) { server::request::MarkTaskInheritedDeadlineExpired(); diff --git a/core/src/components/manager.cpp b/core/src/components/manager.cpp index b575b3b73be2..2a717591c0b4 100644 --- a/core/src/components/manager.cpp +++ b/core/src/components/manager.cpp @@ -1,11 +1,9 @@ #include #include -#include #include #include #include -#include #include #include @@ -23,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -143,12 +142,19 @@ void Manager::TaskProcessorsStorage::WaitForAllTasksBlocking() const noexcept { } } -Manager::Manager(std::unique_ptr&& config, const ComponentList& component_list) +Manager::Manager( + std::unique_ptr&& config, + std::chrono::steady_clock::time_point start_time, + const ComponentList& component_list +) : config_(std::move(config)), task_processors_storage_( std::make_shared(config_->coro_pool, config_->event_thread_pool) ), - start_time_(std::chrono::steady_clock::now()) { + start_time_(start_time), + pre_load_duration_( + std::chrono::duration_cast(std::chrono::steady_clock::now() - start_time_) + ) { LOG_INFO() << "Starting components manager"; for (auto processor_config : config_->task_processors) { @@ -260,6 +266,8 @@ std::chrono::steady_clock::time_point Manager::GetStartTime() const { return sta std::chrono::milliseconds Manager::GetLoadDuration() const { return load_duration_; } +std::chrono::milliseconds Manager::GetPreLoadDuration() const { return pre_load_duration_; } + void Manager::CreateComponentContext(const ComponentList& component_list) { std::set loading_component_names; for (const auto& adder : component_list) { diff --git a/core/src/components/manager.hpp b/core/src/components/manager.hpp index aae2c96dccf1..5b2f3f2d9820 100644 --- a/core/src/components/manager.hpp +++ b/core/src/components/manager.hpp @@ -38,7 +38,11 @@ using TaskProcessorsMap = utils::impl::TransparentMap&& config, const ComponentList& component_list); + Manager( + std::unique_ptr&& config, + std::chrono::steady_clock::time_point start_time, + const ComponentList& component_list + ); ~Manager(); const ManagerConfig& GetConfig() const; @@ -50,6 +54,8 @@ class Manager final { std::chrono::steady_clock::time_point GetStartTime() const; + std::chrono::milliseconds GetPreLoadDuration() const; + std::chrono::milliseconds GetLoadDuration() const; private: @@ -96,6 +102,7 @@ class Manager final { engine::TaskProcessor* default_task_processor_{nullptr}; const std::chrono::steady_clock::time_point start_time_; + const std::chrono::milliseconds pre_load_duration_{0}; std::chrono::milliseconds load_duration_{0}; os_signals::ProcessorComponent* signal_processor_{nullptr}; diff --git a/core/src/components/manager_config.cpp b/core/src/components/manager_config.cpp index 428dd5206b33..7378b2aad5bc 100644 --- a/core/src/components/manager_config.cpp +++ b/core/src/components/manager_config.cpp @@ -237,6 +237,12 @@ additionalProperties: false the balancer a chance to redirect new requests to other hosts and to give the service a chance to finish handling old requests. defaultDescription: 0s + enable_trx_tracker: + type: boolean + description: | + Enable checking of heavy operations (like http calls) while having + active database transactions. + defaultDescription: true )"); } @@ -270,6 +276,7 @@ ManagerConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To(config.graceful_shutdown_interval); + config.enable_trx_tracker = value["enable_trx_tracker"].As(config.enable_trx_tracker); return config; } diff --git a/core/src/components/manager_config.hpp b/core/src/components/manager_config.hpp index 2a30fc8d3cec..b69fded75f15 100644 --- a/core/src/components/manager_config.hpp +++ b/core/src/components/manager_config.hpp @@ -30,6 +30,7 @@ struct ManagerConfig { bool mlock_debug_info{true}; bool disable_phdr_cache{false}; bool preheat_stacktrace_collector{true}; + bool enable_trx_tracker{true}; static ManagerConfig FromString( const std::string&, diff --git a/core/src/components/manager_controller_component.cpp b/core/src/components/manager_controller_component.cpp index cc96eb213ed8..265779a41d94 100644 --- a/core/src/components/manager_controller_component.cpp +++ b/core/src/components/manager_controller_component.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -117,6 +118,11 @@ void ManagerControllerComponent::WriteStatistics(utils::statistics::Writer& writ .count(); writer["load-ms"] = std::chrono::duration_cast(components_manager_.GetLoadDuration()).count(); + + writer["pre-load-ms"] = + std::chrono::duration_cast(components_manager_.GetPreLoadDuration()).count(); + + writer["heavy-operations-in-transactions"] = utils::trx_tracker::GetStatistics().triggers; } void ManagerControllerComponent::OnConfigUpdate(const dynamic_config::Snapshot& cfg) { diff --git a/core/src/components/run.cpp b/core/src/components/run.cpp index efd55cd03948..4d9ce5bf5f7e 100644 --- a/core/src/components/run.cpp +++ b/core/src/components/run.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -28,6 +27,7 @@ #include #include #include +#include #include #include @@ -186,6 +186,8 @@ void DoRun( const ComponentList& component_list, RunMode run_mode ) { + const auto start_time = std::chrono::steady_clock::now(); + utils::impl::FinishStaticRegistration(); utils::SignalCatcher signal_catcher{SIGINT, SIGTERM, SIGQUIT, SIGUSR1, SIGUSR2}; @@ -201,6 +203,8 @@ void DoRun( utils::impl::UserverExperimentsScope experiments_scope; std::optional manager; + const utils::trx_tracker::impl::GlobalEnabler enabler{manager_config.enable_trx_tracker}; + try { experiments_scope.EnableOnly(manager_config.enabled_experiments); @@ -209,7 +213,7 @@ void DoRun( PreheatStacktraceCollector(); } - manager.emplace(std::make_unique(std::move(manager_config)), component_list); + manager.emplace(std::make_unique(std::move(manager_config)), start_time, component_list); } catch (const std::exception& ex) { LOG_ERROR() << "Loading failed: " << ex; throw; diff --git a/core/src/concurrent/background_task_storage_benchmark.cpp b/core/src/concurrent/background_task_storage_benchmark.cpp index ed042d6b1672..5344265ea08d 100644 --- a/core/src/concurrent/background_task_storage_benchmark.cpp +++ b/core/src/concurrent/background_task_storage_benchmark.cpp @@ -8,7 +8,7 @@ USERVER_NAMESPACE_BEGIN -void background_task_storage(benchmark::State& state) { +void BackgroundTaskStorage(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { concurrent::BackgroundTaskStorageCore bts; @@ -21,6 +21,6 @@ void background_task_storage(benchmark::State& state) { }); }); } -BENCHMARK(background_task_storage)->Arg(2)->Arg(4)->Arg(6)->Arg(8)->Arg(12)->Arg(16)->Arg(32); +BENCHMARK(BackgroundTaskStorage)->Arg(2)->Arg(4)->Arg(6)->Arg(8)->Arg(12)->Arg(16)->Arg(32); USERVER_NAMESPACE_END diff --git a/core/src/concurrent/impl/striped_intrusive_pool.hpp b/core/src/concurrent/impl/striped_intrusive_pool.hpp index 89d90f6b6275..54d5918efc75 100644 --- a/core/src/concurrent/impl/striped_intrusive_pool.hpp +++ b/core/src/concurrent/impl/striped_intrusive_pool.hpp @@ -28,7 +28,7 @@ namespace concurrent::impl { /// `HookExtractor::GetHook` static function should get the hook subobject, given `T` node. /// The hook's type must be `SinglyLinkedHook`. /// Additionally, `kHookOffset` must be `offsetof` the hook subobject within `T`. -template +template class StripedIntrusivePool final { public: StripedIntrusivePool() = default; @@ -51,7 +51,7 @@ class StripedIntrusivePool final { const auto newval = reinterpret_cast(&node); UASSERT_MSG( // Unfortunately, there is no legal way to check this at compile-time. - reinterpret_cast(&GetNext(node)) - reinterpret_cast(&node) == kHookOffset, + reinterpret_cast(&GetNext(node)) - reinterpret_cast(&node) == HookOffset, "kHookOffset is invalid" ); GetNext(node).store(reinterpret_cast(expect), std::memory_order_relaxed); @@ -83,7 +83,7 @@ class StripedIntrusivePool final { const auto expectnot = reinterpret_cast(static_cast(nullptr)); const int ret = rseq_load_cbeq_store_add_load_store__ptr( - RSEQ_MO_RELAXED, RSEQ_PERCPU_CPU_ID, &slot, expectnot, kHookOffset, &head, cpu + RSEQ_MO_RELAXED, RSEQ_PERCPU_CPU_ID, &slot, expectnot, HookOffset, &head, cpu ); if (rseq_likely(!ret)) { diff --git a/core/src/concurrent/intrusive_walkable_pool.hpp b/core/src/concurrent/intrusive_walkable_pool.hpp index 82449155cf17..1697870c5128 100644 --- a/core/src/concurrent/intrusive_walkable_pool.hpp +++ b/core/src/concurrent/intrusive_walkable_pool.hpp @@ -20,7 +20,7 @@ namespace concurrent::impl { /// - The element type `T` must include `IntrusiveWalkablePoolHook` with `offsetof == kHookOffset` /// - The nodes are only destroyed on the pool destruction, so the node count is /// monotonically non-decreasing -template +template class IntrusiveWalkablePool final { public: IntrusiveWalkablePool() = default; @@ -76,7 +76,7 @@ class IntrusiveWalkablePool final { using FreeListHookExtractor = CombinedHook::free_list_hook>>; static constexpr std::ptrdiff_t kFreeListHookOffset = - kHookOffset + offsetof(IntrusiveWalkablePoolHook, free_list_hook); + HookOffset + offsetof(IntrusiveWalkablePoolHook, free_list_hook); IntrusiveStack permanent_list_; StripedIntrusivePool free_list_{}; diff --git a/core/src/concurrent/intrusive_walkable_pool_benchmark.cpp b/core/src/concurrent/intrusive_walkable_pool_benchmark.cpp index ac530d9209de..4e234c26f317 100644 --- a/core/src/concurrent/intrusive_walkable_pool_benchmark.cpp +++ b/core/src/concurrent/intrusive_walkable_pool_benchmark.cpp @@ -19,7 +19,7 @@ struct IntNode final { } // namespace -void intrusive_walkable_pool(benchmark::State& state) { +void IntrusiveWalkablePool(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { concurrent::impl::IntrusiveWalkablePool< // IntNode, @@ -36,6 +36,6 @@ void intrusive_walkable_pool(benchmark::State& state) { }); }); } -BENCHMARK(intrusive_walkable_pool)->Arg(1)->Arg(2)->Arg(4)->Arg(6)->Arg(8)->Arg(12)->Arg(16)->Arg(32); +BENCHMARK(IntrusiveWalkablePool)->Arg(1)->Arg(2)->Arg(4)->Arg(6)->Arg(8)->Arg(12)->Arg(16)->Arg(32); USERVER_NAMESPACE_END diff --git a/core/src/concurrent/mutex_set_benchmark.cpp b/core/src/concurrent/mutex_set_benchmark.cpp index a8ce7ff1d7fa..500ca05c61ad 100644 --- a/core/src/concurrent/mutex_set_benchmark.cpp +++ b/core/src/concurrent/mutex_set_benchmark.cpp @@ -27,7 +27,7 @@ T GetKeyForBenchmark(std::size_t i) { } template -void mutex_set_lock_unlock_no_contention(benchmark::State& state) { +void MutexSetLockUnlockNoContention(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { concurrent::MutexSet ms; @@ -64,11 +64,11 @@ void mutex_set_lock_unlock_no_contention(benchmark::State& state) { }); } -BENCHMARK_TEMPLATE(mutex_set_lock_unlock_no_contention, int)->RangeMultiplier(2)->Range(1, 8); -BENCHMARK_TEMPLATE(mutex_set_lock_unlock_no_contention, std::string)->RangeMultiplier(2)->Range(1, 8); +BENCHMARK_TEMPLATE(MutexSetLockUnlockNoContention, int)->RangeMultiplier(2)->Range(1, 8); +BENCHMARK_TEMPLATE(MutexSetLockUnlockNoContention, std::string)->RangeMultiplier(2)->Range(1, 8); template -void mutex_set_lock_unlock_contention(benchmark::State& state) { +void MutexSetLockUnlockContention(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { concurrent::MutexSet ms; @@ -126,11 +126,11 @@ void mutex_set_lock_unlock_contention(benchmark::State& state) { }); } -BENCHMARK_TEMPLATE(mutex_set_lock_unlock_contention, int)->RangeMultiplier(2)->Range(1, 8); -BENCHMARK_TEMPLATE(mutex_set_lock_unlock_contention, std::string)->RangeMultiplier(2)->Range(1, 8); +BENCHMARK_TEMPLATE(MutexSetLockUnlockContention, int)->RangeMultiplier(2)->Range(1, 8); +BENCHMARK_TEMPLATE(MutexSetLockUnlockContention, std::string)->RangeMultiplier(2)->Range(1, 8); template -void mutex_set_8ways_lock_unlock_contention(benchmark::State& state) { +void MutexSet8waysLockUnlockContention(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { constexpr std::size_t kKeysCount = 128; concurrent::MutexSet ms(8); @@ -171,8 +171,8 @@ void mutex_set_8ways_lock_unlock_contention(benchmark::State& state) { }); } -BENCHMARK_TEMPLATE(mutex_set_8ways_lock_unlock_contention, int)->RangeMultiplier(2)->Range(1, 8); -BENCHMARK_TEMPLATE(mutex_set_8ways_lock_unlock_contention, std::string)->RangeMultiplier(2)->Range(1, 8); +BENCHMARK_TEMPLATE(MutexSet8waysLockUnlockContention, int)->RangeMultiplier(2)->Range(1, 8); +BENCHMARK_TEMPLATE(MutexSet8waysLockUnlockContention, std::string)->RangeMultiplier(2)->Range(1, 8); } // namespace diff --git a/core/src/congestion_control/component.cpp b/core/src/congestion_control/component.cpp index 1edad7334d10..3aa74206e21a 100644 --- a/core/src/congestion_control/component.cpp +++ b/core/src/congestion_control/component.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -168,6 +169,8 @@ server::congestion_control::Limiter& Component::GetServerLimiter() { return pimp server::congestion_control::Sensor& Component::GetServerSensor() { return pimpl_->server_sensor; } +const congestion_control::Controller& Component::GetServerController() const { return pimpl_->server_controller; } + yaml_config::Schema Component::GetStaticConfigSchema() { return yaml_config::MergeSchemas(R"( type: object diff --git a/core/src/congestion_control/controllers/linear.cpp b/core/src/congestion_control/controllers/linear.cpp index 986e56d92452..85c41c473875 100644 --- a/core/src/congestion_control/controllers/linear.cpp +++ b/core/src/congestion_control/controllers/linear.cpp @@ -1,5 +1,6 @@ #include +#include #include USERVER_NAMESPACE_BEGIN diff --git a/core/src/congestion_control/controllers/v2.cpp b/core/src/congestion_control/controllers/v2.cpp index 8c0262177ac9..729308d129fd 100644 --- a/core/src/congestion_control/controllers/v2.cpp +++ b/core/src/congestion_control/controllers/v2.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include diff --git a/core/src/curl-ev/easy.hpp b/core/src/curl-ev/easy.hpp index 1ee4672a5c5c..3961da943bf4 100644 --- a/core/src/curl-ev/easy.hpp +++ b/core/src/curl-ev/easy.hpp @@ -6,6 +6,8 @@ C++ wrapper for libcurl's easy interface */ +// NOLINTBEGIN(readability-identifier-naming) + #pragma once #include @@ -748,3 +750,5 @@ class easy final : public std::enable_shared_from_this { #undef IMPLEMENT_CURL_OPTION_GET_LIST USERVER_NAMESPACE_END + +// NOLINTEND(readability-identifier-naming) diff --git a/core/src/curl-ev/error_code.hpp b/core/src/curl-ev/error_code.hpp index 65660baa69de..a7105b0b6904 100644 --- a/core/src/curl-ev/error_code.hpp +++ b/core/src/curl-ev/error_code.hpp @@ -203,6 +203,8 @@ USERVER_NAMESPACE_BEGIN namespace curl::errc { +// NOLINTBEGIN(readability-identifier-naming) + inline std::error_code make_error_code(EasyErrorCode e) { return {static_cast(e), GetEasyCategory()}; } inline std::error_code make_error_code(MultiErrorCode e) { return {static_cast(e), GetMultiCategory()}; } @@ -215,6 +217,8 @@ inline std::error_code make_error_code(UrlErrorCode e) { return {static_cast(e), GetRateLimitCategory()}; } +// NOLINTEND(readability-identifier-naming) + } // namespace curl::errc USERVER_NAMESPACE_END diff --git a/core/src/curl-ev/form.hpp b/core/src/curl-ev/form.hpp index d5d0a21b673d..f0f9b147f6dc 100644 --- a/core/src/curl-ev/form.hpp +++ b/core/src/curl-ev/form.hpp @@ -6,6 +6,8 @@ C++ wrapper for constructing libcurl forms */ +// NOLINTBEGIN(readability-identifier-naming) + #pragma once #include @@ -20,7 +22,7 @@ USERVER_NAMESPACE_BEGIN namespace curl { -class form { +class form { // NOLINT(readability-identifier-naming) public: form(); form(const form&) = delete; @@ -75,3 +77,5 @@ class form { } // namespace curl USERVER_NAMESPACE_END + +// NOLINTEND(readability-identifier-naming) diff --git a/core/src/curl-ev/multi.cpp b/core/src/curl-ev/multi.cpp index 241a20953c30..74b8f62eafd1 100644 --- a/core/src/curl-ev/multi.cpp +++ b/core/src/curl-ev/multi.cpp @@ -6,6 +6,8 @@ Integration of libcurl's multi interface with Boost.Asio */ +// NOLINTBEGIN(readability-identifier-naming) + #include #include #include @@ -388,3 +390,5 @@ void multi::handle_async() { } // namespace curl USERVER_NAMESPACE_END + +// NOLINTEND(readability-identifier-naming) diff --git a/core/src/curl-ev/multi.hpp b/core/src/curl-ev/multi.hpp index 1279a6dfc14e..95a69ef0a4cc 100644 --- a/core/src/curl-ev/multi.hpp +++ b/core/src/curl-ev/multi.hpp @@ -6,6 +6,8 @@ Integration of libcurl's multi interface with Boost.Asio */ +// NOLINTBEGIN(readability-identifier-naming) + #pragma once #include @@ -32,7 +34,7 @@ namespace curl { class easy; struct socket_info; -class multi final { +class multi final { // NOLINT(readability-identifier-naming) public: using Callback = std::function; multi(engine::ev::ThreadControl& thread_control, const std::shared_ptr& connect_rate_limiter); @@ -102,3 +104,5 @@ class multi final { } // namespace curl USERVER_NAMESPACE_END + +// NOLINTEND(readability-identifier-naming) diff --git a/core/src/curl-ev/native.hpp b/core/src/curl-ev/native.hpp index 3075a9be861f..126e04bdfc2d 100644 --- a/core/src/curl-ev/native.hpp +++ b/core/src/curl-ev/native.hpp @@ -41,6 +41,7 @@ namespace curl::native { } // namespace curl::native +// NOLINTNEXTLINE(readability-identifier-naming) inline void throw_error(std::error_code ec, const char* s) { if (ec) throw std::system_error(ec, s); } diff --git a/core/src/curl-ev/share.hpp b/core/src/curl-ev/share.hpp index 1a49c523043c..eedf42372667 100644 --- a/core/src/curl-ev/share.hpp +++ b/core/src/curl-ev/share.hpp @@ -6,6 +6,8 @@ C++ wrapper for libcurl's share interface */ +// NOLINTBEGIN(readability-identifier-naming) + #pragma once #include @@ -16,7 +18,7 @@ USERVER_NAMESPACE_BEGIN namespace curl { -class share final : public std::enable_shared_from_this { +class share final : public std::enable_shared_from_this { // NOLINT(readability-identifier-naming) public: share(); share(const share&) = delete; @@ -46,3 +48,5 @@ class share final : public std::enable_shared_from_this { } // namespace curl USERVER_NAMESPACE_END + +// NOLINTEND(readability-identifier-naming) diff --git a/core/src/curl-ev/socket_info.hpp b/core/src/curl-ev/socket_info.hpp index 2ef3164deaab..d61865009727 100644 --- a/core/src/curl-ev/socket_info.hpp +++ b/core/src/curl-ev/socket_info.hpp @@ -16,7 +16,7 @@ namespace curl { class easy; -struct socket_info { +struct socket_info { // NOLINT(readability-identifier-naming) explicit socket_info(engine::ev::ThreadControl& thread_control) : watcher(thread_control) {} ~socket_info() { UASSERT(!handle); } diff --git a/core/src/curl-ev/string_list.hpp b/core/src/curl-ev/string_list.hpp index 7d57c3636f9f..52a1e4ea23f7 100644 --- a/core/src/curl-ev/string_list.hpp +++ b/core/src/curl-ev/string_list.hpp @@ -19,7 +19,7 @@ USERVER_NAMESPACE_BEGIN namespace curl { -class string_list { +class string_list { // NOLINT(readability-identifier-naming) public: string_list() = default; string_list(const string_list&) = delete; diff --git a/core/src/curl-ev/url.hpp b/core/src/curl-ev/url.hpp index 3757cfebd706..dfd56d300899 100644 --- a/core/src/curl-ev/url.hpp +++ b/core/src/curl-ev/url.hpp @@ -46,7 +46,7 @@ USERVER_NAMESPACE_BEGIN namespace curl { -class url { +class url { // NOLINT(readability-identifier-naming) public: url(); diff --git a/core/src/dump/operations_encrypted.cpp b/core/src/dump/operations_encrypted.cpp index 2cbe32f4d509..b5d3d8df56d3 100644 --- a/core/src/dump/operations_encrypted.cpp +++ b/core/src/dump/operations_encrypted.cpp @@ -30,10 +30,10 @@ struct EncryptedWriter::Impl { std::string filename; Encryption encryption; std::unique_ptr<::CryptoPP::AuthenticatedEncryptionFilter> filter; - utils::StreamingCpuRelax cpu_relax_; + utils::StreamingCpuRelax cpu_relax; Impl(std::string&& filename, tracing::ScopeTime* scope) - : filename(std::move(filename)), cpu_relax_(kCheckTimeAfterBytes, scope) {} + : filename(std::move(filename)), cpu_relax(kCheckTimeAfterBytes, scope) {} std::string GetTempFilename() const { return filename + ".tmp"; } }; @@ -75,7 +75,7 @@ EncryptedWriter::~EncryptedWriter() = default; void EncryptedWriter::WriteRaw(std::string_view data) { // 2. Data impl_->filter->Put(reinterpret_cast(data.data()), data.size()); - impl_->cpu_relax_.Relax(data.size()); + impl_->cpu_relax.Relax(data.size()); } void EncryptedWriter::Finish() { diff --git a/core/src/dynamic_config/client/component.cpp b/core/src/dynamic_config/client/component.cpp index a1a075508a01..c10a37744648 100644 --- a/core/src/dynamic_config/client/component.cpp +++ b/core/src/dynamic_config/client/component.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN diff --git a/core/src/dynamic_config/updates_sink/component.cpp b/core/src/dynamic_config/updates_sink/component.cpp index 8b552c727f31..50ecd1a3b433 100644 --- a/core/src/dynamic_config/updates_sink/component.cpp +++ b/core/src/dynamic_config/updates_sink/component.cpp @@ -13,7 +13,7 @@ USERVER_NAMESPACE_BEGIN namespace components { struct DynamicConfigUpdatesSinkBase::UsedByInfo { - concurrent::Variable component_name_; + concurrent::Variable component_name; }; DynamicConfigUpdatesSinkBase::DynamicConfigUpdatesSinkBase( @@ -38,7 +38,7 @@ void RegisterUpdater( std::string_view sink_component_name, std::string_view updater_component_name ) { - auto locked_ptr = sink.used_by_->component_name_.Lock(); + auto locked_ptr = sink.used_by_->component_name.Lock(); auto& component_name = *locked_ptr; if (component_name.empty()) { diff --git a/core/src/engine/coro/marked_allocator.cpp b/core/src/engine/coro/marked_allocator.cpp index 12fc367f2ab0..f1a0a0974eaa 100644 --- a/core/src/engine/coro/marked_allocator.cpp +++ b/core/src/engine/coro/marked_allocator.cpp @@ -8,8 +8,8 @@ static volatile const std::size_t page_size = MarkedAllocator::traits_type::page static volatile std::size_t allocator_stack_size = MarkedAllocator::traits_type::default_size(); MarkedAllocator::MarkedAllocator(std::size_t size) : boost::coroutines2::protected_fixedsize_stack(size) { - auto aligment = page_size; - allocator_stack_size = (size + aligment - 1) / aligment * aligment; + auto alignment = page_size; + allocator_stack_size = (size + alignment - 1) / alignment * alignment; } } // namespace engine::coro::debug diff --git a/core/src/engine/coro/marked_allocator.hpp b/core/src/engine/coro/marked_allocator.hpp index 6fc007d7ba3b..5715c3463c3f 100644 --- a/core/src/engine/coro/marked_allocator.hpp +++ b/core/src/engine/coro/marked_allocator.hpp @@ -9,7 +9,7 @@ namespace engine::coro::debug { struct MarkedAllocator : boost::coroutines2::protected_fixedsize_stack { MarkedAllocator(std::size_t size = traits_type::default_size()); - const char kCoroutineMark[16] = "ThisIsCoroAlloc"; + const char coroutine_mark[16] = "ThisIsCoroAlloc"; }; } // namespace engine::coro::debug diff --git a/core/src/engine/coro/pool.cpp b/core/src/engine/coro/pool.cpp index 9900a16ee1c2..8fa11952e8c3 100644 --- a/core/src/engine/coro/pool.cpp +++ b/core/src/engine/coro/pool.cpp @@ -13,6 +13,16 @@ USERVER_NAMESPACE_BEGIN namespace engine::coro { +namespace { +bool IsStackUsageMonitorEnabled() { + // NOLINTNEXTLINE(concurrency-mt-unsafe) + auto* enable = std::getenv("USERVER_ENABLE_STACK_USAGE_MONITOR"); + if (!enable) return true; + if (std::string_view(enable) == "0") return false; + return true; +} +} // namespace + Pool::Pool(PoolConfig config, Executor executor) : config_(FixupConfig(std::move(config))), executor_(executor), @@ -26,7 +36,9 @@ Pool::Pool(PoolConfig config, Executor executor) UASSERT(local_coroutine_move_size_ <= config_.local_cache_size); const moodycamel::ProducerToken token(initial_coroutines_); - if (config_.is_stack_usage_monitor_enabled) { + if (!IsStackUsageMonitorEnabled()) { + LOG_WARNING() << "Stack usage monitor is explicitly disabled via USERVER_ENABLE_STACK_USAGE_MONITOR env var"; + } else if (config_.is_stack_usage_monitor_enabled) { stack_usage_monitor_.Start(); } diff --git a/core/src/engine/coro/pool_config.cpp b/core/src/engine/coro/pool_config.cpp index 51d567d830f4..6b4a1eb7854c 100644 --- a/core/src/engine/coro/pool_config.cpp +++ b/core/src/engine/coro/pool_config.cpp @@ -12,6 +12,7 @@ PoolConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To(config.local_cache_size); config.is_stack_usage_monitor_enabled = value["stack_usage_monitor_enabled"].As(config.is_stack_usage_monitor_enabled); + return config; } diff --git a/core/src/engine/coro/stack_usage_monitor.cpp b/core/src/engine/coro/stack_usage_monitor.cpp index 1219a22b6499..72f5654126ca 100644 --- a/core/src/engine/coro/stack_usage_monitor.cpp +++ b/core/src/engine/coro/stack_usage_monitor.cpp @@ -113,7 +113,7 @@ std::uintptr_t GetStackBegin(const void* cb_ptr) noexcept { [[noreturn]] __attribute__((noinline)) void // the name is intentionally caps-ed, to be easily noticeable in the dumps -THE_COROUTINE_OVERFLOWED_ITS_STACK() { +THE_COROUTINE_OVERFLOWED_ITS_STACK() { // NOLINT(readability-identifier-naming) // We are hitting a coroutine stack overflow, which normally would result in a // SIGSEGV with some hard to diagnose coredump (basically, the only // way to diagnose it correctly as SO is to employ some arcane knowledge: @@ -353,7 +353,7 @@ class StackUsageMonitor::Impl final { monitor_thread_.join(); } - for (auto* thread_alt_stack : threads_alt_stacks) { + for (auto* thread_alt_stack : threads_alt_stacks_) { ::munmap(thread_alt_stack, kAltStackSize); } @@ -459,7 +459,7 @@ class StackUsageMonitor::Impl final { const std::lock_guard lock{tid_to_pthread_initialization_mutex_}; thread_id_to_pthread_id_.emplace_back(tid, thread_id); - threads_alt_stacks.push_back(alt_stack); + threads_alt_stacks_.push_back(alt_stack); } } @@ -585,7 +585,7 @@ class StackUsageMonitor::Impl final { // change at runtime. std::mutex tid_to_pthread_initialization_mutex_; boost::container::small_vector, 32> thread_id_to_pthread_id_{}; - boost::container::small_vector threads_alt_stacks{}; + boost::container::small_vector threads_alt_stacks_{}; std::size_t coro_stack_size_; std::thread monitor_thread_; diff --git a/core/src/engine/coro/stack_usage_monitor_benchmark.cpp b/core/src/engine/coro/stack_usage_monitor_benchmark.cpp index 1bf9e6cdac30..d796889d9a0a 100644 --- a/core/src/engine/coro/stack_usage_monitor_benchmark.cpp +++ b/core/src/engine/coro/stack_usage_monitor_benchmark.cpp @@ -41,7 +41,7 @@ __attribute__((noinline)) std::uint64_t SelfLimitingRecursiveFunction() { return scope.Get() + SelfLimitingRecursiveFunction(); } -void create_async(benchmark::State& state, bool enable) { +void CreateAsync(benchmark::State& state, bool enable) { engine::TaskProcessorPoolsConfig config; config.is_stack_usage_monitor_enabled = enable; @@ -52,10 +52,10 @@ void create_async(benchmark::State& state, bool enable) { } // namespace -void stack_usage_monitor_on(benchmark::State& state) { create_async(state, true); } -BENCHMARK(stack_usage_monitor_on); +void StackUsageMonitorOn(benchmark::State& state) { CreateAsync(state, true); } +BENCHMARK(StackUsageMonitorOn); -void stack_usage_monitor_off(benchmark::State& state) { create_async(state, false); } -BENCHMARK(stack_usage_monitor_off); +void StackUsageMonitorOff(benchmark::State& state) { CreateAsync(state, false); } +BENCHMARK(StackUsageMonitorOff); USERVER_NAMESPACE_END diff --git a/core/src/engine/deadline_benchmark.cpp b/core/src/engine/deadline_benchmark.cpp index 3d4f150c58fb..b3835481949e 100644 --- a/core/src/engine/deadline_benchmark.cpp +++ b/core/src/engine/deadline_benchmark.cpp @@ -10,14 +10,14 @@ USERVER_NAMESPACE_BEGIN namespace { -void deadline_from_duration(benchmark::State& state, std::chrono::nanoseconds duration) { +void DeadlineFromDuration(benchmark::State& state, std::chrono::nanoseconds duration) { for ([[maybe_unused]] auto _ : state) { auto deadline = engine::Deadline::FromDuration(duration); benchmark::DoNotOptimize(deadline); } } -void deadline_is_reached(benchmark::State& state, std::chrono::nanoseconds duration) { +void DeadlineIsReached(benchmark::State& state, std::chrono::nanoseconds duration) { auto deadline = engine::Deadline::FromDuration(duration); for ([[maybe_unused]] auto _ : state) { bool is_reached = deadline.IsReached(); @@ -25,31 +25,27 @@ void deadline_is_reached(benchmark::State& state, std::chrono::nanoseconds durat } } -void deadline_1us_interval_construction(benchmark::State& state) { - deadline_from_duration(state, std::chrono::microseconds{1}); +void Deadline1usIntervalConstruction(benchmark::State& state) { + DeadlineFromDuration(state, std::chrono::microseconds{1}); } -void deadline_20ms_interval_construction(benchmark::State& state) { - deadline_from_duration(state, std::chrono::milliseconds{20}); +void Deadline20msIntervalConstruction(benchmark::State& state) { + DeadlineFromDuration(state, std::chrono::milliseconds{20}); } -void deadline_1us_interval_reached(benchmark::State& state) { - deadline_is_reached(state, std::chrono::microseconds{1}); -} +void Deadline1usIntervalReached(benchmark::State& state) { DeadlineIsReached(state, std::chrono::microseconds{1}); } -void deadline_20ms_interval_reached(benchmark::State& state) { - deadline_is_reached(state, std::chrono::milliseconds{20}); -} +void Deadline20msIntervalReached(benchmark::State& state) { DeadlineIsReached(state, std::chrono::milliseconds{20}); } -void deadline_100s_interval_reached(benchmark::State& state) { deadline_is_reached(state, std::chrono::seconds{100}); } +void Deadline100sIntervalReached(benchmark::State& state) { DeadlineIsReached(state, std::chrono::seconds{100}); } } // namespace -BENCHMARK(deadline_1us_interval_construction); -BENCHMARK(deadline_20ms_interval_construction); +BENCHMARK(Deadline1usIntervalConstruction); +BENCHMARK(Deadline20msIntervalConstruction); -BENCHMARK(deadline_1us_interval_reached); -BENCHMARK(deadline_20ms_interval_reached); -BENCHMARK(deadline_100s_interval_reached); +BENCHMARK(Deadline1usIntervalReached); +BENCHMARK(Deadline20msIntervalReached); +BENCHMARK(Deadline100sIntervalReached); USERVER_NAMESPACE_END diff --git a/core/src/engine/ev/watcher_benchmark.cpp b/core/src/engine/ev/watcher_benchmark.cpp index d9da8051b589..2a80a0312f0c 100644 --- a/core/src/engine/ev/watcher_benchmark.cpp +++ b/core/src/engine/ev/watcher_benchmark.cpp @@ -34,7 +34,7 @@ namespace ev = engine::ev; } // namespace -void watcher_async_start(benchmark::State& state) { +void WatcherAsyncStart(benchmark::State& state) { engine::RunStandalone([&]() { Pipe pipe; ev::Watcher watcher{engine::current_task::GetEventThread(), &pipe}; @@ -46,9 +46,9 @@ void watcher_async_start(benchmark::State& state) { } }); } -BENCHMARK(watcher_async_start); +BENCHMARK(WatcherAsyncStart); -void watcher_async_start_multiple(benchmark::State& state) { +void WatcherAsyncStartMultiple(benchmark::State& state) { engine::RunStandalone([&]() { static constexpr unsigned kPipes = 8; Pipe pipes[kPipes]; @@ -68,6 +68,6 @@ void watcher_async_start_multiple(benchmark::State& state) { } }); } -BENCHMARK(watcher_async_start_multiple); +BENCHMARK(WatcherAsyncStartMultiple); USERVER_NAMESPACE_END diff --git a/core/src/engine/future_benchmark.cpp b/core/src/engine/future_benchmark.cpp index f839fecfae10..7a0cddbf1d13 100644 --- a/core/src/engine/future_benchmark.cpp +++ b/core/src/engine/future_benchmark.cpp @@ -104,7 +104,7 @@ struct FutureCoroSetGet { } // namespace -void future_std_single_threaded(benchmark::State& state) { +void FutureStdSingleThreaded(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { std::promise promise; auto future = promise.get_future(); @@ -112,9 +112,9 @@ void future_std_single_threaded(benchmark::State& state) { benchmark::DoNotOptimize(future.get()); } } -BENCHMARK(future_std_single_threaded); +BENCHMARK(FutureStdSingleThreaded); -void future_coro_single_threaded(benchmark::State& state) { +void FutureCoroSingleThreaded(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { engine::Promise promise; @@ -124,16 +124,16 @@ void future_coro_single_threaded(benchmark::State& state) { } }); } -BENCHMARK(future_coro_single_threaded); +BENCHMARK(FutureCoroSingleThreaded); -void future_std_set_and_get(benchmark::State& state) { +void FutureStdSetAndGet(benchmark::State& state) { engine::RunStandalone(2, [&] { RunPrepared(state); }); } -BENCHMARK(future_std_set_and_get); +BENCHMARK(FutureStdSetAndGet); -void future_coro_set_and_get(benchmark::State& state) { +void FutureCoroSetAndGet(benchmark::State& state) { engine::RunStandalone(2, [&] { RunPrepared(state); }); } -BENCHMARK(future_coro_set_and_get); +BENCHMARK(FutureCoroSetAndGet); USERVER_NAMESPACE_END diff --git a/core/src/engine/impl/mutex_impl.hpp b/core/src/engine/impl/mutex_impl.hpp index c605624825f5..a9c3aa0709dd 100644 --- a/core/src/engine/impl/mutex_impl.hpp +++ b/core/src/engine/impl/mutex_impl.hpp @@ -37,8 +37,6 @@ class MutexImpl { private: class MutexWaitStrategy; - bool TryLockWithTaskContext(TaskContext& current); - bool LockFastPath(TaskContext&) noexcept; bool LockSlowPath(TaskContext&, Deadline); @@ -54,7 +52,7 @@ class MutexImpl::MutexWaitStrategy final : public WaitStrategy { EarlyWakeup SetupWakeups() override { WaitList::Lock lock(mutex_.lock_waiters_); - if (mutex_.TryLockWithTaskContext(current_)) { + if (mutex_.LockFastPath(current_)) { return EarlyWakeup{true}; } // A race is not possible here, because check + Append is performed under @@ -175,17 +173,11 @@ void MutexImpl::unlock() { template bool MutexImpl::try_lock() { auto& current = current_task::GetCurrentTaskContext(); - return TryLockWithTaskContext(current); -} -template -bool MutexImpl::TryLockWithTaskContext(TaskContext& current) { #if USERVER_IMPL_HAS_TSAN __tsan_mutex_pre_lock(this, __tsan_mutex_try_lock); #endif - const auto result = LockFastPath(current); - #if USERVER_IMPL_HAS_TSAN __tsan_mutex_post_lock(this, __tsan_mutex_try_lock | (result ? 0 : __tsan_mutex_try_lock_failed), 0); #endif diff --git a/core/src/engine/impl/wait_list_benchmark.cpp b/core/src/engine/impl/wait_list_benchmark.cpp index 711a7ec3bfaf..93c5b16184dc 100644 --- a/core/src/engine/impl/wait_list_benchmark.cpp +++ b/core/src/engine/impl/wait_list_benchmark.cpp @@ -42,7 +42,7 @@ auto MakeContexts() { } // namespace -void wait_list_insertion(benchmark::State& state) { +void WaitListInsertion(benchmark::State& state) { engine::RunStandalone([&] { std::size_t i = 0; WaitList wl; @@ -69,9 +69,9 @@ void wait_list_insertion(benchmark::State& state) { } }); } -BENCHMARK(wait_list_insertion)->Iterations(kIterationsCount); +BENCHMARK(WaitListInsertion)->Iterations(kIterationsCount); -void wait_list_removal(benchmark::State& state) { +void WaitListRemoval(benchmark::State& state) { engine::RunStandalone([&] { WaitList wl; @@ -104,9 +104,9 @@ void wait_list_removal(benchmark::State& state) { } }); } -BENCHMARK(wait_list_removal)->Iterations(kIterationsCount); +BENCHMARK(WaitListRemoval)->Iterations(kIterationsCount); -void wait_list_add_remove_contention(benchmark::State& state) { +void WaitListAddRemoveContention(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { std::atomic run{true}; WaitList wl; @@ -139,9 +139,9 @@ void wait_list_add_remove_contention(benchmark::State& state) { run = false; }); } -BENCHMARK(wait_list_add_remove_contention)->RangeMultiplier(2)->Range(1, 2)->UseRealTime(); +BENCHMARK(WaitListAddRemoveContention)->RangeMultiplier(2)->Range(1, 2)->UseRealTime(); -void wait_list_add_remove_contention_unbalanced(benchmark::State& state) { +void WaitListAddRemoveContentionUnbalanced(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { std::atomic run{true}; WaitList wl; @@ -189,7 +189,7 @@ void wait_list_add_remove_contention_unbalanced(benchmark::State& state) { // threads an opportunity to work on the WaitList. On top of it, for a benchmark // iteration to complete, the rare ownership switch is required to have occurred // A LOT of times (once per TaskContext). -BENCHMARK(wait_list_add_remove_contention_unbalanced) +BENCHMARK(WaitListAddRemoveContentionUnbalanced) ->RangeMultiplier(2) ->Range(1, 1) ->Unit(benchmark::kMillisecond) diff --git a/core/src/engine/io/fd_control_benchmark.cpp b/core/src/engine/io/fd_control_benchmark.cpp index c721e1cbf8c3..6024dd185543 100644 --- a/core/src/engine/io/fd_control_benchmark.cpp +++ b/core/src/engine/io/fd_control_benchmark.cpp @@ -32,7 +32,7 @@ using FdControl = io::impl::FdControl; } // namespace -void fd_control_destroy(benchmark::State& state) { +void FdControlDestroy(benchmark::State& state) { engine::RunStandalone([&]() { for ([[maybe_unused]] auto _ : state) { state.PauseTiming(); @@ -45,9 +45,9 @@ void fd_control_destroy(benchmark::State& state) { } }); } -BENCHMARK(fd_control_destroy); +BENCHMARK(FdControlDestroy); -void fd_control_close_destroy(benchmark::State& state) { +void FdControlCloseDestroy(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { state.PauseTiming(); @@ -61,9 +61,9 @@ void fd_control_close_destroy(benchmark::State& state) { } }); } -BENCHMARK(fd_control_close_destroy); +BENCHMARK(FdControlCloseDestroy); -void fd_control_wait_destroy(benchmark::State& state) { +void FdControlWaitDestroy(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { state.PauseTiming(); @@ -77,9 +77,9 @@ void fd_control_wait_destroy(benchmark::State& state) { } }); } -BENCHMARK(fd_control_wait_destroy); +BENCHMARK(FdControlWaitDestroy); -void fd_control_construct_wait_destroy(benchmark::State& state) { +void FdControlConstructWaitDestroy(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { state.PauseTiming(); @@ -92,6 +92,6 @@ void fd_control_construct_wait_destroy(benchmark::State& state) { } }); } -BENCHMARK(fd_control_construct_wait_destroy); +BENCHMARK(FdControlConstructWaitDestroy); USERVER_NAMESPACE_END diff --git a/core/src/engine/io/fd_poller.cpp b/core/src/engine/io/fd_poller.cpp index 37743f64d37f..8d7cb04a300f 100644 --- a/core/src/engine/io/fd_poller.cpp +++ b/core/src/engine/io/fd_poller.cpp @@ -83,38 +83,38 @@ struct FdPoller::Impl final : public engine::impl::ContextAccessor { static void IoWatcherCb(struct ev_loop*, ev_io*, int) noexcept; void WakeupWaiters(); - void ResetReady() noexcept { waiters_->GetAndResetSignal(); } + void ResetReady() noexcept { waiters->GetAndResetSignal(); } // ContextAccessor implementation - bool IsReady() const noexcept override { return waiters_->IsSignaled(); } + bool IsReady() const noexcept override { return waiters->IsSignaled(); } engine::impl::EarlyWakeup TryAppendWaiter(engine::impl::TaskContext& waiter) override { - if (waiters_->GetSignalOrAppend(&waiter)) { + if (waiters->GetSignalOrAppend(&waiter)) { return engine::impl::EarlyWakeup{true}; } - watcher_.StartAsync(); + watcher.StartAsync(); return engine::impl::EarlyWakeup{false}; } void RemoveWaiter(engine::impl::TaskContext& waiter) noexcept override { - waiters_->Remove(waiter); + waiters->Remove(waiter); // we need to stop watcher manually to avoid racy wakeups later - watcher_.StopAsync(); + watcher.StopAsync(); } - void AfterWait() noexcept override { watcher_.Stop(); } + void AfterWait() noexcept override { watcher.Stop(); } void RethrowErrorResult() const override {} - std::atomic state_{FdPoller::State::kInvalid}; - engine::impl::FastPimplWaitListLight waiters_; - ev::Watcher watcher_; - std::atomic events_that_happened_{}; + std::atomic state{FdPoller::State::kInvalid}; + engine::impl::FastPimplWaitListLight waiters; + ev::Watcher watcher; + std::atomic events_that_happened{}; }; -void FdPoller::Impl::WakeupWaiters() { waiters_->SetSignalAndWakeupOne(); } +void FdPoller::Impl::WakeupWaiters() { waiters->SetSignalAndWakeupOne(); } -FdPoller::Impl::Impl(ev::ThreadControl control) : watcher_(control, this) { watcher_.Init(&IoWatcherCb); } +FdPoller::Impl::Impl(ev::ThreadControl control) : watcher(control, this) { watcher.Init(&IoWatcherCb); } FdPoller::Impl::~Impl() = default; @@ -130,7 +130,7 @@ engine::impl::TaskContext::WakeupSource FdPoller::Impl::DoWait(Deadline deadline * Manually call Stop() here to be sure that after DoWait() no waiter_'s * callback (IoWatcherCb) is running. */ - watcher_.Stop(); + watcher.Stop(); return ret; } @@ -138,7 +138,7 @@ void FdPoller::Impl::Invalidate() { StopWatcher(); auto old_state = State::kReadyToUse; - const auto res = state_.compare_exchange_strong(old_state, State::kInvalid); + const auto res = state.compare_exchange_strong(old_state, State::kInvalid); UINVARIANT( res, @@ -148,7 +148,7 @@ void FdPoller::Impl::Invalidate() { void FdPoller::Impl::StopWatcher() noexcept { UASSERT(IsValid()); - watcher_.Stop(); + watcher.Stop(); } void FdPoller::Impl::IoWatcherCb(struct ev_loop*, ev_io* watcher, int) noexcept { @@ -161,13 +161,13 @@ void FdPoller::Impl::IoWatcherCb(struct ev_loop*, ev_io* watcher, int) noexcept // Cleanup watcher_ first, then awake the coroutine. // Otherwise, the coroutine may close watcher_'s fd before watcher_ is stopped. - const auto guard = self->watcher_.StopWithinEvCallback(); + const auto guard = self->watcher.StopWithinEvCallback(); - self->events_that_happened_.store(GetUserMode(ev_events), std::memory_order_relaxed); + self->events_that_happened.store(GetUserMode(ev_events), std::memory_order_relaxed); self->WakeupWaiters(); } -bool FdPoller::Impl::IsValid() const noexcept { return state_ != State::kInvalid; } +bool FdPoller::Impl::IsValid() const noexcept { return state != State::kInvalid; } FdPoller::FdPoller(const ev::ThreadControl& control) : pimpl_(control) { static_assert(std::atomic::is_always_lock_free); @@ -179,20 +179,20 @@ FdPoller::operator bool() const noexcept { return IsValid(); } bool FdPoller::IsValid() const noexcept { return pimpl_->IsValid(); } -int FdPoller::GetFd() const noexcept { return pimpl_->watcher_.GetFd(); } +int FdPoller::GetFd() const noexcept { return pimpl_->watcher.GetFd(); } std::optional FdPoller::Wait(Deadline deadline) { ResetReady(); if (pimpl_->DoWait(deadline) == engine::impl::TaskContext::WakeupSource::kWaitList) { - return pimpl_->events_that_happened_.load(std::memory_order_relaxed); + return pimpl_->events_that_happened.load(std::memory_order_relaxed); } else { return std::nullopt; } } std::optional FdPoller::GetReady() noexcept { - if (pimpl_->waiters_->GetAndResetSignal()) { - return pimpl_->events_that_happened_.load(std::memory_order_relaxed); + if (pimpl_->waiters->GetAndResetSignal()) { + return pimpl_->events_that_happened.load(std::memory_order_relaxed); } else { return std::nullopt; } @@ -208,7 +208,7 @@ void FdPoller::WakeupWaiters() { pimpl_->WakeupWaiters(); } void FdPoller::SwitchStateToInUse() { auto old_state = State::kReadyToUse; - const auto res = pimpl_->state_.compare_exchange_strong(old_state, State::kInUse); + const auto res = pimpl_->state.compare_exchange_strong(old_state, State::kInUse); UASSERT_MSG( res, @@ -218,7 +218,7 @@ void FdPoller::SwitchStateToInUse() { void FdPoller::SwitchStateToReadyToUse() { auto old_state = State::kInUse; - const auto res = pimpl_->state_.compare_exchange_strong(old_state, State::kReadyToUse); + const auto res = pimpl_->state.compare_exchange_strong(old_state, State::kReadyToUse); UASSERT_MSG( res, fmt::format("Socket misuse: expected socket state is '{}', actual state is '{}'", State::kInUse, old_state) ); @@ -226,9 +226,9 @@ void FdPoller::SwitchStateToReadyToUse() { void FdPoller::Impl::Reset(int fd, Kind kind) { UASSERT(!IsValid()); - UASSERT(watcher_.GetFd() == fd || watcher_.GetFd() == -1); - watcher_.Set(fd, GetEvMode(kind)); - state_ = State::kReadyToUse; + UASSERT(watcher.GetFd() == fd || watcher.GetFd() == -1); + watcher.Set(fd, GetEvMode(kind)); + state = State::kReadyToUse; } void FdPoller::ResetReady() noexcept { pimpl_->ResetReady(); } diff --git a/core/src/engine/io/poller.cpp b/core/src/engine/io/poller.cpp index 24fc7d741fd8..6717a9ec02cd 100644 --- a/core/src/engine/io/poller.cpp +++ b/core/src/engine/io/poller.cpp @@ -109,11 +109,11 @@ void Poller::RemoveImpl(Poller::IoWatcher& watcher) { } Poller::Status Poller::NextEvent(Event& buf, Deadline deadline) { - return EventsFilter([this, deadline](Event& buf_) { return event_consumer_.Pop(buf_, deadline); }, buf); + return EventsFilter([this, deadline](Event& l_buf) { return event_consumer_.Pop(l_buf, deadline); }, buf); } Poller::Status Poller::NextEventNoblock(Event& buf) { - return EventsFilter([this](Event& buf_) { return event_consumer_.PopNoblock(buf_); }, buf); + return EventsFilter([this](Event& l_buf) { return event_consumer_.PopNoblock(l_buf); }, buf); } void Poller::Interrupt() { diff --git a/core/src/engine/io/socket_benchmark.cpp b/core/src/engine/io/socket_benchmark.cpp index 14b62159570a..95c27ab73704 100644 --- a/core/src/engine/io/socket_benchmark.cpp +++ b/core/src/engine/io/socket_benchmark.cpp @@ -25,7 +25,7 @@ constexpr auto kDeadlineMaxTime = std::chrono::seconds{60}; } // namespace -void socket_send_all(benchmark::State& state) { +void SocketSendAll(benchmark::State& state) { engine::RunStandalone([&]() { const auto test_deadline = Deadline::FromDuration(kDeadlineMaxTime); internal::net::TcpListener listener; @@ -49,9 +49,9 @@ void socket_send_all(benchmark::State& state) { task_reader.Get(); }); } -BENCHMARK(socket_send_all); +BENCHMARK(SocketSendAll); -void socket_send_all_v(benchmark::State& state) { +void SocketSendAllV(benchmark::State& state) { engine::RunStandalone([&]() { const auto test_deadline = Deadline::FromDuration(kDeadlineMaxTime); internal::net::TcpListener listener; @@ -73,9 +73,9 @@ void socket_send_all_v(benchmark::State& state) { task_reader.Get(); }); } -BENCHMARK(socket_send_all_v); +BENCHMARK(SocketSendAllV); -[[maybe_unused]] void socket_send_all_v_range(benchmark::State& state) { +[[maybe_unused]] void SocketSendAllVRange(benchmark::State& state) { engine::RunStandalone(2, [&]() { const auto test_deadline = Deadline::FromDuration(kDeadlineMaxTime); internal::net::TcpListener listener; @@ -105,7 +105,7 @@ BENCHMARK(socket_send_all_v); // TODO(TAXICOMMON-5510) flaky, sometimes throws engine::io::IoTimeout // BENCHMARK(socket_send_all_v_range)->RangeMultiplier(10)->Range(10, 10000); -[[maybe_unused]] void socket_send_all_range(benchmark::State& state) { +[[maybe_unused]] void SocketSendAllRange(benchmark::State& state) { engine::RunStandalone(2, [&]() { const auto test_deadline = Deadline::FromDuration(kDeadlineMaxTime); internal::net::TcpListener listener; diff --git a/core/src/engine/io/tls_wrapper.cpp b/core/src/engine/io/tls_wrapper.cpp index ccd4890aef31..a0b78aa1d373 100644 --- a/core/src/engine/io/tls_wrapper.cpp +++ b/core/src/engine/io/tls_wrapper.cpp @@ -21,11 +21,6 @@ USERVER_NAMESPACE_BEGIN namespace engine::io { namespace { -struct SslCtxDeleter { - void operator()(SSL_CTX* ctx) const noexcept { SSL_CTX_free(ctx); } -}; -using SslCtx = std::unique_ptr; - struct SslDeleter { void operator()(SSL* ssl) const noexcept { SSL_free(ssl); } }; @@ -190,65 +185,11 @@ int SSL_write_ex(SSL* ssl, const void* data, size_t len, size_t* bytes_written) } #endif -SslCtx MakeSslCtx() { - crypto::Openssl::Init(); - - SslCtx ssl_ctx{SSL_CTX_new(SSLv23_method())}; - if (!ssl_ctx) { - throw TlsException(crypto::FormatSslError("Failed create an SSL context: SSL_CTX_new")); - } -#if OPENSSL_VERSION_NUMBER >= 0x010100000L - if (1 != SSL_CTX_set_min_proto_version(ssl_ctx.get(), TLS1_VERSION)) { - throw TlsException(crypto::FormatSslError("Failed create an SSL context: SSL_CTX_set_min_proto_version")); - } -#endif - - constexpr auto options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION -#if OPENSSL_VERSION_NUMBER >= 0x010100000L - | SSL_OP_NO_RENEGOTIATION -#endif - ; - SSL_CTX_set_options(ssl_ctx.get(), options); - SSL_CTX_set_mode(ssl_ctx.get(), SSL_MODE_ENABLE_PARTIAL_WRITE); - SSL_CTX_clear_mode(ssl_ctx.get(), SSL_MODE_AUTO_RETRY); - if (1 != SSL_CTX_set_default_verify_paths(ssl_ctx.get())) { - LOG_LIMITED_WARNING() << crypto::FormatSslError("Failed create an SSL context: SSL_CTX_set_default_verify_paths" - ); - } - return ssl_ctx; -} - enum InterruptAction { kPass, kFail, }; -void SetServerName(SslCtx& ctx, std::string_view server_name) { - if (server_name.empty()) { - return; - } - - X509_VERIFY_PARAM* verify_param = SSL_CTX_get0_param(ctx.get()); - if (!verify_param) { - throw TlsException("Failed to set up client TLS wrapper: SSL_CTX_get0_param"); - } - if (1 != X509_VERIFY_PARAM_set1_host(verify_param, server_name.data(), server_name.size())) { - throw TlsException(crypto::FormatSslError("Failed to set up client TLS wrapper: X509_VERIFY_PARAM_set1_host")); - } - SSL_CTX_set_verify(ctx.get(), SSL_VERIFY_PEER, nullptr); -} - -void AddCertAuthorities(SslCtx& ctx, const std::vector& cert_authorities) { - UASSERT(!cert_authorities.empty()); - auto* store = SSL_CTX_get_cert_store(ctx.get()); - UASSERT(store); - for (const auto& ca : cert_authorities) { - if (1 != X509_STORE_add_cert(store, ca.GetNative())) { - throw TlsException(crypto::FormatSslError("Failed to set up client TLS wrapper: X509_STORE_add_cert")); - } - } -} - } // namespace class TlsWrapper::ReadContextAccessor final : public engine::impl::ContextAccessor { @@ -267,7 +208,7 @@ class TlsWrapper::ReadContextAccessor final : public engine::impl::ContextAccess engine::impl::ContextAccessor& GetSocketContextAccessor() const noexcept; - TlsWrapper::Impl& impl_; + TlsWrapper::Impl& impl; }; class TlsWrapper::Impl { @@ -284,7 +225,7 @@ class TlsWrapper::Impl { SyncBioData(SSL_get_rbio(ssl.get()), &other.bio_data); } - void SetUp(SslCtx&& ssl_ctx) { + void SetUp(const crypto::SslCtx& ssl_ctx) { Bio socket_bio{BIO_new(GetSocketBioMethod())}; if (!socket_bio) { throw TlsException(crypto::FormatSslError("Failed to set up TLS wrapper: BIO_new")); @@ -293,7 +234,7 @@ class TlsWrapper::Impl { SyncBioData(socket_bio.get(), nullptr); BIO_set_init(socket_bio.get(), 1); - ssl.reset(SSL_new(ssl_ctx.get())); + ssl.reset(SSL_new(static_cast(ssl_ctx.GetRawSslCtx()))); if (!ssl) { throw TlsException(crypto::FormatSslError("Failed to set up TLS wrapper: SSL_new")); } @@ -329,6 +270,41 @@ class TlsWrapper::Impl { } } + template + std::optional PerformSslIoOptional(SslIoFunc&& io_func, void* buf, size_t len, const char* context) { + UASSERT(ssl); + if (!len) return 0; + + char* const begin = static_cast(buf); + char* const end = begin + len; + size_t chunk_size = 0; + + bio_data.current_deadline = Deadline::Passed(); + + const int io_ret = io_func(ssl.get(), begin, end - begin, &chunk_size); + if (io_ret == 1) { + return chunk_size; + } + + const int ssl_error = SSL_get_error(ssl.get(), io_ret); + switch (ssl_error) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + return std::nullopt; + + case SSL_ERROR_ZERO_RETURN: + return 0; + + case SSL_ERROR_SYSCALL: + case SSL_ERROR_SSL: + ssl.reset(); + [[fallthrough]]; + + default: + throw TlsException(crypto::FormatSslError(std::string{context} + " failed")); + } + } + template size_t PerformSslIo( SslIoFunc&& io_func, @@ -422,16 +398,16 @@ class TlsWrapper::Impl { } }; -TlsWrapper::ReadContextAccessor::ReadContextAccessor(TlsWrapper::Impl& impl) : impl_(impl) {} +TlsWrapper::ReadContextAccessor::ReadContextAccessor(TlsWrapper::Impl& impl) : impl(impl) {} bool TlsWrapper::ReadContextAccessor::IsReady() const noexcept { - auto* ssl = impl_.ssl.get(); + auto* ssl = impl.ssl.get(); if (!ssl || SSL_has_pending(ssl)) return true; return GetSocketContextAccessor().IsReady(); } engine::impl::EarlyWakeup TlsWrapper::ReadContextAccessor::TryAppendWaiter(engine::impl::TaskContext& waiter) { - auto* ssl = impl_.ssl.get(); + auto* ssl = impl.ssl.get(); if (!ssl || SSL_has_pending(ssl)) return engine::impl::EarlyWakeup{true}; return GetSocketContextAccessor().TryAppendWaiter(waiter); @@ -446,7 +422,7 @@ void TlsWrapper::ReadContextAccessor::AfterWait() noexcept { GetSocketContextAcc void TlsWrapper::ReadContextAccessor::RethrowErrorResult() const { GetSocketContextAccessor().RethrowErrorResult(); } engine::impl::ContextAccessor& TlsWrapper::ReadContextAccessor::GetSocketContextAccessor() const noexcept { - auto* ca = impl_.bio_data.socket.GetReadableBase().TryGetContextAccessor(); + auto* ca = impl.bio_data.socket.GetReadableBase().TryGetContextAccessor(); UASSERT(ca); return *ca; } @@ -454,11 +430,8 @@ engine::impl::ContextAccessor& TlsWrapper::ReadContextAccessor::GetSocketContext TlsWrapper::TlsWrapper(Socket&& socket) : impl_(std::move(socket)) { SetupContextAccessors(); } TlsWrapper TlsWrapper::StartTlsClient(Socket&& socket, const std::string& server_name, Deadline deadline) { - auto ssl_ctx = MakeSslCtx(); - SetServerName(ssl_ctx, server_name); - TlsWrapper wrapper{std::move(socket)}; - wrapper.impl_->SetUp(std::move(ssl_ctx)); + wrapper.impl_->SetUp(crypto::SslCtx::CreateClientTlsContext(server_name)); wrapper.impl_->ClientConnect(server_name, deadline); return wrapper; } @@ -471,79 +444,15 @@ TlsWrapper TlsWrapper::StartTlsClient( Deadline deadline, const std::vector& extra_cert_authorities ) { - auto ssl_ctx = MakeSslCtx(); - SetServerName(ssl_ctx, server_name); - - if (!extra_cert_authorities.empty()) { - AddCertAuthorities(ssl_ctx, extra_cert_authorities); - } - - if (cert) { - if (1 != SSL_CTX_use_certificate(ssl_ctx.get(), cert.GetNative())) { - throw TlsException(crypto::FormatSslError("Failed to set up client TLS wrapper: SSL_CTX_use_certificate")); - } - } - - if (key) { - if (1 != SSL_CTX_use_PrivateKey(ssl_ctx.get(), key.GetNative())) { - throw TlsException(crypto::FormatSslError("Failed to set up client TLS wrapper: SSL_CTX_use_PrivateKey")); - } - } - TlsWrapper wrapper{std::move(socket)}; - wrapper.impl_->SetUp(std::move(ssl_ctx)); + wrapper.impl_->SetUp(crypto::SslCtx::CreateClientTlsContext(server_name, cert, key, extra_cert_authorities)); wrapper.impl_->ClientConnect(server_name, deadline); return wrapper; } -TlsWrapper TlsWrapper::StartTlsServer( - Socket&& socket, - const crypto::CertificatesChain& cert_chain, - const crypto::PrivateKey& key, - Deadline deadline, - const std::vector& extra_cert_authorities -) { - auto ssl_ctx = MakeSslCtx(); - - if (!extra_cert_authorities.empty()) { - AddCertAuthorities(ssl_ctx, extra_cert_authorities); - SSL_CTX_set_verify(ssl_ctx.get(), SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr); - LOG_INFO() << "Client SSL cert will be verified"; - } else { - LOG_INFO() << "Client SSL cert will not be verified"; - } - - if (cert_chain.empty()) { - throw TlsException(crypto::FormatSslError("Empty certificate chain provided")); - } - - if (1 != SSL_CTX_use_certificate(ssl_ctx.get(), cert_chain.begin()->GetNative())) { - throw TlsException(crypto::FormatSslError("Failed to set up server TLS wrapper: SSL_CTX_use_certificate")); - } - - if (cert_chain.size() > 1) { - auto cert_it = std::next(cert_chain.begin()); - for (; cert_it != cert_chain.end(); ++cert_it) { - // cast in openssl1.0 macro expansion - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast) - if (SSL_CTX_add_extra_chain_cert(ssl_ctx.get(), cert_it->GetNative()) <= 0) { - throw TlsException( - crypto::FormatSslError("Failed to set up server TLS wrapper: SSL_CTX_add_extra_chain_cert") - ); - } - - // After SSL_CTX_add_extra_chain_cert we should not free the cert - const auto ret = X509_up_ref(cert_it->GetNative()); - UASSERT(ret == 1); - } - } - - if (1 != SSL_CTX_use_PrivateKey(ssl_ctx.get(), key.GetNative())) { - throw TlsException(crypto::FormatSslError("Failed to set up server TLS wrapper: SSL_CTX_use_PrivateKey")); - } - +TlsWrapper TlsWrapper::StartTlsServer(Socket&& socket, const crypto::SslCtx& ctx, Deadline deadline) { TlsWrapper wrapper{std::move(socket)}; - wrapper.impl_->SetUp(std::move(ssl_ctx)); + wrapper.impl_->SetUp(ctx); wrapper.impl_->bio_data.current_deadline = deadline; auto ret = SSL_accept(wrapper.impl_->ssl.get()); @@ -609,6 +518,11 @@ size_t TlsWrapper::RecvAll(void* buf, size_t len, Deadline deadline) { ); } +std::optional TlsWrapper::RecvNoblock(void* buf, size_t len) { + impl_->CheckAlive(); + return impl_->PerformSslIoOptional(&SSL_read_ex, buf, len, "RecvNoblock"); +} + size_t TlsWrapper::SendAll(const void* buf, size_t len, Deadline deadline) { impl_->CheckAlive(); return impl_->PerformSslIo( diff --git a/core/src/engine/io/tls_wrapper_benchmark.cpp b/core/src/engine/io/tls_wrapper_benchmark.cpp index e617bf19cc34..dddef9fc65ea 100644 --- a/core/src/engine/io/tls_wrapper_benchmark.cpp +++ b/core/src/engine/io/tls_wrapper_benchmark.cpp @@ -85,7 +85,7 @@ constexpr auto kDeadlineMaxTime = std::chrono::seconds{60}; } // namespace -[[maybe_unused]] void tls_write_all_buffered(benchmark::State& state) { +[[maybe_unused]] void TlsWriteAllBuffered(benchmark::State& state) { engine::RunStandalone(2, [&]() { const auto deadline = Deadline::FromDuration(kDeadlineMaxTime); @@ -93,14 +93,13 @@ constexpr auto kDeadlineMaxTime = std::chrono::seconds{60}; auto [server, client] = tcp_listener.MakeSocketPair(deadline); std::atomic reading{true}; + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) + ); auto server_task = engine::AsyncNoSpan( - [&reading, deadline](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - deadline - ); + [&reading, deadline, &ssl_ctx](auto&& server) { + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, deadline); std::array buf{}; while (tls_server.RecvSome(buf.data(), buf.size(), deadline) > 0 && reading) { @@ -127,9 +126,9 @@ constexpr auto kDeadlineMaxTime = std::chrono::seconds{60}; }); } -BENCHMARK(tls_write_all_buffered)->RangeMultiplier(2)->Range(1 << 6, 1 << 12)->Unit(benchmark::kNanosecond); +BENCHMARK(TlsWriteAllBuffered)->RangeMultiplier(2)->Range(1 << 6, 1 << 12)->Unit(benchmark::kNanosecond); -[[maybe_unused]] void tls_write_all_default(benchmark::State& state) { +[[maybe_unused]] void TlsWriteAllDefault(benchmark::State& state) { engine::RunStandalone(2, [&]() { const auto deadline = Deadline::FromDuration(kDeadlineMaxTime); @@ -137,14 +136,13 @@ BENCHMARK(tls_write_all_buffered)->RangeMultiplier(2)->Range(1 << 6, 1 << 12)->U auto [server, client] = tcp_listener.MakeSocketPair(deadline); std::atomic reading{true}; + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) + ); auto server_task = engine::AsyncNoSpan( - [&reading, deadline](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - deadline - ); + [&reading, deadline, &ssl_ctx](auto&& server) { + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, deadline); std::array buf{}; while (tls_server.RecvSome(buf.data(), buf.size(), deadline) > 0 && reading) { @@ -173,6 +171,6 @@ BENCHMARK(tls_write_all_buffered)->RangeMultiplier(2)->Range(1 << 6, 1 << 12)->U }); } -BENCHMARK(tls_write_all_default)->RangeMultiplier(2)->Range(1 << 6, 1 << 12)->Unit(benchmark::kNanosecond); +BENCHMARK(TlsWriteAllDefault)->RangeMultiplier(2)->Range(1 << 6, 1 << 12)->Unit(benchmark::kNanosecond); USERVER_NAMESPACE_END diff --git a/core/src/engine/io/tls_wrapper_test.cpp b/core/src/engine/io/tls_wrapper_test.cpp index 68cfa30eb1a9..178859764d74 100644 --- a/core/src/engine/io/tls_wrapper_test.cpp +++ b/core/src/engine/io/tls_wrapper_test.cpp @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include #include @@ -213,6 +215,77 @@ ud5lWYZNZ6ygIOvwFz1VST30jT4Lj3Bmrg== -----END CERTIFICATE-----)"; constexpr auto kShortTimeout = std::chrono::milliseconds{10}; +void Tests2Servers2Clients(const crypto::SslCtx& ssl_ctx1, const crypto::SslCtx& ssl_ctx2) { + const auto test_deadline = Deadline::FromDuration(utest::kMaxTestWaitTime); + + TcpListener tcp_listener; + auto [server, client] = tcp_listener.MakeSocketPair(test_deadline); + auto [other_server, other_client] = tcp_listener.MakeSocketPair(test_deadline); + + auto server_task = engine::AsyncNoSpan( + [test_deadline, &ssl_ctx1](auto&& server) { + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx1, test_deadline); + EXPECT_EQ(1, tls_server.SendAll("1", 1, test_deadline)); + char c = 0; + EXPECT_EQ(1, tls_server.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('2', c); + + auto raw_server = tls_server.StopTls(test_deadline); + EXPECT_EQ(1, raw_server.SendAll("3", 1, test_deadline)); + EXPECT_EQ(1, raw_server.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('4', c); + }, + std::move(server) + ); + + auto other_server_task = engine::AsyncNoSpan( + [test_deadline, &ssl_ctx2](auto&& server) { + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx2, test_deadline); + EXPECT_EQ(1, tls_server.SendAll("5", 1, test_deadline)); + char c = 0; + EXPECT_EQ(1, tls_server.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('6', c); + + auto raw_server = tls_server.StopTls(test_deadline); + EXPECT_EQ(1, raw_server.SendAll("7", 1, test_deadline)); + EXPECT_EQ(1, raw_server.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('8', c); + }, + std::move(other_server) + ); + + auto other_client_task = engine::AsyncNoSpan( + [test_deadline](auto&& client) { + auto tls_client = io::TlsWrapper::StartTlsClient(std::forward(client), {}, test_deadline); + char c = 0; + EXPECT_EQ(1, tls_client.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('5', c); + EXPECT_EQ(1, tls_client.SendAll("6", 1, test_deadline)); + auto raw_client = tls_client.StopTls(test_deadline); + EXPECT_EQ(1, raw_client.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('7', c); + EXPECT_EQ(1, raw_client.SendAll("8", 1, test_deadline)); + }, + std::move(other_client) + ); + + auto tls_client = io::TlsWrapper::StartTlsClient(std::move(client), {}, test_deadline); + char c = 0; + EXPECT_EQ(1, tls_client.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('1', c); + EXPECT_EQ(1, tls_client.SendAll("2", 1, test_deadline)); + auto raw_client = tls_client.StopTls(test_deadline); + EXPECT_EQ(1, raw_client.RecvSome(&c, 1, test_deadline)); + EXPECT_EQ('3', c); + EXPECT_EQ(1, raw_client.SendAll("4", 1, test_deadline)); + + server_task.Get(); + other_server_task.Get(); + other_client_task.Get(); +} + } // namespace UTEST(TlsWrapper, InitListSmall) { @@ -232,12 +305,11 @@ UTEST(TlsWrapper, InitListSmall) { auto server_task = utils::Async( "tls-server", [deadline, kDataA, kDataB, kDataC, kDataD](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( crypto::LoadCertificatesChainFromString(cert_chain), - crypto::PrivateKey::LoadFromString(chain_private_key), - deadline + crypto::PrivateKey::LoadFromString(chain_private_key) ); + auto tls_server = io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, deadline); if (tls_server.WriteAll({kDataA, kDataB, kDataC, kDataD}, deadline) != kDataA.len + kDataB.len + kDataC.len + kDataD.len) { throw std::runtime_error("Couldn't send data"); @@ -272,12 +344,10 @@ UTEST(TlsWrapper, InitListLarge) { auto server_task = utils::Async( "tls-server", [deadline, kDataA, kDataB, kDataC, kDataD](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, deadline); if (tls_server.WriteAll({kDataA, kDataB, kDataC, kDataD}, deadline) != kDataA.len + kDataB.len + kDataC.len + kDataD.len) { throw std::runtime_error("Couldn't send data"); @@ -308,12 +378,10 @@ UTEST(TlsWrapper, InitListSmallThenLarge) { auto server_task = utils::Async( "tls-server", [deadline, kDataSmall, kDataLarge](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, deadline); if (tls_server.WriteAll({kDataSmall, kDataSmall, kDataSmall, kDataSmall, kDataLarge}, deadline) != kDataSmall.len * 4 + kDataLarge.len) { throw std::runtime_error("Couldn't send data"); @@ -340,12 +408,11 @@ UTEST_MT(TlsWrapper, Smoke, 2) { auto server_task = engine::AsyncNoSpan( [test_deadline](auto&& server) { try { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - test_deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, test_deadline); EXPECT_EQ(1, tls_server.SendAll("1", 1, test_deadline)); char c = 0; EXPECT_EQ(1, tls_server.RecvSome(&c, 1, test_deadline)); @@ -387,12 +454,10 @@ UTEST_MT(TlsWrapper, DocTest, 2) { auto server_task = utils::Async( "tls-server", [deadline](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, deadline); if (tls_server.SendAll(kData.data(), kData.size(), deadline) != kData.size()) { throw std::runtime_error("Couldn't send data"); } @@ -419,12 +484,11 @@ UTEST(TlsWrapper, Move) { auto server_task = engine::AsyncNoSpan( [test_deadline](auto&& server) { try { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - test_deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, test_deadline); engine::AsyncNoSpan( [test_deadline](auto&& tls_server) { EXPECT_EQ(1, tls_server.SendAll("1", 1, test_deadline)); @@ -468,13 +532,14 @@ UTEST(TlsWrapper, ConnectTimeout) { static_cast(io::TlsWrapper::StartTlsClient(std::move(client), {}, Deadline::FromDuration(kShortTimeout))), io::IoTimeout ); + + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) + ); EXPECT_THROW( - static_cast(io::TlsWrapper::StartTlsServer( - std::move(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - Deadline::FromDuration(kShortTimeout) - )), + static_cast( + io::TlsWrapper::StartTlsServer(std::move(server), ssl_ctx, Deadline::FromDuration(kShortTimeout)) + ), io::IoException ); } @@ -488,12 +553,11 @@ UTEST_MT(TlsWrapper, IoTimeout, 2) { engine::SingleConsumerEvent timeout_happened; auto server_task = engine::AsyncNoSpan( [test_deadline, &timeout_happened](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - test_deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, test_deadline); char c = 0; UEXPECT_THROW( static_cast(tls_server.RecvSome(&c, 1, Deadline::FromDuration(kShortTimeout))), io::IoTimeout @@ -525,12 +589,11 @@ UTEST(TlsWrapper, Cancel) { auto server_task = engine::AsyncNoSpan( [test_deadline](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - test_deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, test_deadline); char c = 0; UEXPECT_THROW(static_cast(tls_server.RecvSome(&c, 1, test_deadline)), io::IoInterrupted); }, @@ -551,21 +614,21 @@ UTEST_MT(TlsWrapper, CertKeyMismatch, 2) { auto server_task = engine::AsyncNoSpan( [test_deadline](auto&& server) { - UEXPECT_THROW( - static_cast(io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(other_key), - test_deadline - )), - io::TlsException - ); + UEXPECT_THROW(crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), + crypto::PrivateKey::LoadFromString(other_key) + ); + static_cast(io::TlsWrapper::StartTlsServer( + std::forward(server), ssl_ctx, test_deadline + )), + crypto::CryptoException); }, std::move(server) ); EXPECT_THROW( - static_cast(io::TlsWrapper::StartTlsClient(std::move(client), {}, test_deadline)), io::IoException + static_cast(io::TlsWrapper::StartTlsClient(std::move(client), {}, test_deadline)), + std::exception // Could be engine::io::TlsException or engine::io::IoSystemError ); server_task.Get(); } @@ -578,15 +641,14 @@ UTEST_MT(TlsWrapper, NonTlsClient, 2) { auto server_task = engine::AsyncNoSpan( [test_deadline](auto&& server) { - UEXPECT_THROW( - static_cast(io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(other_key), - test_deadline - )), - io::TlsException - ); + UEXPECT_THROW(crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), + crypto::PrivateKey::LoadFromString(other_key) + ); + static_cast(io::TlsWrapper::StartTlsServer( + std::forward(server), ssl_ctx, test_deadline + )), + crypto::CryptoException); }, std::move(server) ); @@ -612,94 +674,45 @@ UTEST_MT(TlsWrapper, NonTlsServer, 2) { } UTEST_MT(TlsWrapper, DoubleSmoke, 4) { - const auto test_deadline = Deadline::FromDuration(utest::kMaxTestWaitTime); - - TcpListener tcp_listener; - auto [server, client] = tcp_listener.MakeSocketPair(test_deadline); - auto [other_server, other_client] = tcp_listener.MakeSocketPair(test_deadline); - - auto server_task = engine::AsyncNoSpan( - [test_deadline](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - test_deadline - ); - EXPECT_EQ(1, tls_server.SendAll("1", 1, test_deadline)); - char c = 0; - EXPECT_EQ(1, tls_server.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('2', c); - - auto raw_server = tls_server.StopTls(test_deadline); - EXPECT_EQ(1, raw_server.SendAll("3", 1, test_deadline)); - EXPECT_EQ(1, raw_server.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('4', c); - }, - std::move(server) + crypto::SslCtx ssl_ctx1 = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + crypto::SslCtx ssl_ctx2 = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(other_cert), crypto::PrivateKey::LoadFromString(other_key) + ); + Tests2Servers2Clients(ssl_ctx1, ssl_ctx2); +} - auto other_server_task = engine::AsyncNoSpan( - [test_deadline](auto&& server) { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(other_cert), - crypto::PrivateKey::LoadFromString(other_key), - test_deadline - ); - EXPECT_EQ(1, tls_server.SendAll("5", 1, test_deadline)); - char c = 0; - EXPECT_EQ(1, tls_server.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('6', c); - - auto raw_server = tls_server.StopTls(test_deadline); - EXPECT_EQ(1, raw_server.SendAll("7", 1, test_deadline)); - EXPECT_EQ(1, raw_server.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('8', c); - }, - std::move(other_server) +UTEST_MT(TlsWrapper, DoubleSmokeSameCtx, 4) { + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + Tests2Servers2Clients(ssl_ctx, ssl_ctx); +} - auto other_client_task = engine::AsyncNoSpan( - [test_deadline](auto&& client) { - auto tls_client = io::TlsWrapper::StartTlsClient(std::forward(client), {}, test_deadline); - char c = 0; - EXPECT_EQ(1, tls_client.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('5', c); - EXPECT_EQ(1, tls_client.SendAll("6", 1, test_deadline)); - auto raw_client = tls_client.StopTls(test_deadline); - EXPECT_EQ(1, raw_client.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('7', c); - EXPECT_EQ(1, raw_client.SendAll("8", 1, test_deadline)); - }, - std::move(other_client) +UTEST_MT(TlsWrapper, SmokeSameCtxTorture, 4) { + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); - auto tls_client = io::TlsWrapper::StartTlsClient(std::move(client), {}, test_deadline); - char c = 0; - EXPECT_EQ(1, tls_client.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('1', c); - EXPECT_EQ(1, tls_client.SendAll("2", 1, test_deadline)); - auto raw_client = tls_client.StopTls(test_deadline); - EXPECT_EQ(1, raw_client.RecvSome(&c, 1, test_deadline)); - EXPECT_EQ('3', c); - EXPECT_EQ(1, raw_client.SendAll("4", 1, test_deadline)); + constexpr unsigned kTasksCount = 30; + std::vector> tasks; + tasks.reserve(kTasksCount); + for (unsigned i = 0; i < kTasksCount; ++i) { + tasks.push_back(engine::AsyncNoSpan(&Tests2Servers2Clients, std::ref(ssl_ctx), std::ref(ssl_ctx))); + } - server_task.Get(); - other_server_task.Get(); - other_client_task.Get(); + engine::GetAll(tasks); } UTEST(TlsWrapper, InvalidSocket) { const auto test_deadline = Deadline::FromDuration(utest::kMaxTestWaitTime); UEXPECT_THROW(static_cast(io::TlsWrapper::StartTlsClient({}, {}, test_deadline)), io::TlsException); - UEXPECT_THROW( - static_cast(io::TlsWrapper::StartTlsServer( - {}, crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline - )), - io::TlsException + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + UEXPECT_THROW(static_cast(io::TlsWrapper::StartTlsServer({}, ssl_ctx, test_deadline)), io::TlsException); } UTEST(TlsWrapper, PeerShutdown) { @@ -711,12 +724,11 @@ UTEST(TlsWrapper, PeerShutdown) { auto server_task = engine::AsyncNoSpan( [test_deadline](auto&& server) { try { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - test_deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, test_deadline); char c = 0; // Get a non-fatal error on the channel EXPECT_THROW( @@ -753,12 +765,11 @@ UTEST(TlsWrapper, PeerDisconnect) { auto server_task = engine::AsyncNoSpan( [test_deadline](auto&& server) { try { - auto tls_server = io::TlsWrapper::StartTlsServer( - std::forward(server), - crypto::LoadCertificatesChainFromString(cert), - crypto::PrivateKey::LoadFromString(key), - test_deadline + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) ); + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, test_deadline); char c = 0; // Get a non-fatal error on the channel EXPECT_THROW( @@ -790,4 +801,94 @@ UTEST(TlsWrapper, PeerDisconnect) { server_task.Get(); } +UTEST(TlsWrapper, RecvNoblock) { + const auto test_deadline = Deadline::FromDuration(utest::kMaxTestWaitTime); + + TcpListener tcp_listener; + auto [server, client] = tcp_listener.MakeSocketPair(test_deadline); + + auto server_task = engine::AsyncNoSpan( + [test_deadline](auto&& server) { + try { + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) + ); + auto tls_server = io::TlsWrapper::StartTlsServer( + std::forward(server), std::move(ssl_ctx), test_deadline + ); + EXPECT_EQ(1, tls_server.SendAll("1", 1, test_deadline)); + + char c = 0; + std::optional read_bytes; + while (true) { + read_bytes = tls_server.RecvNoblock(&c, 1); + if (read_bytes.has_value()) { + EXPECT_EQ(1, read_bytes.value()); + EXPECT_EQ('2', c); + return; + } + + ASSERT_FALSE(test_deadline.IsReached()); + } + } catch (const std::exception& e) { + LOG_ERROR() << e; + FAIL() << e.what(); + } + }, + std::move(server) + ); + + auto tls_client = io::TlsWrapper::StartTlsClient(std::move(client), {}, test_deadline); + + char c = 0; + auto read_bytes = tls_client.RecvAll(&c, 1, test_deadline); + EXPECT_EQ(1, read_bytes); + EXPECT_EQ('1', c); + + EXPECT_EQ(1, tls_client.SendAll("2", 1, test_deadline)); + server_task.Get(); +} + +UTEST(TlsWrapper, RecvNoblockNoData) { + const auto test_deadline = Deadline::FromDuration(utest::kMaxTestWaitTime); + + TcpListener tcp_listener; + auto [server, client] = tcp_listener.MakeSocketPair(test_deadline); + + auto server_task = engine::AsyncNoSpan( + [test_deadline](auto&& server) { + try { + crypto::SslCtx ssl_ctx = crypto::SslCtx::CreateServerTlsContext( + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key) + ); + auto tls_server = + io::TlsWrapper::StartTlsServer(std::forward(server), ssl_ctx, test_deadline); + + char server_char = 0; + + // Test that the call does not block if there is no data + const auto read_bytes = tls_server.RecvNoblock(&server_char, 1); + ASSERT_FALSE(read_bytes.has_value()); + EXPECT_EQ(1, tls_server.SendAll("1", 1, test_deadline)); + } catch (const std::exception& e) { + LOG_ERROR() << e; + FAIL() << e.what(); + } + }, + std::move(server) + ); + + auto tls_client = io::TlsWrapper::StartTlsClient(std::move(client), {}, test_deadline); + + char c = 0; + auto read_bytes = tls_client.RecvAll(&c, 1, test_deadline); + EXPECT_EQ(1, read_bytes); + EXPECT_EQ('1', c); + + // Test that the call does not block if there is no data or remote closed the socket + ASSERT_EQ(tls_client.RecvNoblock(&c, 1).value_or(0), 0); + + server_task.Get(); +} + USERVER_NAMESPACE_END diff --git a/core/src/engine/mutex_benchmark.cpp b/core/src/engine/mutex_benchmark.cpp index 243be2a9a655..a72379d49761 100644 --- a/core/src/engine/mutex_benchmark.cpp +++ b/core/src/engine/mutex_benchmark.cpp @@ -21,7 +21,7 @@ namespace { //////// Generic cases for benchmarking template -void generic_lock(benchmark::State& state) { +void GenericLock(benchmark::State& state) { constexpr std::size_t kMutexCount = 256; std::size_t i = 0; Mutex mutexes[kMutexCount]; @@ -45,7 +45,7 @@ void generic_lock(benchmark::State& state) { } template -void generic_unlock(benchmark::State& state) { +void GenericUnlock(benchmark::State& state) { constexpr std::size_t kMutexCount = 256; std::size_t i = 0; Mutex mutexes[kMutexCount]; @@ -73,7 +73,7 @@ void generic_unlock(benchmark::State& state) { } template -void generic_contention(benchmark::State& state) { +void GenericContention(benchmark::State& state) { std::atomic lock_unlock_count{0}; concurrent::impl::InterferenceShield m; @@ -96,7 +96,7 @@ void generic_contention(benchmark::State& state) { } template -void generic_contention_with_payload(benchmark::State& state) { +void GenericContentionWithPayload(benchmark::State& state) { std::atomic lock_unlock_count{0}; concurrent::impl::InterferenceShield m; @@ -127,64 +127,62 @@ void generic_contention_with_payload(benchmark::State& state) { // avoid any side-effects (RunStandalone spawns additional std::threads and uses // some synchronization primitives). -void mutex_coro_lock(benchmark::State& state) { - engine::RunStandalone([&] { generic_lock(state); }); +void MutexCoroLock(benchmark::State& state) { + engine::RunStandalone([&] { GenericLock(state); }); } -void mutex_std_lock(benchmark::State& state) { generic_lock(state); } +void MutexStdLock(benchmark::State& state) { GenericLock(state); } -void single_waiting_task_mutex_lock(benchmark::State& state) { - engine::RunStandalone([&] { generic_lock(state); }); +void SingleWaitingTaskMutexLock(benchmark::State& state) { + engine::RunStandalone([&] { GenericLock(state); }); } -void mutex_coro_unlock(benchmark::State& state) { - engine::RunStandalone([&] { generic_unlock(state); }); +void MutexCoroUnlock(benchmark::State& state) { + engine::RunStandalone([&] { GenericUnlock(state); }); } -void mutex_std_unlock(benchmark::State& state) { generic_lock(state); } +void MutexStdUnlock(benchmark::State& state) { GenericLock(state); } -void single_waiting_task_mutex_unlock(benchmark::State& state) { - engine::RunStandalone([&] { generic_unlock(state); }); +void SingleWaitingTaskMutexUnlock(benchmark::State& state) { + engine::RunStandalone([&] { GenericUnlock(state); }); } -void mutex_coro_contention(benchmark::State& state) { - engine::RunStandalone(state.range(0), [&] { generic_contention(state); }); +void MutexCoroContention(benchmark::State& state) { + engine::RunStandalone(state.range(0), [&] { GenericContention(state); }); } -void mutex_std_contention(benchmark::State& state) { generic_contention(state); } +void MutexStdContention(benchmark::State& state) { GenericContention(state); } -void single_waiting_task_mutex_contention(benchmark::State& state) { - engine::RunStandalone(state.range(0), [&] { generic_contention(state); }); +void SingleWaitingTaskMutexContention(benchmark::State& state) { + engine::RunStandalone(state.range(0), [&] { GenericContention(state); }); } -void mutex_coro_contention_with_payload(benchmark::State& state) { - engine::RunStandalone(state.range(0), [&] { generic_contention_with_payload(state); }); +void MutexCoroContentionWithPayload(benchmark::State& state) { + engine::RunStandalone(state.range(0), [&] { GenericContentionWithPayload(state); }); } -void mutex_std_contention_with_payload(benchmark::State& state) { generic_contention_with_payload(state); } +void MutexStdContentionWithPayload(benchmark::State& state) { GenericContentionWithPayload(state); } -void single_waiting_task_mutex_contention_with_payload(benchmark::State& state) { - engine::RunStandalone(state.range(0), [&] { - generic_contention_with_payload(state); - }); +void SingleWaitingTaskMutexContentionWithPayload(benchmark::State& state) { + engine::RunStandalone(state.range(0), [&] { GenericContentionWithPayload(state); }); } } // namespace -BENCHMARK(mutex_coro_lock); -BENCHMARK(mutex_std_lock); -BENCHMARK(single_waiting_task_mutex_lock); +BENCHMARK(MutexCoroLock); +BENCHMARK(MutexStdLock); +BENCHMARK(SingleWaitingTaskMutexLock); -BENCHMARK(mutex_coro_unlock); -BENCHMARK(mutex_std_unlock); -BENCHMARK(single_waiting_task_mutex_unlock); +BENCHMARK(MutexCoroUnlock); +BENCHMARK(MutexStdUnlock); +BENCHMARK(SingleWaitingTaskMutexUnlock); -BENCHMARK(mutex_coro_contention)->RangeMultiplier(2)->Range(1, 32); -BENCHMARK(mutex_std_contention)->RangeMultiplier(2)->Range(1, 32); -BENCHMARK(single_waiting_task_mutex_contention)->Range(1, 2); +BENCHMARK(MutexCoroContention)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(MutexStdContention)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(SingleWaitingTaskMutexContention)->Range(1, 2); -BENCHMARK(mutex_coro_contention_with_payload)->RangeMultiplier(2)->Range(1, 32); -BENCHMARK(mutex_std_contention_with_payload)->RangeMultiplier(2)->Range(1, 32); -BENCHMARK(single_waiting_task_mutex_contention_with_payload)->Range(1, 2); +BENCHMARK(MutexCoroContentionWithPayload)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(MutexStdContentionWithPayload)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(SingleWaitingTaskMutexContentionWithPayload)->Range(1, 2); USERVER_NAMESPACE_END diff --git a/core/src/engine/semaphore.cpp b/core/src/engine/semaphore.cpp index 081cca608746..4cf18ac134c5 100644 --- a/core/src/engine/semaphore.cpp +++ b/core/src/engine/semaphore.cpp @@ -105,8 +105,9 @@ void CancellableSemaphore::lock_shared_count(const Counter count) { void CancellableSemaphore::unlock_shared() { unlock_shared_count(1); } void CancellableSemaphore::unlock_shared_count(const Counter count) { - UASSERT(count > 0); - + if (count == 0) { + return; + } const auto old_acquired_locks = acquired_locks_.fetch_sub(count, std::memory_order_acq_rel); UASSERT_MSG( old_acquired_locks >= old_acquired_locks - count, @@ -175,8 +176,9 @@ CancellableSemaphore::TryLockStatus CancellableSemaphore::DoTryLock(const Counte } CancellableSemaphore::TryLockStatus CancellableSemaphore::LockFastPath(const Counter count) { - UASSERT(count > 0); - + if (count == 0) { + return TryLockStatus::kSuccess; + } const auto status = DoTryLock(count); return status; } diff --git a/core/src/engine/semaphore_benchmark.cpp b/core/src/engine/semaphore_benchmark.cpp index dc0c642115e0..e80c48617da1 100644 --- a/core/src/engine/semaphore_benchmark.cpp +++ b/core/src/engine/semaphore_benchmark.cpp @@ -12,7 +12,7 @@ USERVER_NAMESPACE_BEGIN /// [RunStandalone sample] -void semaphore_lock(benchmark::State& state) { +void SemaphoreLock(benchmark::State& state) { engine::RunStandalone([&]() { std::size_t i = 0; engine::Semaphore sem{std::numeric_limits::max()}; @@ -27,10 +27,10 @@ void semaphore_lock(benchmark::State& state) { } }); } -BENCHMARK(semaphore_lock); +BENCHMARK(SemaphoreLock); /// [RunStandalone sample] -void semaphore_unlock(benchmark::State& state) { +void SemaphoreUnlock(benchmark::State& state) { engine::RunStandalone([&]() { unsigned i = 0; engine::Semaphore sem{std::numeric_limits::max()}; @@ -58,9 +58,9 @@ void semaphore_unlock(benchmark::State& state) { } }); } -BENCHMARK(semaphore_unlock); +BENCHMARK(SemaphoreUnlock); -void semaphore_lock_unlock_contention(benchmark::State& state) { +void SemaphoreLockUnlockContention(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { engine::Semaphore sem{1}; @@ -72,9 +72,9 @@ void semaphore_lock_unlock_contention(benchmark::State& state) { }); }); } -BENCHMARK(semaphore_lock_unlock_contention)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(SemaphoreLockUnlockContention)->RangeMultiplier(2)->Range(1, 32); -void semaphore_lock_unlock_payload_contention(benchmark::State& state) { +void SemaphoreLockUnlockPayloadContention(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { engine::Semaphore sem{1}; @@ -90,9 +90,9 @@ void semaphore_lock_unlock_payload_contention(benchmark::State& state) { }); }); } -BENCHMARK(semaphore_lock_unlock_payload_contention)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(SemaphoreLockUnlockPayloadContention)->RangeMultiplier(2)->Range(1, 32); -void semaphore_lock_unlock_coro_contention(benchmark::State& state) { +void SemaphoreLockUnlockCoroContention(benchmark::State& state) { engine::RunStandalone(4, [&] { engine::Semaphore sem{1}; @@ -104,9 +104,9 @@ void semaphore_lock_unlock_coro_contention(benchmark::State& state) { }); }); } -BENCHMARK(semaphore_lock_unlock_coro_contention)->RangeMultiplier(2)->Range(1, 1024); +BENCHMARK(SemaphoreLockUnlockCoroContention)->RangeMultiplier(2)->Range(1, 1024); -void semaphore_lock_unlock_payload_coro_contention(benchmark::State& state) { +void SemaphoreLockUnlockPayloadCoroContention(benchmark::State& state) { engine::RunStandalone(4, [&] { engine::Semaphore sem{1}; @@ -122,9 +122,9 @@ void semaphore_lock_unlock_payload_coro_contention(benchmark::State& state) { }); }); } -BENCHMARK(semaphore_lock_unlock_payload_coro_contention)->RangeMultiplier(2)->Range(1, 1024); +BENCHMARK(SemaphoreLockUnlockPayloadCoroContention)->RangeMultiplier(2)->Range(1, 1024); -void semaphore_lock_unlock_st_coro_contention(benchmark::State& state) { +void SemaphoreLockUnlockStCoroContention(benchmark::State& state) { engine::RunStandalone([&]() { engine::Semaphore sem{1}; @@ -137,6 +137,6 @@ void semaphore_lock_unlock_st_coro_contention(benchmark::State& state) { }); }); } -BENCHMARK(semaphore_lock_unlock_st_coro_contention)->RangeMultiplier(2)->Range(1, 1024); +BENCHMARK(SemaphoreLockUnlockStCoroContention)->RangeMultiplier(2)->Range(1, 1024); USERVER_NAMESPACE_END diff --git a/core/src/engine/semaphore_test.cpp b/core/src/engine/semaphore_test.cpp index 7eb2240e864a..6b8baea7955a 100644 --- a/core/src/engine/semaphore_test.cpp +++ b/core/src/engine/semaphore_test.cpp @@ -439,4 +439,33 @@ UTEST(SemaphoreLock, SampleSemaphore) { /// [Sample engine::Semaphore usage] } +UTEST(Semaphore, LockZeroUnitsWithRemaining) { + engine::Semaphore sem{1}; + EXPECT_EQ(sem.GetCapacity(), 1); + EXPECT_EQ(sem.RemainingApprox(), 1); + EXPECT_EQ(sem.UsedApprox(), 0); + + sem.lock_shared_count(0); + EXPECT_EQ(sem.GetCapacity(), 1); + EXPECT_EQ(sem.RemainingApprox(), 1); + EXPECT_EQ(sem.UsedApprox(), 0); + sem.unlock_shared_count(0); +} + +UTEST(Semaphore, LockZeroUnitsWithoutRemaining) { + engine::Semaphore sem{1}; + sem.lock_shared_count(1); + EXPECT_EQ(sem.GetCapacity(), 1); + EXPECT_EQ(sem.RemainingApprox(), 0); + EXPECT_EQ(sem.UsedApprox(), 1); + + sem.lock_shared_count(0); + EXPECT_EQ(sem.GetCapacity(), 1); + EXPECT_EQ(sem.RemainingApprox(), 0); + EXPECT_EQ(sem.UsedApprox(), 1); + sem.unlock_shared_count(0); + + sem.unlock_shared_count(1); +} + USERVER_NAMESPACE_END diff --git a/core/src/engine/shared_mutex_benchmark.cpp b/core/src/engine/shared_mutex_benchmark.cpp index 3c31a4fd485b..bb27de345c5a 100644 --- a/core/src/engine/shared_mutex_benchmark.cpp +++ b/core/src/engine/shared_mutex_benchmark.cpp @@ -7,7 +7,7 @@ USERVER_NAMESPACE_BEGIN -void shared_mutex_benchmark(benchmark::State& state) { +void SharedMutexBenchmark(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { int variable = 0; engine::SharedMutex mutex; @@ -26,6 +26,6 @@ void shared_mutex_benchmark(benchmark::State& state) { }); }); } -BENCHMARK(shared_mutex_benchmark)->DenseRange(1, 6); +BENCHMARK(SharedMutexBenchmark)->DenseRange(1, 6); USERVER_NAMESPACE_END diff --git a/core/src/engine/single_use_event.cpp b/core/src/engine/single_use_event.cpp index f7e017e1499c..4fb23fafb13a 100644 --- a/core/src/engine/single_use_event.cpp +++ b/core/src/engine/single_use_event.cpp @@ -27,9 +27,8 @@ void SingleUseEvent::Wait() { "Timeout is not expected here due to unreachable " "Deadline at Sleep" ); -#ifdef NDEBUG - [[fallthrough]]; -#endif + // Never reaches + break; case FutureStatus::kCancelled: throw WaitInterruptedException(current_task::CancellationReason()); } diff --git a/core/src/engine/sleep_benchmark.cpp b/core/src/engine/sleep_benchmark.cpp index 0c8d8e5aaced..9620517fd4ba 100644 --- a/core/src/engine/sleep_benchmark.cpp +++ b/core/src/engine/sleep_benchmark.cpp @@ -9,7 +9,7 @@ using namespace std::chrono_literals; USERVER_NAMESPACE_BEGIN -void sleep_benchmark_us(benchmark::State& state) { +void SleepBenchmarkUs(benchmark::State& state) { engine::RunStandalone([&] { const std::chrono::microseconds sleep_duration{state.range(0)}; for ([[maybe_unused]] auto _ : state) { @@ -18,9 +18,9 @@ void sleep_benchmark_us(benchmark::State& state) { } }); } -BENCHMARK(sleep_benchmark_us)->RangeMultiplier(2)->Range(1, 1024 * 128)->Unit(benchmark::kMicrosecond); +BENCHMARK(SleepBenchmarkUs)->RangeMultiplier(2)->Range(1, 1024 * 128)->Unit(benchmark::kMicrosecond); -void run_in_ev_loop_benchmark(benchmark::State& state) { +void RunInEvLoopBenchmark(benchmark::State& state) { engine::RunStandalone([&] { auto& ev_thread = engine::current_task::GetEventThread(); for ([[maybe_unused]] auto _ : state) { @@ -28,9 +28,9 @@ void run_in_ev_loop_benchmark(benchmark::State& state) { } }); } -BENCHMARK(run_in_ev_loop_benchmark); +BENCHMARK(RunInEvLoopBenchmark); -[[maybe_unused]] void successful_wait_for_benchmark(benchmark::State& state) { +[[maybe_unused]] void SuccessfulWaitForBenchmark(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { auto task = engine::AsyncNoSpan([] { engine::Yield(); }); @@ -40,9 +40,9 @@ BENCHMARK(run_in_ev_loop_benchmark); } }); } -BENCHMARK(successful_wait_for_benchmark); +BENCHMARK(SuccessfulWaitForBenchmark); -void unreached_task_deadline_benchmark(benchmark::State& state, bool has_task_deadline) { +void UnreachedTaskDeadlineBenchmark(benchmark::State& state, bool has_task_deadline) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { const auto sleep_deadline = engine::Deadline::FromDuration(20s); @@ -56,7 +56,7 @@ void unreached_task_deadline_benchmark(benchmark::State& state, bool has_task_de } }); } -BENCHMARK_CAPTURE(unreached_task_deadline_benchmark, no_task_deadline, false); -BENCHMARK_CAPTURE(unreached_task_deadline_benchmark, unreached_task_deadline, true); +BENCHMARK_CAPTURE(UnreachedTaskDeadlineBenchmark, no_task_deadline, false); +BENCHMARK_CAPTURE(UnreachedTaskDeadlineBenchmark, unreached_task_deadline, true); USERVER_NAMESPACE_END diff --git a/core/src/engine/task/async_benchmark.cpp b/core/src/engine/task/async_benchmark.cpp index fd640ddc3b54..0ce1598272ab 100644 --- a/core/src/engine/task/async_benchmark.cpp +++ b/core/src/engine/task/async_benchmark.cpp @@ -20,7 +20,7 @@ using WrappedSpanCall = utils::impl::WrappedCallImplTypeRangeMultiplier(2)->Range(1, 32); +BENCHMARK(AsyncComparisonsCoro)->RangeMultiplier(2)->Range(1, 32); -void wrap_call_single(benchmark::State& state) { +void WrapCallSingle(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { WrappedSpanCall(utils::impl::SpanLazyPrvalue(""), []() {}); } }); } -BENCHMARK(wrap_call_single); +BENCHMARK(WrapCallSingle); -void wrap_call_multiple(benchmark::State& state) { +void WrapCallMultiple(benchmark::State& state) { engine::RunStandalone([&] { constexpr std::size_t kInMemoryInstancesCount = 100; utils::FixedArray> calls{kInMemoryInstancesCount}; @@ -66,9 +66,9 @@ void wrap_call_multiple(benchmark::State& state) { } }); } -BENCHMARK(wrap_call_multiple); +BENCHMARK(WrapCallMultiple); -void wrap_call_and_perform(benchmark::State& state) { +void WrapCallAndPerform(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { WrappedSpanCall wrapped_call{utils::impl::SpanLazyPrvalue(""), []() {}}; @@ -81,9 +81,9 @@ void wrap_call_and_perform(benchmark::State& state) { } }); } -BENCHMARK(wrap_call_and_perform); +BENCHMARK(WrapCallAndPerform); -void async_comparisons_coro_spanned(benchmark::State& state) { +void AsyncComparisonsCoroSpanned(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { std::uint64_t constructed_joined_count = 0; for ([[maybe_unused]] auto _ : state) { @@ -93,6 +93,6 @@ void async_comparisons_coro_spanned(benchmark::State& state) { benchmark::DoNotOptimize(constructed_joined_count); }); } -BENCHMARK(async_comparisons_coro_spanned)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(AsyncComparisonsCoroSpanned)->RangeMultiplier(2)->Range(1, 32); USERVER_NAMESPACE_END diff --git a/core/src/engine/task/task_benchmark.cpp b/core/src/engine/task/task_benchmark.cpp index a75bfb11f9c9..ad36d88ee33d 100644 --- a/core/src/engine/task/task_benchmark.cpp +++ b/core/src/engine/task/task_benchmark.cpp @@ -23,16 +23,16 @@ USERVER_NAMESPACE_BEGIN -void engine_task_create(benchmark::State& state) { +void EngineTaskCreate(benchmark::State& state) { // We use 2 threads to ensure that detached tasks are deallocated, // otherwise this benchmark OOMs after some time. engine::RunStandalone(2, [&] { for ([[maybe_unused]] auto _ : state) engine::DetachUnscopedUnsafe(engine::AsyncNoSpan([]() {})); }); } -BENCHMARK(engine_task_create); +BENCHMARK(EngineTaskCreate); -void engine_task_yield_single_thread(benchmark::State& state) { +void EngineTaskYieldSingleThread(benchmark::State& state) { engine::RunStandalone([&] { RunParallelBenchmark(state, [](auto& range) { for ([[maybe_unused]] auto _ : range) { @@ -41,9 +41,9 @@ void engine_task_yield_single_thread(benchmark::State& state) { }); }); } -BENCHMARK(engine_task_yield_single_thread)->RangeMultiplier(2)->Range(1, 128); +BENCHMARK(EngineTaskYieldSingleThread)->RangeMultiplier(2)->Range(1, 128); -void engine_task_yield_multiple_threads(benchmark::State& state) { +void EngineTaskYieldMultipleThreads(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { std::atomic total_yields{0}; @@ -61,9 +61,9 @@ void engine_task_yield_multiple_threads(benchmark::State& state) { benchmark::Counter(static_cast(total_yields) / state.range(0), benchmark::Counter::kIsRate); }); } -BENCHMARK(engine_task_yield_multiple_threads)->RangeMultiplier(2)->Range(1, 32)->Arg(6)->Arg(12); +BENCHMARK(EngineTaskYieldMultipleThreads)->RangeMultiplier(2)->Range(1, 32)->Arg(6)->Arg(12); -void engine_task_yield_multiple_task_processors(benchmark::State& state) { +void EngineTaskYieldMultipleTaskProcessors(benchmark::State& state) { engine::RunStandalone([&] { auto tp_pool = engine::SingleThreadedTaskProcessorsPool::MakeForTests(state.range(0) - 1); @@ -98,14 +98,14 @@ void engine_task_yield_multiple_task_processors(benchmark::State& state) { benchmark::Counter(static_cast(yields_performed) / state.range(0), benchmark::Counter::kIsRate); }); } -BENCHMARK(engine_task_yield_multiple_task_processors)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(EngineTaskYieldMultipleTaskProcessors)->RangeMultiplier(2)->Range(1, 32); -void thread_yield(benchmark::State& state) { +void ThreadYield(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) std::this_thread::yield(); } -BENCHMARK(thread_yield)->RangeMultiplier(2)->ThreadRange(1, 32); +BENCHMARK(ThreadYield)->RangeMultiplier(2)->ThreadRange(1, 32); -void engine_multiple_tasks_multiple_threads(benchmark::State& state) { +void EngineMultipleTasksMultipleThreads(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { std::atomic tasks_count_total = 0; RunParallelBenchmark(state, [&](auto& range) { @@ -120,9 +120,9 @@ void engine_multiple_tasks_multiple_threads(benchmark::State& state) { benchmark::DoNotOptimize(tasks_count_total); }); } -BENCHMARK(engine_multiple_tasks_multiple_threads)->RangeMultiplier(2)->Range(1, 32)->Arg(6)->Arg(12); +BENCHMARK(EngineMultipleTasksMultipleThreads)->RangeMultiplier(2)->Range(1, 32)->Arg(6)->Arg(12); -void engine_multiple_yield_two_task_processor_no_extra_wakeups(benchmark::State& state) { +void EngineMultipleYieldTwoTaskProcessorNoExtraWakeups(benchmark::State& state) { engine::RunStandalone([&] { std::vector> processors; for (int i = 0; i < 2; i++) { @@ -170,9 +170,9 @@ void engine_multiple_yield_two_task_processor_no_extra_wakeups(benchmark::State& benchmark::Counter(static_cast(yields_performed) / state.range(0), benchmark::Counter::kIsRate); }); } -BENCHMARK(engine_multiple_yield_two_task_processor_no_extra_wakeups)->RangeMultiplier(2)->Range(2, 32)->Arg(6)->Arg(12); +BENCHMARK(EngineMultipleYieldTwoTaskProcessorNoExtraWakeups)->RangeMultiplier(2)->Range(2, 32)->Arg(6)->Arg(12); -void engine_tasks_from_another_task_processor(benchmark::State& state) { +void EngineTasksFromAnotherTaskProcessor(benchmark::State& state) { engine::RunStandalone([&] { engine::TaskProcessorConfig proc_config; proc_config.name = "benchmark"; @@ -192,6 +192,6 @@ void engine_tasks_from_another_task_processor(benchmark::State& state) { } }); } -BENCHMARK(engine_tasks_from_another_task_processor)->RangeMultiplier(2)->Range(2, 32)->Arg(6)->Arg(12); +BENCHMARK(EngineTasksFromAnotherTaskProcessor)->RangeMultiplier(2)->Range(2, 32)->Arg(6)->Arg(12); USERVER_NAMESPACE_END diff --git a/core/src/engine/task/task_context.hpp b/core/src/engine/task/task_context.hpp index 70430929e960..6ea1da48c011 100644 --- a/core/src/engine/task/task_context.hpp +++ b/core/src/engine/task/task_context.hpp @@ -245,8 +245,8 @@ class TaskContext final : public ContextAccessor { // refcounter for resources and memory deallocation std::atomic intrusive_refcount_{1}; - friend void intrusive_ptr_add_ref(TaskContext* p) noexcept; - friend void intrusive_ptr_release(TaskContext* p) noexcept; + friend void intrusive_ptr_add_ref(TaskContext* p) noexcept; // NOLINT(readability-identifier-naming) + friend void intrusive_ptr_release(TaskContext* p) noexcept; // NOLINT(readability-identifier-naming) public: using WaitListHook = typename boost::intrusive::make_list_member_hook< @@ -256,8 +256,8 @@ class TaskContext final : public ContextAccessor { WaitListHook wait_list_hook; }; -void intrusive_ptr_add_ref(TaskContext* p) noexcept; -void intrusive_ptr_release(TaskContext* p) noexcept; +void intrusive_ptr_add_ref(TaskContext* p) noexcept; // NOLINT(readability-identifier-naming) +void intrusive_ptr_release(TaskContext* p) noexcept; // NOLINT(readability-identifier-naming) bool HasWaitSucceeded(TaskContext::WakeupSource) noexcept; diff --git a/core/src/engine/task/task_with_result_test.cpp b/core/src/engine/task/task_with_result_test.cpp index 9cc652537f20..555e9ab1cf9a 100644 --- a/core/src/engine/task/task_with_result_test.cpp +++ b/core/src/engine/task/task_with_result_test.cpp @@ -57,6 +57,19 @@ UTEST(TaskWithResult, LifetimeIfTaskCancelledBeforeStart) { EXPECT_TRUE(is_func_destroyed); } +UTEST(TaskWithResult, InvalidationOfCancelledTask) { + auto task = utils::Async("some_task", [value = std::move("some_value")] { + engine::InterruptibleSleepFor(std::chrono::milliseconds(100)); + return value; + }); + + task.RequestCancel(); + task.Wait(); + EXPECT_EQ(task.GetState(), engine::Task::State::kCancelled); + UEXPECT_THROW(task.Get(), engine::TaskCancelledException); + EXPECT_FALSE(task.IsValid()); +} + static_assert(std::is_move_constructible_v>); static_assert(std::is_move_assignable_v>); diff --git a/core/src/engine/wait_any_test.cpp b/core/src/engine/wait_any_test.cpp index ba86f0e3d928..3294b6292996 100644 --- a/core/src/engine/wait_any_test.cpp +++ b/core/src/engine/wait_any_test.cpp @@ -69,6 +69,17 @@ UTEST(WaitAny, Cancelled) { task.SyncCancel(); } +UTEST(WaitAny, VectorWithCancelledTask) { + std::vector> tasks; + tasks.push_back(engine::AsyncNoSpan([] { return std::string{"some_value"}; })); + tasks[0].RequestCancel(); + + auto task_idx_opt = engine::WaitAny(tasks); + EXPECT_TRUE(!!task_idx_opt); + UEXPECT_THROW(tasks[*task_idx_opt].Get(), engine::TaskCancelledException); + EXPECT_EQ(engine::WaitAny(tasks), std::nullopt); +} + UTEST(WaitAny, WaitAnyFor) { engine::TaskWithResult tasks[] = { engine::AsyncNoSpan([] { diff --git a/core/src/fs/fs_cache_client.cpp b/core/src/fs/fs_cache_client.cpp index dd0cf6627ecb..a8d84eeceebd 100644 --- a/core/src/fs/fs_cache_client.cpp +++ b/core/src/fs/fs_cache_client.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/core/src/fs/temp_file.cpp b/core/src/fs/temp_file.cpp index d03c69e4290e..91c8ce45757f 100644 --- a/core/src/fs/temp_file.cpp +++ b/core/src/fs/temp_file.cpp @@ -1,6 +1,7 @@ #include #include +#include USERVER_NAMESPACE_BEGIN @@ -27,7 +28,13 @@ TempFile::Create(std::string_view parent_path, std::string_view name_prefix, eng }; } -TempFile::~TempFile() { std::move(*this).Remove(); } +TempFile::~TempFile() noexcept { + try { + std::move(*this).Remove(); + } catch (const std::exception& ex) { + LOG_ERROR() << "fs::~TempFile failed with exception:" << ex; + } +} TempFile TempFile::Adopt(std::string path, engine::TaskProcessor& fs_task_processor) { return {fs_task_processor, blocking::TempFile::Adopt(std::move(path))}; diff --git a/core/src/fs/temp_file_test.cpp b/core/src/fs/temp_file_test.cpp new file mode 100644 index 000000000000..7a14a19ecfea --- /dev/null +++ b/core/src/fs/temp_file_test.cpp @@ -0,0 +1,75 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace { +using FsTempFileWithLog = utest::LogCaptureFixture<>; +} + +UTEST_F(FsTempFileWithLog, TempFileDestructorWithException) { + std::string parent_dir; + { + const auto parent = fs::blocking::TempDirectory::Create(); + parent_dir = parent.GetPath(); + + { + // Create temp file in temporaty directory + const auto file = fs::TempFile::Create( + parent_dir, "TempFileDestructorWithException", engine::current_task::GetTaskProcessor() + ); + + const auto dir_status = boost::filesystem::status(parent_dir); + const auto original_perms = dir_status.permissions(); + + LOG_DEBUG() << "Original directory permissions: " << static_cast(original_perms); + + // Check that the directory has write permissions + ASSERT_TRUE((original_perms & boost::filesystem::perms::owner_write) != boost::filesystem::perms::no_perms); + + // Change permissions for the directory + boost::filesystem::permissions( + parent_dir, + boost::filesystem::perms::owner_read | boost::filesystem::perms::owner_exe | + boost::filesystem::perms::group_read | boost::filesystem::perms::group_exe | + boost::filesystem::perms::others_read | boost::filesystem::perms::others_exe + ); + LOG_DEBUG() << "Changed directory permissions, write is now forbidden"; + + // Check that the file can not be removed + try { + boost::filesystem::remove(file.GetPath()); + FAIL() << "Expected exception when removing file"; + } catch (const std::exception& ex) { + LOG_DEBUG() << "Confirmed file cannot be removed"; + } + + // We don't expect that ~TempFile to throw an exception + } + + // Check that log with error from ~TempFile exists + EXPECT_THAT(GetLogCapture().Filter("fs::~TempFile failed with exception:"), testing::SizeIs(1)); + + // Rollback directory permissions + boost::filesystem::permissions( + parent_dir, + boost::filesystem::perms::owner_read | boost::filesystem::perms::owner_write | + boost::filesystem::perms::owner_exe | boost::filesystem::perms::group_read | + boost::filesystem::perms::group_write | boost::filesystem::perms::group_exe | + boost::filesystem::perms::others_read | boost::filesystem::perms::others_write | + boost::filesystem::perms::others_exe + ); + LOG_DEBUG() << "Changed directory permissions, write is now available"; + } + ASSERT_FALSE(boost::filesystem::exists(parent_dir)); +} + +USERVER_NAMESPACE_END diff --git a/core/src/logging/impl/file_sinks_benchmark.cpp b/core/src/logging/impl/file_sinks_benchmark.cpp index c82259d75f6d..c9c66b3d86a9 100644 --- a/core/src/logging/impl/file_sinks_benchmark.cpp +++ b/core/src/logging/impl/file_sinks_benchmark.cpp @@ -10,7 +10,7 @@ USERVER_NAMESPACE_BEGIN constexpr ssize_t kCountLogs = 100000; -void check_file_sink(benchmark::State& state) { +void CheckFileSink(benchmark::State& state) { const auto temp_root = fs::blocking::TempDirectory::Create(); const std::string filename = temp_root.GetPath() + "/temp_file_" + std::to_string(utils::Rand()); auto sink = logging::impl::FileSink(filename); @@ -21,9 +21,9 @@ void check_file_sink(benchmark::State& state) { } sink.Flush(); } -BENCHMARK(check_file_sink); +BENCHMARK(CheckFileSink); -void check_buffered_file_sink(benchmark::State& state) { +void CheckBufferedFileSink(benchmark::State& state) { const auto temp_root = fs::blocking::TempDirectory::Create(); const std::string filename = temp_root.GetPath() + "/temp_file_" + std::to_string(utils::Rand()); auto sink = logging::impl::BufferedFileSink(filename); @@ -34,6 +34,6 @@ void check_buffered_file_sink(benchmark::State& state) { } sink.Flush(); } -BENCHMARK(check_buffered_file_sink); +BENCHMARK(CheckBufferedFileSink); USERVER_NAMESPACE_END diff --git a/core/src/logging/log_message_benchmark.cpp b/core/src/logging/log_message_benchmark.cpp index 9ef43d6cedde..2c8c2aef9dae 100644 --- a/core/src/logging/log_message_benchmark.cpp +++ b/core/src/logging/log_message_benchmark.cpp @@ -91,14 +91,14 @@ BENCHMARK_DEFINE_F(LogHelperBenchmark, LogCheck)(benchmark::State& state) { BENCHMARK_REGISTER_F(LogHelperBenchmark, LogCheck); struct StreamedStruct { - int64_t intVal; - std::string stringVal; + int64_t int_val; + std::string string_val; }; std::ostream& operator<<(std::ostream& os, const StreamedStruct& value) { const std::ostream::sentry s(os); if (s) { - os << value.intVal << " " << value.stringVal; + os << value.int_val << " " << value.string_val; } return os; } diff --git a/core/src/logging/logger.cpp b/core/src/logging/logger.cpp index 2d333f4115a0..6603f0494c4a 100644 --- a/core/src/logging/logger.cpp +++ b/core/src/logging/logger.cpp @@ -66,7 +66,7 @@ LoggerPtr MakeFileLogger(const std::string& name, const std::string& path, Forma return MakeSimpleLogger(name, std::make_unique(path), level, format); } -namespace impl::default_ { +namespace impl { bool DoShouldLog(Level level) noexcept { const auto* const span = tracing::Span::CurrentSpanUnchecked(); @@ -98,7 +98,7 @@ void PrependCommonTags(TagWriter writer, Level logger_level) { } } -} // namespace impl::default_ +} // namespace impl } // namespace logging diff --git a/core/src/logging/tp_logger.cpp b/core/src/logging/tp_logger.cpp index c527e591b3e7..380f2bc0d5d6 100644 --- a/core/src/logging/tp_logger.cpp +++ b/core/src/logging/tp_logger.cpp @@ -154,9 +154,9 @@ void TpLogger::Log(Level level, impl::formatters::LoggerItemRef item) { } } -void TpLogger::PrependCommonTags(TagWriter writer) const { impl::default_::PrependCommonTags(writer, GetLevel()); } +void TpLogger::PrependCommonTags(TagWriter writer) const { impl::PrependCommonTags(writer, GetLevel()); } -bool TpLogger::DoShouldLog(Level level) const noexcept { return impl::default_::DoShouldLog(level); } +bool TpLogger::DoShouldLog(Level level) const noexcept { return impl::DoShouldLog(level); } void TpLogger::AddSink(impl::SinkPtr&& sink) { UASSERT(sink); diff --git a/core/src/rcu/atomic_shared_ptr_benchmark.cpp b/core/src/rcu/atomic_shared_ptr_benchmark.cpp index 240e2656136f..56745761c048 100644 --- a/core/src/rcu/atomic_shared_ptr_benchmark.cpp +++ b/core/src/rcu/atomic_shared_ptr_benchmark.cpp @@ -28,7 +28,7 @@ class AtomicSharedPtr { std::shared_ptr storage_; }; -void atomic_shared_ptr_read(benchmark::State& state) { +void AtomicSharedPtrRead(benchmark::State& state) { engine::RunStandalone([&] { const AtomicSharedPtr ptr(std::make_unique(1)); @@ -38,9 +38,9 @@ void atomic_shared_ptr_read(benchmark::State& state) { } }); } -BENCHMARK(atomic_shared_ptr_read); +BENCHMARK(AtomicSharedPtrRead); -void atomic_shared_ptr_contention(benchmark::State& state) { +void AtomicSharedPtrContention(benchmark::State& state) { engine::RunStandalone(state.range(0), [&] { std::atomic run{true}; AtomicSharedPtr> ptr{std::make_shared>()}; @@ -74,6 +74,6 @@ void atomic_shared_ptr_contention(benchmark::State& state) { run = false; }); } -BENCHMARK(atomic_shared_ptr_contention)->RangeMultiplier(2)->Ranges({{2, 32}, {false, true}}); +BENCHMARK(AtomicSharedPtrContention)->RangeMultiplier(2)->Ranges({{2, 32}, {false, true}}); USERVER_NAMESPACE_END diff --git a/core/src/rcu/rcu_benchmark.cpp b/core/src/rcu/rcu_benchmark.cpp index 855989e640d6..6f2fa91aebb1 100644 --- a/core/src/rcu/rcu_benchmark.cpp +++ b/core/src/rcu/rcu_benchmark.cpp @@ -15,7 +15,7 @@ USERVER_NAMESPACE_BEGIN template -void rcu_read(benchmark::State& state) { +void RcuRead(benchmark::State& state) { engine::RunStandalone([&] { rcu::Variable vars[VariableCount]; { @@ -34,12 +34,12 @@ void rcu_read(benchmark::State& state) { } }); } -BENCHMARK_TEMPLATE(rcu_read, 1); -BENCHMARK_TEMPLATE(rcu_read, 2); -BENCHMARK_TEMPLATE(rcu_read, 4); +BENCHMARK_TEMPLATE(RcuRead, 1); +BENCHMARK_TEMPLATE(RcuRead, 2); +BENCHMARK_TEMPLATE(RcuRead, 4); template -void rcu_write(benchmark::State& state) { +void RcuWrite(benchmark::State& state) { engine::RunStandalone([&] { rcu::Variable vars[VariableCount]; @@ -50,11 +50,11 @@ void rcu_write(benchmark::State& state) { } }); } -BENCHMARK_TEMPLATE(rcu_write, 1); -BENCHMARK_TEMPLATE(rcu_write, 2); -BENCHMARK_TEMPLATE(rcu_write, 4); +BENCHMARK_TEMPLATE(RcuWrite, 1); +BENCHMARK_TEMPLATE(RcuWrite, 2); +BENCHMARK_TEMPLATE(RcuWrite, 4); -void rcu_contention(benchmark::State& state) { +void RcuContention(benchmark::State& state) { const std::size_t readers_count = state.range(0); const std::size_t writers_count = state.range(1); const std::size_t kept_readable_pointers_count = state.range(2); @@ -112,12 +112,9 @@ void rcu_contention(benchmark::State& state) { } }); } -BENCHMARK(rcu_contention) - ->RangeMultiplier(2) - ->Ranges({{1, 16}, {0, 1}, {1, 4}}) - ->Ranges({{2048, 2048}, {0, 1}, {1, 4}}); +BENCHMARK(RcuContention)->RangeMultiplier(2)->Ranges({{1, 16}, {0, 1}, {1, 4}})->Ranges({{2048, 2048}, {0, 1}, {1, 4}}); -void rcu_of_shared_ptr(benchmark::State& state) { +void RcuOfSharedPtr(benchmark::State& state) { const std::size_t readers_count = state.range(0); engine::RunStandalone(readers_count, [&] { @@ -131,6 +128,6 @@ void rcu_of_shared_ptr(benchmark::State& state) { }); }); } -BENCHMARK(rcu_of_shared_ptr)->RangeMultiplier(2)->Range(1, 32); +BENCHMARK(RcuOfSharedPtr)->RangeMultiplier(2)->Range(1, 32); USERVER_NAMESPACE_END diff --git a/core/src/rcu/rcu_test.cpp b/core/src/rcu/rcu_test.cpp index dd6c4941dc23..292202fe763f 100644 --- a/core/src/rcu/rcu_test.cpp +++ b/core/src/rcu/rcu_test.cpp @@ -217,7 +217,7 @@ UTEST(Rcu, ReadablePtrMoveAssign) { UTEST(Rcu, NoCopy) { struct X { - X(int x_, bool y_) : x(x_), y(y_) {} + X(int x, bool y) : x(x), y(y) {} X(X&&) = default; X(const X&) = delete; diff --git a/core/src/server/component.cpp b/core/src/server/component.cpp index 54092d6fe7bb..061832739d5a 100644 --- a/core/src/server/component.cpp +++ b/core/src/server/component.cpp @@ -79,7 +79,7 @@ additionalProperties: false description: set to logger name from components::Logging component to write access logs in TSKV format into it; do not set to avoid writing access logs max_response_size_in_flight: type: integer - description: set it to the size of response in bytes and the component will drop bigger responses from handlers that allow throttling + description: drop incomming requests if the handler allows throttling and the size of waiting for send responses in bytes is greater than this value server-name: type: string description: value to send in HTTP Server header diff --git a/core/src/server/handlers/auth/digest/auth_checker_base.cpp b/core/src/server/handlers/auth/digest/auth_checker_base.cpp index 4c82a777f02b..c4ee87a3de33 100644 --- a/core/src/server/handlers/auth/digest/auth_checker_base.cpp +++ b/core/src/server/handlers/auth/digest/auth_checker_base.cpp @@ -44,8 +44,7 @@ class ServerDigestSecretKey { const ServerDigestAuthSecret& GetSecretKey() const { if (!secret_key_.has_value()) { throw std::runtime_error( - "Secret key storage is missing. Field " - "'http_server_digest_auth_secret' was missing in json." + "Secret key storage is missing. Field 'http_server_digest_auth_secret' was missing in json." ); } @@ -61,31 +60,47 @@ class ServerDigestSecretKey { UserData::UserData(HA1 ha1, std::string nonce, TimePoint timestamp, std::int64_t nonce_count) : ha1(std::move(ha1)), nonce(std::move(nonce)), timestamp(timestamp), nonce_count(nonce_count) {} -Hasher::Hasher(std::string_view algorithm, const SecdistConfig& secdist_config) : secdist_config_(secdist_config) { - switch (kHashAlgToType.TryFindICase(algorithm).value_or(HashAlgTypes::kUnknown)) { - case HashAlgTypes::kMD5: - hash_algorithm_ = &crypto::hash::weak::Md5; - break; - case HashAlgTypes::kSHA256: - hash_algorithm_ = &crypto::hash::Sha256; - break; - case HashAlgTypes::kSHA512: - hash_algorithm_ = &crypto::hash::Sha512; - break; - default: - throw std::runtime_error("Unknown hash algorithm"); +Hasher::Hasher(std::string_view algorithm, const SecdistConfig& secdist_config) + : hash_algorithm_{kHashAlgToType.TryFindICase(algorithm).value_or(HashAlgTypes::kUnknown)}, + secdist_config_(secdist_config) { + if (hash_algorithm_ == HashAlgTypes::kUnknown) { + throw std::runtime_error("Unknown hash algorithm"); } } std::string Hasher::GenerateNonce(std::string_view etag) const { - auto timestamp = std::to_string(std::chrono::system_clock::now().time_since_epoch().count()); - return GetHash(fmt::format( - "{}:{}:{}", timestamp, etag, secdist_config_.Get().GetSecretKey().GetUnderlying() - )); + const auto timestamp = std::to_string(std::chrono::system_clock::now().time_since_epoch().count()); + return GetHash( + {timestamp, ":", etag, ":", secdist_config_.Get().GetSecretKey().GetUnderlying()} + ); } std::string Hasher::GetHash(std::string_view data) const { - return hash_algorithm_(data, crypto::hash::OutputEncoding::kHex); + switch (hash_algorithm_) { + case HashAlgTypes::kMD5: + return crypto::hash::weak::Md5(data, crypto::hash::OutputEncoding::kHex); + case HashAlgTypes::kSHA256: + return crypto::hash::Sha256(data, crypto::hash::OutputEncoding::kHex); + case HashAlgTypes::kSHA512: + return crypto::hash::Sha512(data, crypto::hash::OutputEncoding::kHex); + case HashAlgTypes::kUnknown: + UASSERT_MSG(false, "Unknown hash algorithm"); + } + UINVARIANT(false, "Unknown hash algorithm"); +} + +std::string Hasher::GetHash(std::initializer_list data) const { + switch (hash_algorithm_) { + case HashAlgTypes::kMD5: + return crypto::hash::weak::Md5(data, crypto::hash::OutputEncoding::kHex); + case HashAlgTypes::kSHA256: + return crypto::hash::Sha256(data, crypto::hash::OutputEncoding::kHex); + case HashAlgTypes::kSHA512: + return crypto::hash::Sha512(data, crypto::hash::OutputEncoding::kHex); + case HashAlgTypes::kUnknown: + UASSERT_MSG(false, "Unknown hash algorithm"); + } + UINVARIANT(false, "Unknown hash algorithm"); } AuthCheckerBase::AuthCheckerBase( @@ -275,19 +290,21 @@ std::string AuthCheckerBase::CalculateDigest( ha1 = fmt::format("{}:{}:{}", ha1, client_context.nonce, client_context.cnonce); } - auto a2 = fmt::format("{}:{}", ToString(request_method), client_context.uri); - auto ha2 = digest_hasher_.GetHash(a2); - - auto request_digest = fmt::format( - "{}:{}:{}:{}:{}:{}", - ha1, - client_context.nonce, - client_context.nc, - client_context.cnonce, - client_context.qop, - ha2 + const auto ha2 = digest_hasher_.GetHash({ToString(request_method), ":", client_context.uri}); + + return digest_hasher_.GetHash( + {ha1, + ":", + client_context.nonce, + ":", + client_context.nc, + ":", + client_context.cnonce, + ":", + client_context.qop, + ":", + ha2} ); - return digest_hasher_.GetHash(request_digest); } } // namespace server::handlers::auth::digest diff --git a/core/src/server/handlers/auth/digest/directives_parser.cpp b/core/src/server/handlers/auth/digest/directives_parser.cpp index b5914472d7c4..fa3c124ffec1 100644 --- a/core/src/server/handlers/auth/digest/directives_parser.cpp +++ b/core/src/server/handlers/auth/digest/directives_parser.cpp @@ -18,7 +18,7 @@ namespace { constexpr utils::StringLiteral kDigestWord = "Digest"; -enum class kClientDirectiveTypes { +enum class ClientDirectiveTypes { kUsername, kRealm, kNonce, @@ -35,25 +35,25 @@ enum class kClientDirectiveTypes { const utils::TrivialBiMap kClientDirectivesMap = [](auto selector) { return selector() - .Case(directives::kUsername, kClientDirectiveTypes::kUsername) - .Case(directives::kRealm, kClientDirectiveTypes::kRealm) - .Case(directives::kNonce, kClientDirectiveTypes::kNonce) - .Case(directives::kUri, kClientDirectiveTypes::kUri) - .Case(directives::kResponse, kClientDirectiveTypes::kResponse) - .Case(directives::kAlgorithm, kClientDirectiveTypes::kAlgorithm) - .Case(directives::kCnonce, kClientDirectiveTypes::kCnonce) - .Case(directives::kOpaque, kClientDirectiveTypes::kOpaque) - .Case(directives::kQop, kClientDirectiveTypes::kQop) - .Case(directives::kNonceCount, kClientDirectiveTypes::kNonceCount) - .Case(directives::kAuthParam, kClientDirectiveTypes::kAuthParam); + .Case(directives::kUsername, ClientDirectiveTypes::kUsername) + .Case(directives::kRealm, ClientDirectiveTypes::kRealm) + .Case(directives::kNonce, ClientDirectiveTypes::kNonce) + .Case(directives::kUri, ClientDirectiveTypes::kUri) + .Case(directives::kResponse, ClientDirectiveTypes::kResponse) + .Case(directives::kAlgorithm, ClientDirectiveTypes::kAlgorithm) + .Case(directives::kCnonce, ClientDirectiveTypes::kCnonce) + .Case(directives::kOpaque, ClientDirectiveTypes::kOpaque) + .Case(directives::kQop, ClientDirectiveTypes::kQop) + .Case(directives::kNonceCount, ClientDirectiveTypes::kNonceCount) + .Case(directives::kAuthParam, ClientDirectiveTypes::kAuthParam); }; -const std::array kMandatoryDirectives = { - kClientDirectiveTypes::kRealm, - kClientDirectiveTypes::kNonce, - kClientDirectiveTypes::kResponse, - kClientDirectiveTypes::kUri, - kClientDirectiveTypes::kUsername}; +const std::array kMandatoryDirectives = { + ClientDirectiveTypes::kRealm, + ClientDirectiveTypes::kNonce, + ClientDirectiveTypes::kResponse, + ClientDirectiveTypes::kUri, + ClientDirectiveTypes::kUsername}; enum class State { kStateSpace, @@ -170,44 +170,44 @@ ContextFromClient Parser::ParseAuthInfo(std::string_view auth_header_value) { void Parser::PushToClientContext(std::string&& directive, std::string&& value, ContextFromClient& client_context) { const auto directive_type = - kClientDirectivesMap.TryFind(std::move(directive)).value_or(kClientDirectiveTypes::kUnknown); + kClientDirectivesMap.TryFind(std::move(directive)).value_or(ClientDirectiveTypes::kUnknown); const auto index = static_cast(directive_type); - if (directive_type != kClientDirectiveTypes::kUnknown) directives_counter_[index]++; + if (directive_type != ClientDirectiveTypes::kUnknown) directives_counter_[index]++; switch (directive_type) { - case kClientDirectiveTypes::kUsername: + case ClientDirectiveTypes::kUsername: client_context.username = std::move(value); break; - case kClientDirectiveTypes::kRealm: + case ClientDirectiveTypes::kRealm: client_context.realm = std::move(value); break; - case kClientDirectiveTypes::kNonce: + case ClientDirectiveTypes::kNonce: client_context.nonce = std::move(value); break; - case kClientDirectiveTypes::kUri: + case ClientDirectiveTypes::kUri: client_context.uri = std::move(value); break; - case kClientDirectiveTypes::kResponse: + case ClientDirectiveTypes::kResponse: client_context.response = std::move(value); break; - case kClientDirectiveTypes::kAlgorithm: + case ClientDirectiveTypes::kAlgorithm: client_context.algorithm = std::move(value); break; - case kClientDirectiveTypes::kCnonce: + case ClientDirectiveTypes::kCnonce: client_context.cnonce = std::move(value); break; - case kClientDirectiveTypes::kOpaque: + case ClientDirectiveTypes::kOpaque: client_context.opaque = std::move(value); break; - case kClientDirectiveTypes::kQop: + case ClientDirectiveTypes::kQop: client_context.qop = std::move(value); break; - case kClientDirectiveTypes::kNonceCount: + case ClientDirectiveTypes::kNonceCount: client_context.nc = std::move(value); break; - case kClientDirectiveTypes::kAuthParam: + case ClientDirectiveTypes::kAuthParam: client_context.authparam = std::move(value); break; - case kClientDirectiveTypes::kUnknown: + case ClientDirectiveTypes::kUnknown: throw ParseException("Unknown directive found"); break; } @@ -237,7 +237,7 @@ void Parser::CheckDuplicateDirectivesExist() const { if (it != directives_counter_.end()) { const auto index = std::distance(directives_counter_.begin(), it); - const auto directive_type = static_cast(index); + const auto directive_type = static_cast(index); auto directive = kClientDirectivesMap.TryFind(directive_type).value_or(utils::StringLiteral{"unknown_directive"}); UASSERT(directive != "unknown_directive"); diff --git a/core/src/server/handlers/auth/digest/standalone_checker_test.cpp b/core/src/server/handlers/auth/digest/standalone_checker_test.cpp index acfbfdff24b8..9f66001ef3a5 100644 --- a/core/src/server/handlers/auth/digest/standalone_checker_test.cpp +++ b/core/src/server/handlers/auth/digest/standalone_checker_test.cpp @@ -67,7 +67,7 @@ class StandAloneCheckerTest : public ::testing::Test { {}} ), secdist_config({&default_loader, std::chrono::milliseconds::zero()}), - digest_settings_({ + digest_settings({ "MD5", // algorithm std::vector{"/"}, // domains std::vector{"auth"}, // qops @@ -75,8 +75,8 @@ class StandAloneCheckerTest : public ::testing::Test { false, // is_session kNonceTTL // nonce_ttl }), - checker_(digest_settings_, "testrealm@host.com", secdist_config), - correct_client_context_({ + checker(digest_settings, "testrealm@host.com", secdist_config), + correct_client_context({ "Mufasa", // username "testrealm@host.com", // realm kValidNonce, // nonce @@ -89,58 +89,58 @@ class StandAloneCheckerTest : public ::testing::Test { "00000001", // nc "auth-param" // authparam }) { - client_context_ = correct_client_context_; + client_context = correct_client_context; } TempFileProvider temp_file_provider; storages::secdist::DefaultLoader default_loader; storages::secdist::SecdistConfig secdist_config; - AuthCheckerSettings digest_settings_; - StandAloneChecker checker_; - ContextFromClient client_context_; - ContextFromClient correct_client_context_; + AuthCheckerSettings digest_settings; + StandAloneChecker checker; + ContextFromClient client_context; + ContextFromClient correct_client_context; }; UTEST_F(StandAloneCheckerTest, NonceTTL) { utils::datetime::MockNowSet(utils::datetime::Now()); - checker_.PushUnnamedNonce(kValidNonce); + checker.PushUnnamedNonce(kValidNonce); const UserData test_data{kValidHA1, kValidNonce, utils::datetime::Now(), 0}; utils::datetime::MockSleep(kNonceTTL - std::chrono::milliseconds(100)); - EXPECT_EQ(checker_.ValidateUserData(client_context_, test_data), ValidateResult::kOk); + EXPECT_EQ(checker.ValidateUserData(client_context, test_data), ValidateResult::kOk); utils::datetime::MockSleep(kNonceTTL + std::chrono::milliseconds(100)); - EXPECT_EQ(checker_.ValidateUserData(client_context_, test_data), ValidateResult::kWrongUserData); + EXPECT_EQ(checker.ValidateUserData(client_context, test_data), ValidateResult::kWrongUserData); } UTEST_F(StandAloneCheckerTest, NonceCount) { - checker_.PushUnnamedNonce(kValidNonce); + checker.PushUnnamedNonce(kValidNonce); UserData test_data{kValidHA1, kValidNonce, utils::datetime::Now(), 0}; - EXPECT_EQ(checker_.ValidateUserData(client_context_, test_data), ValidateResult::kOk); + EXPECT_EQ(checker.ValidateUserData(client_context, test_data), ValidateResult::kOk); test_data.nonce_count++; - EXPECT_EQ(checker_.ValidateUserData(client_context_, test_data), ValidateResult::kDuplicateRequest); + EXPECT_EQ(checker.ValidateUserData(client_context, test_data), ValidateResult::kDuplicateRequest); - correct_client_context_.nc = "00000002"; - client_context_ = correct_client_context_; - EXPECT_EQ(checker_.ValidateUserData(client_context_, test_data), ValidateResult::kOk); + correct_client_context.nc = "00000002"; + client_context = correct_client_context; + EXPECT_EQ(checker.ValidateUserData(client_context, test_data), ValidateResult::kOk); } UTEST_F(StandAloneCheckerTest, InvalidNonce) { const auto* invalid_nonce_ = "abc88743bacdf9238"; UserData test_data{kValidHA1, invalid_nonce_, utils::datetime::Now(), 0}; - EXPECT_EQ(checker_.ValidateUserData(client_context_, test_data), ValidateResult::kWrongUserData); + EXPECT_EQ(checker.ValidateUserData(client_context, test_data), ValidateResult::kWrongUserData); test_data.nonce = kValidNonce; - EXPECT_EQ(checker_.ValidateUserData(client_context_, test_data), ValidateResult::kOk); + EXPECT_EQ(checker.ValidateUserData(client_context, test_data), ValidateResult::kOk); } UTEST_F(StandAloneCheckerTest, NonceCountConvertingThrow) { - client_context_.nc = "not-a-hex-number"; + client_context.nc = "not-a-hex-number"; const UserData test_data{kValidHA1, kValidNonce, utils::datetime::Now(), 0}; - EXPECT_THROW(checker_.ValidateUserData(client_context_, test_data), std::runtime_error); + EXPECT_THROW(checker.ValidateUserData(client_context, test_data), std::runtime_error); } } // namespace server::handlers::auth::digest::test diff --git a/core/src/server/handlers/exceptions_test.cpp b/core/src/server/handlers/exceptions_test.cpp index 4c544409d646..5c92e0e2dca1 100644 --- a/core/src/server/handlers/exceptions_test.cpp +++ b/core/src/server/handlers/exceptions_test.cpp @@ -17,13 +17,13 @@ class CustomErrorBuilder { formats::json::ValueBuilder error; error["code"] = std::move(status); error["message"] = std::move(msg); - json_error_body = formats::json::ToString(error.ExtractValue()); + json_error_body_ = formats::json::ToString(error.ExtractValue()); } - std::string GetExternalBody() const { return json_error_body; } + std::string GetExternalBody() const { return json_error_body_; } private: - std::string json_error_body; + std::string json_error_body_; }; TEST(CustomHandlerException, BuilderSample) { diff --git a/core/src/server/handlers/server_monitor.cpp b/core/src/server/handlers/server_monitor.cpp index 0b085dea82e3..b9d92eea07dc 100644 --- a/core/src/server/handlers/server_monitor.cpp +++ b/core/src/server/handlers/server_monitor.cpp @@ -131,6 +131,8 @@ std::string ServerMonitor::HandleRequestThrow(const http::HttpRequest& request, } UINVARIANT(false, "Unexpected 'format' value"); + // never reaches + return {}; } std::string diff --git a/core/src/server/http/http_cached_date.cpp b/core/src/server/http/http_cached_date.cpp index cd0ae75eaca4..2b1bf4c46b9d 100644 --- a/core/src/server/http/http_cached_date.cpp +++ b/core/src/server/http/http_cached_date.cpp @@ -46,7 +46,14 @@ std::string_view GetCachedDate() { cache->last_time_string_size = time_str.size(); } +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-stack-address" +#endif return std::string_view{cache->last_time_string, cache->last_time_string_size}; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif } } // namespace impl diff --git a/core/src/server/http/http_cached_date_benchmark.cpp b/core/src/server/http/http_cached_date_benchmark.cpp index a0b9f532ba22..f63b59de4acc 100644 --- a/core/src/server/http/http_cached_date_benchmark.cpp +++ b/core/src/server/http/http_cached_date_benchmark.cpp @@ -5,18 +5,18 @@ USERVER_NAMESPACE_BEGIN -void http_get_cached_date_benchmark(benchmark::State& state) { +void HttpGetCachedDateBenchmark(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { benchmark::DoNotOptimize(server::http::impl::GetCachedDate()); } } -BENCHMARK(http_get_cached_date_benchmark); +BENCHMARK(HttpGetCachedDateBenchmark); -void http_make_date_benchmark(benchmark::State& state) { +void HttpMakeDateBenchmark(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { benchmark::DoNotOptimize(server::http::impl::MakeHttpDate(utils::datetime::WallCoarseClock::now())); } } -BENCHMARK(http_make_date_benchmark); +BENCHMARK(HttpMakeDateBenchmark); USERVER_NAMESPACE_END diff --git a/core/src/server/http/http_method.cpp b/core/src/server/http/http_method.cpp index b52f5b87846f..366d962be71e 100644 --- a/core/src/server/http/http_method.cpp +++ b/core/src/server/http/http_method.cpp @@ -6,6 +6,8 @@ namespace server::http { namespace { +// NOLINTBEGIN(readability-identifier-naming) +// Struct because of static initialization order fiasco. struct HttpMethodStrings { const std::string kDelete = "DELETE"; const std::string kGet = "GET"; @@ -17,6 +19,7 @@ struct HttpMethodStrings { const std::string kOptions = "OPTIONS"; const std::string kUnknown = "unknown"; }; +// NOLINTEND(readability-identifier-naming) // Modern SSO could hold 7 chars without dynamic allocation const HttpMethodStrings& GetHttpMethodStrings() noexcept { diff --git a/core/src/server/http/http_request.cpp b/core/src/server/http/http_request.cpp index 3128bc3a8f57..4748efadcf4a 100644 --- a/core/src/server/http/http_request.cpp +++ b/core/src/server/http/http_request.cpp @@ -72,17 +72,17 @@ HttpRequest::HttpRequest(request::ResponseDataAccounter& data_accounter, utils:: HttpRequest::~HttpRequest() = default; -const HttpMethod& HttpRequest::GetMethod() const { return pimpl_->method_; } +const HttpMethod& HttpRequest::GetMethod() const { return pimpl_->method; } -const std::string& HttpRequest::GetMethodStr() const { return ToString(pimpl_->method_); } +const std::string& HttpRequest::GetMethodStr() const { return ToString(pimpl_->method); } -int HttpRequest::GetHttpMajor() const { return pimpl_->http_major_; } +int HttpRequest::GetHttpMajor() const { return pimpl_->http_major; } -int HttpRequest::GetHttpMinor() const { return pimpl_->http_minor_; } +int HttpRequest::GetHttpMinor() const { return pimpl_->http_minor; } -const std::string& HttpRequest::GetUrl() const { return pimpl_->url_; } +const std::string& HttpRequest::GetUrl() const { return pimpl_->url; } -const std::string& HttpRequest::GetRequestPath() const { return pimpl_->request_path_; } +const std::string& HttpRequest::GetRequestPath() const { return pimpl_->request_path; } std::chrono::duration HttpRequest::GetRequestTime() const { return GetHttpResponse().SentTime() - GetStartTime(); @@ -94,44 +94,44 @@ std::chrono::duration HttpRequest::GetResponseTime() const { const std::string& HttpRequest::GetHost() const { return GetHeader(USERVER_NAMESPACE::http::headers::kHost); } -const engine::io::Sockaddr& HttpRequest::GetRemoteAddress() const { return pimpl_->remote_address_; } +const engine::io::Sockaddr& HttpRequest::GetRemoteAddress() const { return pimpl_->remote_address; } const std::string& HttpRequest::GetArg(std::string_view arg_name) const { #ifndef NDEBUG - pimpl_->args_referenced_ = true; + pimpl_->args_referenced = true; #endif - const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->request_args_, arg_name); + const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->request_args, arg_name); if (!ptr) return kEmptyString; return ptr->at(0); } const std::vector& HttpRequest::GetArgVector(std::string_view arg_name) const { #ifndef NDEBUG - pimpl_->args_referenced_ = true; + pimpl_->args_referenced = true; #endif - const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->request_args_, arg_name); + const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->request_args, arg_name); if (!ptr) return kEmptyVector; return *ptr; } bool HttpRequest::HasArg(std::string_view arg_name) const { - const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->request_args_, arg_name); + const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->request_args, arg_name); return !!ptr; } -size_t HttpRequest::ArgCount() const { return pimpl_->request_args_.size(); } +size_t HttpRequest::ArgCount() const { return pimpl_->request_args.size(); } std::vector HttpRequest::ArgNames() const { std::vector res; - res.reserve(pimpl_->request_args_.size()); - for (const auto& arg : pimpl_->request_args_) res.push_back(arg.first); + res.reserve(pimpl_->request_args.size()); + for (const auto& arg : pimpl_->request_args) res.push_back(arg.first); return res; } const FormDataArg& HttpRequest::GetFormDataArg(std::string_view arg_name) const { static const FormDataArg kEmptyFormDataArg{}; - const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->form_data_args_, arg_name); + const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->form_data_args, arg_name); if (!ptr) return kEmptyFormDataArg; return ptr->at(0); } @@ -139,202 +139,198 @@ const FormDataArg& HttpRequest::GetFormDataArg(std::string_view arg_name) const const std::vector& HttpRequest::GetFormDataArgVector(std::string_view arg_name) const { static const std::vector kEmptyFormDataArgVector{}; - const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->form_data_args_, arg_name); + const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->form_data_args, arg_name); if (!ptr) return kEmptyFormDataArgVector; return *ptr; } bool HttpRequest::HasFormDataArg(std::string_view arg_name) const { - const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->form_data_args_, arg_name); + const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->form_data_args, arg_name); return !!ptr; } -size_t HttpRequest::FormDataArgCount() const { return pimpl_->form_data_args_.size(); } +size_t HttpRequest::FormDataArgCount() const { return pimpl_->form_data_args.size(); } std::vector HttpRequest::FormDataArgNames() const { std::vector res; - res.reserve(pimpl_->form_data_args_.size()); - for (const auto& [name, _] : pimpl_->form_data_args_) res.push_back(name); + res.reserve(pimpl_->form_data_args.size()); + for (const auto& [name, _] : pimpl_->form_data_args) res.push_back(name); return res; } const std::string& HttpRequest::GetPathArg(std::string_view arg_name) const { - const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->path_args_by_name_index_, arg_name); + const auto* ptr = utils::impl::FindTransparentOrNullptr(pimpl_->path_args_by_name_index, arg_name); if (!ptr) return kEmptyString; - UASSERT(*ptr < pimpl_->path_args_.size()); - return pimpl_->path_args_[*ptr]; + UASSERT(*ptr < pimpl_->path_args.size()); + return pimpl_->path_args[*ptr]; } const std::string& HttpRequest::GetPathArg(size_t index) const { - return index < PathArgCount() ? pimpl_->path_args_[index] : kEmptyString; + return index < PathArgCount() ? pimpl_->path_args[index] : kEmptyString; } bool HttpRequest::HasPathArg(std::string_view arg_name) const { - return !!utils::impl::FindTransparentOrNullptr(pimpl_->path_args_by_name_index_, arg_name); + return !!utils::impl::FindTransparentOrNullptr(pimpl_->path_args_by_name_index, arg_name); } bool HttpRequest::HasPathArg(size_t index) const { return index < PathArgCount(); } -size_t HttpRequest::PathArgCount() const { return pimpl_->path_args_.size(); } +size_t HttpRequest::PathArgCount() const { return pimpl_->path_args.size(); } const std::string& HttpRequest::GetHeader(std::string_view header_name) const { - auto it = pimpl_->headers_.find(header_name); - if (it == pimpl_->headers_.end()) return kEmptyString; + auto it = pimpl_->headers.find(header_name); + if (it == pimpl_->headers.end()) return kEmptyString; return it->second; } const std::string& HttpRequest::GetHeader(const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name) const { - auto it = pimpl_->headers_.find(header_name); - if (it == pimpl_->headers_.end()) return kEmptyString; + auto it = pimpl_->headers.find(header_name); + if (it == pimpl_->headers.end()) return kEmptyString; return it->second; } -bool HttpRequest::HasHeader(std::string_view header_name) const { return pimpl_->headers_.count(header_name) != 0; } +bool HttpRequest::HasHeader(std::string_view header_name) const { return pimpl_->headers.count(header_name) != 0; } bool HttpRequest::HasHeader(const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name) const { - return pimpl_->headers_.count(header_name) != 0; + return pimpl_->headers.count(header_name) != 0; } -size_t HttpRequest::HeaderCount() const { return pimpl_->headers_.size(); } +size_t HttpRequest::HeaderCount() const { return pimpl_->headers.size(); } -void HttpRequest::RemoveHeader(std::string_view header_name) { pimpl_->headers_.erase(header_name); } +void HttpRequest::RemoveHeader(std::string_view header_name) { pimpl_->headers.erase(header_name); } void HttpRequest::RemoveHeader(const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name) { - pimpl_->headers_.erase(header_name); + pimpl_->headers.erase(header_name); } -HttpRequest::HeadersMapKeys HttpRequest::GetHeaderNames() const { - return HttpRequest::HeadersMapKeys{pimpl_->headers_}; -} +HttpRequest::HeadersMapKeys HttpRequest::GetHeaderNames() const { return HttpRequest::HeadersMapKeys{pimpl_->headers}; } -const HttpRequest::HeadersMap& HttpRequest::GetHeaders() const { return pimpl_->headers_; } +const HttpRequest::HeadersMap& HttpRequest::GetHeaders() const { return pimpl_->headers; } const std::string& HttpRequest::GetCookie(const std::string& cookie_name) const { - auto it = pimpl_->cookies_.find(cookie_name); - if (it == pimpl_->cookies_.end()) return kEmptyString; + auto it = pimpl_->cookies.find(cookie_name); + if (it == pimpl_->cookies.end()) return kEmptyString; return it->second; } -bool HttpRequest::HasCookie(const std::string& cookie_name) const { return pimpl_->cookies_.count(cookie_name); } +bool HttpRequest::HasCookie(const std::string& cookie_name) const { return pimpl_->cookies.count(cookie_name); } -size_t HttpRequest::CookieCount() const { return pimpl_->cookies_.size(); } +size_t HttpRequest::CookieCount() const { return pimpl_->cookies.size(); } -HttpRequest::CookiesMapKeys HttpRequest::GetCookieNames() const { - return HttpRequest::CookiesMapKeys{pimpl_->cookies_}; -} +HttpRequest::CookiesMapKeys HttpRequest::GetCookieNames() const { return HttpRequest::CookiesMapKeys{pimpl_->cookies}; } -const HttpRequest::CookiesMap& HttpRequest::RequestCookies() const { return pimpl_->cookies_; } +const HttpRequest::CookiesMap& HttpRequest::RequestCookies() const { return pimpl_->cookies; } -const std::string& HttpRequest::RequestBody() const { return pimpl_->request_body_; } +const std::string& HttpRequest::RequestBody() const { return pimpl_->request_body; } -std::string HttpRequest::ExtractRequestBody() { return std::move(pimpl_->request_body_); } +std::string HttpRequest::ExtractRequestBody() { return std::move(pimpl_->request_body); } -void HttpRequest::SetRequestBody(std::string body) { pimpl_->request_body_ = std::move(body); } +void HttpRequest::SetRequestBody(std::string body) { pimpl_->request_body = std::move(body); } void HttpRequest::ParseArgsFromBody() { #ifndef NDEBUG UASSERT_MSG( - !pimpl_->args_referenced_, + !pimpl_->args_referenced, "References to arguments could be invalidated by ParseArgsFromBody(). " "Avoid calling GetArg()/GetArgVector() before ParseArgsFromBody()" ); #endif USERVER_NAMESPACE::http::parser::ParseAndConsumeArgs( - pimpl_->request_body_, + pimpl_->request_body, [this](std::string&& key, std::string&& value) { - pimpl_->request_args_[std::move(key)].push_back(std::move(value)); + pimpl_->request_args[std::move(key)].push_back(std::move(value)); } ); } -bool HttpRequest::IsFinal() const { return pimpl_->is_final_; } +bool HttpRequest::IsFinal() const { return pimpl_->is_final; } -void HttpRequest::SetResponseStatus(HttpStatus status) const { pimpl_->response_.SetStatus(status); } +void HttpRequest::SetResponseStatus(HttpStatus status) const { pimpl_->response.SetStatus(status); } bool HttpRequest::IsBodyCompressed() const { const auto& encoding = GetHeader(USERVER_NAMESPACE::http::headers::kContentEncoding); return !encoding.empty() && encoding != "identity"; } -HttpResponse& HttpRequest::GetHttpResponse() const { return pimpl_->response_; } +HttpResponse& HttpRequest::GetHttpResponse() const { return pimpl_->response; } -std::chrono::steady_clock::time_point HttpRequest::GetStartTime() const { return pimpl_->start_time_; } +std::chrono::steady_clock::time_point HttpRequest::GetStartTime() const { return pimpl_->start_time; } -bool HttpRequest::IsUpgradeWebsocket() const { return static_cast(pimpl_->upgrade_websocket_cb_); } +bool HttpRequest::IsUpgradeWebsocket() const { return static_cast(pimpl_->upgrade_websocket_cb); } -void HttpRequest::SetUpgradeWebsocket(UpgradeCallback cb) const { pimpl_->upgrade_websocket_cb_ = std::move(cb); } +void HttpRequest::SetUpgradeWebsocket(UpgradeCallback cb) const { pimpl_->upgrade_websocket_cb = std::move(cb); } void HttpRequest::DoUpgrade(std::unique_ptr&& socket, engine::io::Sockaddr&& peer_name) const { - pimpl_->upgrade_websocket_cb_(std::move(socket), std::move(peer_name)); + pimpl_->upgrade_websocket_cb(std::move(socket), std::move(peer_name)); } void HttpRequest::SetPathArgs(std::vector> args) { - pimpl_->path_args_.clear(); - pimpl_->path_args_.reserve(args.size()); + pimpl_->path_args.clear(); + pimpl_->path_args.reserve(args.size()); - pimpl_->path_args_by_name_index_.clear(); + pimpl_->path_args_by_name_index.clear(); for (auto& [name, value] : args) { - pimpl_->path_args_.push_back(std::move(value)); + pimpl_->path_args.push_back(std::move(value)); if (!name.empty()) { - pimpl_->path_args_by_name_index_[std::move(name)] = pimpl_->path_args_.size() - 1; + pimpl_->path_args_by_name_index[std::move(name)] = pimpl_->path_args.size() - 1; } } } void HttpRequest::AccountResponseTime() { - if (pimpl_->request_statistics_) { + if (pimpl_->request_statistics) { auto timing = std::chrono::duration_cast( - pimpl_->finish_send_response_time_ - pimpl_->start_time_ + pimpl_->finish_send_response_time - pimpl_->start_time ); - pimpl_->request_statistics_->ForMethod(GetMethod()).Account(handlers::HttpRequestStatisticsEntry{timing}); + pimpl_->request_statistics->ForMethod(GetMethod()).Account(handlers::HttpRequestStatisticsEntry{timing}); } } void HttpRequest::MarkAsInternalServerError() const { // TODO : refactor, this being here is a bit ridiculous - pimpl_->response_.SetStatus(http::HttpStatus::kInternalServerError); - pimpl_->response_.SetData({}); + pimpl_->response.SetStatus(http::HttpStatus::kInternalServerError); + pimpl_->response.SetData({}); - std::string server_header = pimpl_->response_.GetHeader(USERVER_NAMESPACE::http::headers::kServer); - pimpl_->response_.ClearHeaders(); + std::string server_header = pimpl_->response.GetHeader(USERVER_NAMESPACE::http::headers::kServer); + pimpl_->response.ClearHeaders(); if (!server_header.empty()) { - pimpl_->response_.SetHeader(USERVER_NAMESPACE::http::headers::kServer, std::move(server_header)); + pimpl_->response.SetHeader(USERVER_NAMESPACE::http::headers::kServer, std::move(server_header)); } } -void HttpRequest::SetHttpHandler(const handlers::HttpHandlerBase& handler) { pimpl_->handler_ = &handler; } +void HttpRequest::SetHttpHandler(const handlers::HttpHandlerBase& handler) { pimpl_->handler = &handler; } -const handlers::HttpHandlerBase* HttpRequest::GetHttpHandler() const { return pimpl_->handler_; } +const handlers::HttpHandlerBase* HttpRequest::GetHttpHandler() const { return pimpl_->handler; } -void HttpRequest::SetTaskProcessor(engine::TaskProcessor& task_processor) { pimpl_->task_processor_ = &task_processor; } +void HttpRequest::SetTaskProcessor(engine::TaskProcessor& task_processor) { pimpl_->task_processor = &task_processor; } -engine::TaskProcessor* HttpRequest::GetTaskProcessor() const { return pimpl_->task_processor_; } +engine::TaskProcessor* HttpRequest::GetTaskProcessor() const { return pimpl_->task_processor; } void HttpRequest::SetHttpHandlerStatistics(handlers::HttpRequestStatistics& stats) { - pimpl_->request_statistics_ = &stats; + pimpl_->request_statistics = &stats; } -void HttpRequest::SetResponseStreamId(std::int32_t stream_id) { pimpl_->response_.SetStreamId(stream_id); } +void HttpRequest::SetResponseStreamId(std::int32_t stream_id) { pimpl_->response.SetStreamId(stream_id); } void HttpRequest::SetStreamProducer(impl::Http2StreamEventProducer&& producer) { - pimpl_->response_.SetStreamProdicer(std::move(producer)); + pimpl_->response.SetStreamProdicer(std::move(producer)); } -void HttpRequest::SetTaskCreateTime() { pimpl_->task_create_time_ = std::chrono::steady_clock::now(); } +void HttpRequest::SetTaskCreateTime() { pimpl_->task_create_time = std::chrono::steady_clock::now(); } -void HttpRequest::SetTaskStartTime() { pimpl_->task_start_time_ = std::chrono::steady_clock::now(); } +void HttpRequest::SetTaskStartTime() { pimpl_->task_start_time = std::chrono::steady_clock::now(); } void HttpRequest::SetResponseNotifyTime() { SetResponseNotifyTime(std::chrono::steady_clock::now()); } void HttpRequest::SetResponseNotifyTime(std::chrono::steady_clock::time_point now) { - pimpl_->response_notify_time_ = now; + pimpl_->response_notify_time = now; } -void HttpRequest::SetStartSendResponseTime() { pimpl_->start_send_response_time_ = std::chrono::steady_clock::now(); } +void HttpRequest::SetStartSendResponseTime() { pimpl_->start_send_response_time = std::chrono::steady_clock::now(); } void HttpRequest::SetFinishSendResponseTime() { - pimpl_->finish_send_response_time_ = std::chrono::steady_clock::now(); + pimpl_->finish_send_response_time = std::chrono::steady_clock::now(); AccountResponseTime(); } @@ -367,7 +363,7 @@ void HttpRequest::WriteAccessLog( EscapeForAccessLog(GetUrl()), GetHttpMajor(), GetHttpMinor(), - static_cast(pimpl_->response_.GetStatus()), + static_cast(pimpl_->response.GetStatus()), EscapeForAccessLog(GetHeader("Referer")), EscapeForAccessLog(GetHeader("User-Agent")), EscapeForAccessLog(GetHeader("Cookie")), @@ -407,7 +403,7 @@ void HttpRequest::WriteAccessTskvLog( "\tupstream_response_time={:0.3f}" "\trequest_body={}", utils::datetime::LocalTimezoneTimestring(tp, "timestamp=%Y-%m-%dT%H:%M:%S\ttimezone=%Ez"), - static_cast(pimpl_->response_.GetStatus()), + static_cast(pimpl_->response.GetStatus()), GetHttpMajor(), GetHttpMinor(), EscapeForAccessTskvLog(GetMethodStr()), diff --git a/core/src/server/http/http_request_builder.cpp b/core/src/server/http/http_request_builder.cpp index 347e63d8b8d9..9958888092e2 100644 --- a/core/src/server/http/http_request_builder.cpp +++ b/core/src/server/http/http_request_builder.cpp @@ -25,22 +25,22 @@ HttpRequestBuilder::HttpRequestBuilder(request::ResponseDataAccounter& data_acco HttpRequestBuilder::HttpRequestBuilder() : HttpRequestBuilder(default_data_accounter) {} HttpRequestBuilder& HttpRequestBuilder::SetRemoteAddress(engine::io::Sockaddr remote_address) { - request_->pimpl_->remote_address_ = std::move(remote_address); + request_->pimpl_->remote_address = std::move(remote_address); return *this; } HttpRequestBuilder& HttpRequestBuilder::SetMethod(HttpMethod method) { - request_->pimpl_->method_ = method; + request_->pimpl_->method = method; return *this; } HttpRequestBuilder& HttpRequestBuilder::SetHttpMajor(int http_major) { - request_->pimpl_->http_major_ = http_major; + request_->pimpl_->http_major = http_major; return *this; } HttpRequestBuilder& HttpRequestBuilder::SetHttpMinor(int http_minor) { - request_->pimpl_->http_minor_ = http_minor; + request_->pimpl_->http_minor = http_minor; return *this; } @@ -50,12 +50,12 @@ HttpRequestBuilder& HttpRequestBuilder::SetBody(std::string&& body) { } HttpRequestBuilder& HttpRequestBuilder::AddHeader(std::string&& header, std::string&& value) { - request_->pimpl_->headers_.InsertOrAppend(std::move(header), std::move(value)); + request_->pimpl_->headers.InsertOrAppend(std::move(header), std::move(value)); return *this; } HttpRequestBuilder& HttpRequestBuilder::AddRequestArg(std::string&& key, std::string&& value) { - request_->pimpl_->request_args_[std::move(key)].push_back(std::move(value)); + request_->pimpl_->request_args[std::move(key)].push_back(std::move(value)); return *this; } @@ -65,24 +65,24 @@ HttpRequestBuilder& HttpRequestBuilder::SetPathArgs(std::vectorpimpl_->url_ = std::move(url); + request_->pimpl_->url = std::move(url); return *this; } HttpRequestBuilder& HttpRequestBuilder::SetRequestPath(std::string&& path) { - request_->pimpl_->request_path_ = std::move(path); + request_->pimpl_->request_path = std::move(path); return *this; } HttpRequestBuilder& HttpRequestBuilder::SetIsFinal(bool is_final) { - request_->pimpl_->is_final_ = is_final; + request_->pimpl_->is_final = is_final; return *this; } HttpRequestBuilder& HttpRequestBuilder::SetFormDataArgs( utils::impl::TransparentMap, utils::StrCaseHash>&& form_data_args ) { - request_->pimpl_->form_data_args_ = std::move(form_data_args); + request_->pimpl_->form_data_args = std::move(form_data_args); return *this; } @@ -127,15 +127,15 @@ std::shared_ptr HttpRequestBuilder::Build() { ParseCookies(); UASSERT(std::all_of( - request_->pimpl_->request_args_.begin(), - request_->pimpl_->request_args_.end(), + request_->pimpl_->request_args.begin(), + request_->pimpl_->request_args.end(), [](const auto& arg) { return !arg.second.empty(); } )); LOG_TRACE() << "method=" << request_->GetMethodStr(); - LOG_TRACE() << "request_args:" << request_->pimpl_->request_args_; - LOG_TRACE() << "headers:" << request_->pimpl_->headers_; - LOG_TRACE() << "cookies:" << request_->pimpl_->cookies_; + LOG_TRACE() << "request_args:" << request_->pimpl_->request_args; + LOG_TRACE() << "headers:" << request_->pimpl_->headers; + LOG_TRACE() << "cookies:" << request_->pimpl_->cookies; return request_; } @@ -166,7 +166,7 @@ void HttpRequestBuilder::ParseCookies() { } Strip(key_begin, key_end); if (key_begin < key_end) { - request_->pimpl_->cookies_.emplace( + request_->pimpl_->cookies.emplace( std::piecewise_construct, std::tie(key_begin, key_end), std::tie(value_begin, value_end) ); } diff --git a/core/src/server/http/http_request_constructor.cpp b/core/src/server/http/http_request_constructor.cpp index 4b60dde91838..60a2fbebe2fd 100644 --- a/core/src/server/http/http_request_constructor.cpp +++ b/core/src/server/http/http_request_constructor.cpp @@ -213,6 +213,11 @@ void HttpRequestConstructor::FinalizeImpl() { builder_.SetFormDataArgs(std::move(form_data_args)); } } + + const auto& host = request.GetHeader(USERVER_NAMESPACE::http::headers::kHost); + if (host.empty()) { + LOG_LIMITED_WARNING() << "\"Host\" header in request is empty or absent"; + } } void HttpRequestConstructor::ParseArgs(const HttpParserUrl& url) { diff --git a/core/src/server/http/http_request_constructor_benchmark.cpp b/core/src/server/http/http_request_constructor_benchmark.cpp index 75cc159025af..a7237ee51f51 100644 --- a/core/src/server/http/http_request_constructor_benchmark.cpp +++ b/core/src/server/http/http_request_constructor_benchmark.cpp @@ -7,7 +7,7 @@ USERVER_NAMESPACE_BEGIN namespace { -void http_request_constructor_url_decode(benchmark::State& state) { +void HttpRequestConstructorUrlDecode(benchmark::State& state) { const std::string tmp = "1"; std::string input; @@ -16,6 +16,6 @@ void http_request_constructor_url_decode(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) benchmark::DoNotOptimize(USERVER_NAMESPACE::http::parser::UrlDecode(input)); } } // namespace -BENCHMARK(http_request_constructor_url_decode)->RangeMultiplier(2)->Range(1, 1024); +BENCHMARK(HttpRequestConstructorUrlDecode)->RangeMultiplier(2)->Range(1, 1024); USERVER_NAMESPACE_END diff --git a/core/src/server/http/http_request_headers_benchmark.cpp b/core/src/server/http/http_request_headers_benchmark.cpp index b1124541aaa9..c7fade2b3cde 100644 --- a/core/src/server/http/http_request_headers_benchmark.cpp +++ b/core/src/server/http/http_request_headers_benchmark.cpp @@ -25,7 +25,7 @@ constexpr PredefinedHeader kHeadersArray[kHeadersCount] = { PredefinedHeader{"TestHeader30"}, PredefinedHeader{"TestHeader31"}, }; -void http_request_headers_insert(benchmark::State& state) { +void HttpRequestHeadersInsert(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { server::http::HttpRequest::HeadersMap map; @@ -35,7 +35,7 @@ void http_request_headers_insert(benchmark::State& state) { } } -void http_request_headers_get(benchmark::State& state) { +void HttpRequestHeadersGet(benchmark::State& state) { server::http::HttpRequest::HeadersMap map; for (const auto& header : kHeadersArray) map[header] = "1"; @@ -47,8 +47,8 @@ void http_request_headers_get(benchmark::State& state) { } } // namespace -BENCHMARK(http_request_headers_insert)->RangeMultiplier(2)->Range(1, kHeadersCount); +BENCHMARK(HttpRequestHeadersInsert)->RangeMultiplier(2)->Range(1, kHeadersCount); -BENCHMARK(http_request_headers_get); +BENCHMARK(HttpRequestHeadersGet); USERVER_NAMESPACE_END diff --git a/core/src/server/http/http_request_impl.hpp b/core/src/server/http/http_request_impl.hpp index 80479d242d8a..7dfbd61cb649 100644 --- a/core/src/server/http/http_request_impl.hpp +++ b/core/src/server/http/http_request_impl.hpp @@ -19,44 +19,44 @@ struct HttpRequest::Impl { // unordered_maps because we don't need different seeds and want to avoid its // overhead. Impl(HttpRequest& http_request, request::ResponseDataAccounter& data_accounter) - : start_time_(std::chrono::steady_clock::now()), - form_data_args_(impl::kZeroAllocationBucketCount, request_args_.hash_function()), - path_args_by_name_index_(impl::kZeroAllocationBucketCount, request_args_.hash_function()), - headers_(impl::kBucketCount), - cookies_(impl::kZeroAllocationBucketCount, request_args_.hash_function()), - response_(http_request, data_accounter, start_time_, cookies_.hash_function()) {} + : start_time(std::chrono::steady_clock::now()), + form_data_args(impl::kZeroAllocationBucketCount, request_args.hash_function()), + path_args_by_name_index(impl::kZeroAllocationBucketCount, request_args.hash_function()), + headers(impl::kBucketCount), + cookies(impl::kZeroAllocationBucketCount, request_args.hash_function()), + response(http_request, data_accounter, start_time, cookies.hash_function()) {} - std::chrono::steady_clock::time_point start_time_; - std::chrono::steady_clock::time_point task_create_time_; - std::chrono::steady_clock::time_point task_start_time_; - std::chrono::steady_clock::time_point response_notify_time_; - std::chrono::steady_clock::time_point start_send_response_time_; - std::chrono::steady_clock::time_point finish_send_response_time_; + std::chrono::steady_clock::time_point start_time; + std::chrono::steady_clock::time_point task_create_time; + std::chrono::steady_clock::time_point task_start_time; + std::chrono::steady_clock::time_point response_notify_time; + std::chrono::steady_clock::time_point start_send_response_time; + std::chrono::steady_clock::time_point finish_send_response_time; - HttpMethod method_{HttpMethod::kUnknown}; - int http_major_{1}; - int http_minor_{1}; - std::string url_; - std::string request_path_; - std::string request_body_; - utils::impl::TransparentMap, utils::StrCaseHash> request_args_; - utils::impl::TransparentMap, utils::StrCaseHash> form_data_args_; - std::vector path_args_; - utils::impl::TransparentMap path_args_by_name_index_; - HeadersMap headers_; - CookiesMap cookies_; - bool is_final_{false}; + HttpMethod method{HttpMethod::kUnknown}; + int http_major{1}; + int http_minor{1}; + std::string url; + std::string request_path; + std::string request_body; + utils::impl::TransparentMap, utils::StrCaseHash> request_args; + utils::impl::TransparentMap, utils::StrCaseHash> form_data_args; + std::vector path_args; + utils::impl::TransparentMap path_args_by_name_index; + HeadersMap headers; + CookiesMap cookies; + bool is_final{false}; #ifndef NDEBUG - mutable bool args_referenced_{false}; + mutable bool args_referenced{false}; #endif // TODO - mutable UpgradeCallback upgrade_websocket_cb_; + mutable UpgradeCallback upgrade_websocket_cb; - mutable HttpResponse response_; - engine::io::Sockaddr remote_address_; - engine::TaskProcessor* task_processor_{nullptr}; - const handlers::HttpHandlerBase* handler_{nullptr}; - handlers::HttpRequestStatistics* request_statistics_{nullptr}; + mutable HttpResponse response; + engine::io::Sockaddr remote_address; + engine::TaskProcessor* task_processor{nullptr}; + const handlers::HttpHandlerBase* handler{nullptr}; + handlers::HttpRequestStatistics* request_statistics{nullptr}; }; } // namespace server::http diff --git a/core/src/server/http/http_request_parser_benchmark.cpp b/core/src/server/http/http_request_parser_benchmark.cpp index 8310e6efa873..b4f7a9948f3d 100644 --- a/core/src/server/http/http_request_parser_benchmark.cpp +++ b/core/src/server/http/http_request_parser_benchmark.cpp @@ -43,7 +43,7 @@ inline server::http::HttpRequestParser CreateBenchmarkParser(server::http::HttpR } // namespace -void http_request_parser_parse_benchmark_small(benchmark::State& state) { +void HttpRequestParserParseBenchmarkSmall(benchmark::State& state) { auto parser = CreateBenchmarkParser([](std::shared_ptr&&) {}); for ([[maybe_unused]] auto _ : state) { @@ -51,7 +51,7 @@ void http_request_parser_parse_benchmark_small(benchmark::State& state) { } } -void http_request_parser_parse_benchmark_middle(benchmark::State& state) { +void HttpRequestParserParseBenchmarkMiddle(benchmark::State& state) { auto parser = CreateBenchmarkParser([](std::shared_ptr&&) {}); for ([[maybe_unused]] auto _ : state) { @@ -59,7 +59,7 @@ void http_request_parser_parse_benchmark_middle(benchmark::State& state) { } } -void http_request_parser_parse_benchmark_large_url(benchmark::State& state) { +void HttpRequestParserParseBenchmarkLargeUrl(benchmark::State& state) { auto parser = CreateBenchmarkParser([](std::shared_ptr&&) {}); std::string large_url; @@ -73,7 +73,7 @@ void http_request_parser_parse_benchmark_large_url(benchmark::State& state) { } } -void http_request_parser_parse_benchmark_large_body(benchmark::State& state) { +void HttpRequestParserParseBenchmarkLargeBody(benchmark::State& state) { auto parser = CreateBenchmarkParser([](std::shared_ptr&&) {}); std::string large_body; @@ -92,7 +92,7 @@ void http_request_parser_parse_benchmark_large_body(benchmark::State& state) { } } -void http_request_parser_parse_benchmark_many_headers(benchmark::State& state) { +void HttpRequestParserParseBenchmarkManyHeaders(benchmark::State& state) { auto parser = CreateBenchmarkParser([](std::shared_ptr&&) {}); std::string headers; @@ -110,10 +110,10 @@ void http_request_parser_parse_benchmark_many_headers(benchmark::State& state) { } } -BENCHMARK(http_request_parser_parse_benchmark_small); -BENCHMARK(http_request_parser_parse_benchmark_middle); -BENCHMARK(http_request_parser_parse_benchmark_large_url); -BENCHMARK(http_request_parser_parse_benchmark_large_body); -BENCHMARK(http_request_parser_parse_benchmark_many_headers); +BENCHMARK(HttpRequestParserParseBenchmarkSmall); +BENCHMARK(HttpRequestParserParseBenchmarkMiddle); +BENCHMARK(HttpRequestParserParseBenchmarkLargeUrl); +BENCHMARK(HttpRequestParserParseBenchmarkLargeBody); +BENCHMARK(HttpRequestParserParseBenchmarkManyHeaders); USERVER_NAMESPACE_END diff --git a/core/src/server/http/http_request_parser_test.cpp b/core/src/server/http/http_request_parser_test.cpp index 88146947b24d..63612e9bc4c3 100644 --- a/core/src/server/http/http_request_parser_test.cpp +++ b/core/src/server/http/http_request_parser_test.cpp @@ -11,6 +11,9 @@ constexpr std::string_view kHttpRequestSmall = "GET / HTTP/1.1\r\n\r\n"; constexpr std::string_view kHttpRequestOriginUrl = "GET /foo/bar?query1=value1&query2=value2 HTTP/1.1\r\n\r\n"; +constexpr std::string_view kFullHttpRequestOriginUrl = + "GET http://www.example.org/foo/bar?query1=value1&query2=value2 HTTP/1.1\r\n\r\n"; + constexpr std::string_view kHttpRequestAbsoluteUrl = "GET http://www.example.org/pub/WWW/TheProject.html HTTP/1.1\r\n\r\n"; @@ -41,16 +44,13 @@ UTEST(HttpRequestParserParser, Small) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.GetUrl(), "/"); + EXPECT_EQ(request->GetUrl(), "/"); - EXPECT_EQ(http_request_impl.GetHttpMajor(), 1); - EXPECT_EQ(http_request_impl.GetHttpMinor(), 1); + EXPECT_EQ(request->GetHttpMajor(), 1); + EXPECT_EQ(request->GetHttpMinor(), 1); }); parser->Parse(kHttpRequestSmall); @@ -61,35 +61,47 @@ UTEST(HttpRequestParserParser, OriginUrl) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.GetUrl(), "/foo/bar?query1=value1&query2=value2"); - EXPECT_EQ(http_request_impl.GetRequestPath(), "/foo/bar"); - EXPECT_EQ(http_request_impl.ArgCount(), 2); - EXPECT_EQ(http_request_impl.GetArg("query1"), "value1"); - EXPECT_EQ(http_request_impl.GetArg("query2"), "value2"); + EXPECT_EQ(request->GetUrl(), "/foo/bar?query1=value1&query2=value2"); + EXPECT_EQ(request->GetRequestPath(), "/foo/bar"); + EXPECT_EQ(request->ArgCount(), 2); + EXPECT_EQ(request->GetArg("query1"), "value1"); + EXPECT_EQ(request->GetArg("query2"), "value2"); }); parser->Parse(kHttpRequestOriginUrl); EXPECT_EQ(parsed, true); } +UTEST(HttpRequestParserParser, GetRequestPath) { + bool parsed = false; + auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { + parsed = true; + + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); + + EXPECT_EQ(request->GetUrl(), "http://www.example.org/foo/bar?query1=value1&query2=value2"); + EXPECT_EQ(request->GetRequestPath(), "/foo/bar"); + EXPECT_EQ(request->ArgCount(), 2); + EXPECT_EQ(request->GetArg("query1"), "value1"); + EXPECT_EQ(request->GetArg("query2"), "value2"); + }); + + parser->Parse(kFullHttpRequestOriginUrl); + EXPECT_EQ(parsed, true); +} + UTEST(HttpRequestParserParser, AbsoluteUrl) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.GetUrl(), "http://www.example.org/pub/WWW/TheProject.html"); - EXPECT_EQ(http_request_impl.GetRequestPath(), "/pub/WWW/TheProject.html"); + EXPECT_EQ(request->GetUrl(), "http://www.example.org/pub/WWW/TheProject.html"); + EXPECT_EQ(request->GetRequestPath(), "/pub/WWW/TheProject.html"); }); parser->Parse(kHttpRequestAbsoluteUrl); @@ -100,18 +112,15 @@ UTEST(HttpRequestParserParser, HeadersSimple) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.HeaderCount(), 2); - EXPECT_EQ(http_request_impl.HasHeader("host"), true); - EXPECT_EQ(http_request_impl.GetHeader("host"), "localhost:11235"); + EXPECT_EQ(request->HeaderCount(), 2); + EXPECT_EQ(request->HasHeader("host"), true); + EXPECT_EQ(request->GetHeader("host"), "localhost:11235"); - EXPECT_EQ(http_request_impl.HasHeader("user-agent"), true); - EXPECT_EQ(http_request_impl.GetHeader("user-agent"), "curl/7.58.0"); + EXPECT_EQ(request->HasHeader("user-agent"), true); + EXPECT_EQ(request->GetHeader("user-agent"), "curl/7.58.0"); }); parser->Parse(kHttpRequestHeadersSimple); @@ -122,18 +131,15 @@ UTEST(HttpRequestParserParser, HeadersNoSpaces) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.HeaderCount(), 2); - EXPECT_EQ(http_request_impl.HasHeader("host"), true); - EXPECT_EQ(http_request_impl.GetHeader("host"), "localhost:11235"); + EXPECT_EQ(request->HeaderCount(), 2); + EXPECT_EQ(request->HasHeader("host"), true); + EXPECT_EQ(request->GetHeader("host"), "localhost:11235"); - EXPECT_EQ(http_request_impl.HasHeader("user-agent"), true); - EXPECT_EQ(http_request_impl.GetHeader("user-agent"), "curl/7.58.0"); + EXPECT_EQ(request->HasHeader("user-agent"), true); + EXPECT_EQ(request->GetHeader("user-agent"), "curl/7.58.0"); }); parser->Parse(kHttpRequestHeadersNoSpaces); @@ -144,18 +150,15 @@ UTEST(HttpRequestParserParser, HeadersCaseInsensitive) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.HeaderCount(), 2); - EXPECT_EQ(http_request_impl.HasHeader("host"), true); - EXPECT_EQ(http_request_impl.GetHeader("host"), "localhost:11235"); + EXPECT_EQ(request->HeaderCount(), 2); + EXPECT_EQ(request->HasHeader("host"), true); + EXPECT_EQ(request->GetHeader("host"), "localhost:11235"); - EXPECT_EQ(http_request_impl.HasHeader("user-agent"), true); - EXPECT_EQ(http_request_impl.GetHeader("user-agent"), "curl/7.58.0"); + EXPECT_EQ(request->HasHeader("user-agent"), true); + EXPECT_EQ(request->GetHeader("user-agent"), "curl/7.58.0"); }); parser->Parse(kHttpRequestHeadersCaseInsensitive); @@ -166,18 +169,15 @@ UTEST(HttpRequestParserParser, HeaderValues) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.HeaderCount(), 2); - EXPECT_EQ(http_request_impl.HasHeader("host"), true); - EXPECT_EQ(http_request_impl.GetHeader("host"), "*\"@!%"); + EXPECT_EQ(request->HeaderCount(), 2); + EXPECT_EQ(request->HasHeader("host"), true); + EXPECT_EQ(request->GetHeader("host"), "*\"@!%"); - EXPECT_EQ(http_request_impl.HasHeader("user-agent"), true); - EXPECT_EQ(http_request_impl.GetHeader("user-agent"), "[-]{~},/"); + EXPECT_EQ(request->HasHeader("user-agent"), true); + EXPECT_EQ(request->GetHeader("user-agent"), "[-]{~},/"); }); parser->Parse(kHttpRequestHeaderValues); @@ -188,13 +188,10 @@ UTEST(HttpRequestParserParser, BodySimple) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kGet); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kGet); - EXPECT_EQ(http_request_impl.RequestBody(), "body"); + EXPECT_EQ(request->RequestBody(), "body"); }); parser->Parse(kHttpRequestBodySimple); @@ -222,11 +219,8 @@ UTEST(HttpRequestParserParser, MethodWrongCase) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kUnknown); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kUnknown); }); parser->Parse(kHttpRequestMethodWrongCase); @@ -237,11 +231,8 @@ UTEST(HttpRequestParserParser, NoURL) { bool parsed = false; auto parser = server::CreateTestParser([&parsed](std::shared_ptr&& request) { parsed = true; - auto& http_request_impl = - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - static_cast(*request); - EXPECT_EQ(http_request_impl.GetMethod(), server::http::HttpMethod::kUnknown); + EXPECT_EQ(request->GetMethod(), server::http::HttpMethod::kUnknown); }); parser->Parse(kHttpRequestNoURL); diff --git a/core/src/server/http/http_response_benchmark.cpp b/core/src/server/http/http_response_benchmark.cpp index f483cefb116d..d8d40aeaaf0c 100644 --- a/core/src/server/http/http_response_benchmark.cpp +++ b/core/src/server/http/http_response_benchmark.cpp @@ -42,7 +42,7 @@ void OutputHeader(std::string& header, std::string_view key, std::string_view va append(kCrlf); } -void http_headers_serialization_inplace(benchmark::State& state) { +void HttpHeadersSerializationInplace(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { USERVER_NAMESPACE::http::headers::HeadersString os; @@ -71,7 +71,7 @@ void http_headers_serialization_inplace(benchmark::State& state) { } } -void http_headers_serialization_no_ostreams(benchmark::State& state) { +void HttpHeadersSerializationNoOstreams(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { std::string os; os.reserve(1024); @@ -93,7 +93,7 @@ void http_headers_serialization_no_ostreams(benchmark::State& state) { } } -void http_headers_serialization_ostreams(benchmark::State& state) { +void HttpHeadersSerializationOstreams(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { std::ostringstream os; @@ -137,9 +137,9 @@ void HttpResponseSetHeaderBenchmark(benchmark::State& state) { } // namespace -BENCHMARK(http_headers_serialization_inplace); -BENCHMARK(http_headers_serialization_no_ostreams); -BENCHMARK(http_headers_serialization_ostreams); +BENCHMARK(HttpHeadersSerializationInplace); +BENCHMARK(HttpHeadersSerializationNoOstreams); +BENCHMARK(HttpHeadersSerializationOstreams); BENCHMARK(HttpResponseSetHeaderBenchmark); USERVER_NAMESPACE_END diff --git a/core/src/server/http/http_response_cookie_benchmark.cpp b/core/src/server/http/http_response_cookie_benchmark.cpp index 03664daffcd0..762b2e3a3db5 100644 --- a/core/src/server/http/http_response_cookie_benchmark.cpp +++ b/core/src/server/http/http_response_cookie_benchmark.cpp @@ -6,7 +6,7 @@ USERVER_NAMESPACE_BEGIN -void http_cookie_serialization(benchmark::State& state) { +void HttpCookieSerialization(benchmark::State& state) { auto cookie = server::http::Cookie::FromString( "name1=value1; Domain=domain.com; Path=/; Expires=Wed, 12 Jun 2019 " "16:51:45 GMT; Max-Age=3600; Secure; SameSite=None; HttpOnly" @@ -18,6 +18,6 @@ void http_cookie_serialization(benchmark::State& state) { } } -BENCHMARK(http_cookie_serialization); +BENCHMARK(HttpCookieSerialization); USERVER_NAMESPACE_END diff --git a/core/src/server/net/connection.cpp b/core/src/server/net/connection.cpp index 2904609c9e65..aa0023017b19 100644 --- a/core/src/server/net/connection.cpp +++ b/core/src/server/net/connection.cpp @@ -148,6 +148,8 @@ void Connection::ListenForRequests() noexcept { } bool Connection::WaitOnSocket(engine::Deadline deadline) { + if (!peer_socket_) return false; + bool is_readable = true; if (pending_data_size_ != pending_data_.size()) { if (is_http2_parser_) { @@ -213,7 +215,10 @@ void Connection::ProcessRequest(std::shared_ptr&& request_ptr auto task = HandleQueueItem(request_ptr); SendResponse(*request_ptr); - if (request_ptr->IsUpgradeWebsocket()) request_ptr->DoUpgrade(std::move(peer_socket_), std::move(remote_address_)); + if (request_ptr->IsUpgradeWebsocket()) { + request_ptr->DoUpgrade(std::move(peer_socket_), std::move(remote_address_)); + is_accepting_requests_ = false; + } } bool Connection::ReadSome() { diff --git a/core/src/server/net/listener_config.cpp b/core/src/server/net/listener_config.cpp index 752e047c5162..4ad5693f0c9f 100644 --- a/core/src/server/net/listener_config.cpp +++ b/core/src/server/net/listener_config.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -102,6 +103,12 @@ void PortConfig::ReadTlsSettings(const storages::secdist::SecdistConfig& secdist } } +void PortConfig::InitSslCtx() { + if (tls) { + ssl_ctx = crypto::SslCtx::CreateServerTlsContext(tls_cert_chain, tls_private_key, tls_certificate_authorities); + } +} + } // namespace server::net USERVER_NAMESPACE_END diff --git a/core/src/server/net/listener_config.hpp b/core/src/server/net/listener_config.hpp index daa64cd335ed..e6d11e9d2465 100644 --- a/core/src/server/net/listener_config.hpp +++ b/core/src/server/net/listener_config.hpp @@ -6,10 +6,13 @@ #include #include #include +#include #include #include #include +#include + #include "connection_config.hpp" USERVER_NAMESPACE_BEGIN @@ -29,7 +32,10 @@ struct PortConfig { crypto::PrivateKey tls_private_key; std::vector tls_certificate_authorities; + std::optional ssl_ctx; + void ReadTlsSettings(const storages::secdist::SecdistConfig& secdist); + void InitSslCtx(); }; struct ListenerConfig { diff --git a/core/src/server/net/listener_impl.cpp b/core/src/server/net/listener_impl.cpp index f52f08d97863..da55ab3139d4 100644 --- a/core/src/server/net/listener_impl.cpp +++ b/core/src/server/net/listener_impl.cpp @@ -34,7 +34,7 @@ ListenerImpl::ListenerImpl( stats_(std::make_shared()), data_accounter_(data_accounter) { for (const auto& port : endpoint_info_->listener_config.ports) { - socket_listener_tasks.push_back(engine::CriticalAsyncNoSpan( + socket_listener_tasks_.push_back(engine::CriticalAsyncNoSpan( task_processor_, [this](engine::io::Socket&& request_socket) { while (!engine::current_task::ShouldCancel()) { @@ -58,7 +58,7 @@ ListenerImpl::ListenerImpl( ListenerImpl::~ListenerImpl() { LOG_TRACE() << "Stopping socket listener task"; - for (auto& task : socket_listener_tasks) task.SyncCancel(); + for (auto& task : socket_listener_tasks_) task.SyncCancel(); LOG_TRACE() << "Stopped socket listener task"; connections_.CancelAndWait(); @@ -104,13 +104,10 @@ void ListenerImpl::ProcessConnection(engine::io::Socket peer_socket, const PortC std::unique_ptr socket; auto remote_address = peer_socket.Getpeername(); if (port_config.tls) { - socket = std::make_unique(engine::io::TlsWrapper::StartTlsServer( - std::move(peer_socket), - port_config.tls_cert_chain, - port_config.tls_private_key, - {}, - port_config.tls_certificate_authorities - )); + UASSERT(port_config.ssl_ctx.has_value()); + socket = std::make_unique( + engine::io::TlsWrapper::StartTlsServer(std::move(peer_socket), port_config.ssl_ctx.value(), {}) + ); } else { socket = std::make_unique(std::move(peer_socket)); } diff --git a/core/src/server/net/listener_impl.hpp b/core/src/server/net/listener_impl.hpp index 70355cd08860..5c6e43e67a59 100644 --- a/core/src/server/net/listener_impl.hpp +++ b/core/src/server/net/listener_impl.hpp @@ -38,7 +38,7 @@ class ListenerImpl final { concurrent::BackgroundTaskStorageCore connections_; - std::vector> socket_listener_tasks; + std::vector> socket_listener_tasks_; }; } // namespace server::net diff --git a/core/src/server/request/response_base.cpp b/core/src/server/request/response_base.cpp index a636a9ae7f0c..2ac5c5d7f6fe 100644 --- a/core/src/server/request/response_base.cpp +++ b/core/src/server/request/response_base.cpp @@ -31,22 +31,22 @@ std::chrono::milliseconds ToMsFromStart(std::chrono::steady_clock::time_point tp } // namespace void ResponseDataAccounter::StartRequest(size_t size, std::chrono::steady_clock::time_point create_time) { - count_.Add(1); - current_ += size; + pending_responses_count_.Add(1); + pending_responses_size_in_bytes_ += size; auto ms = ToMsFromStart(create_time); time_sum_.Add(ms.count()); } void ResponseDataAccounter::StopRequest(size_t size, std::chrono::steady_clock::time_point create_time) { - current_ -= size; + pending_responses_size_in_bytes_ -= size; auto ms = ToMsFromStart(create_time); time_sum_.Subtract(ms.count()); - count_.Subtract(1); + pending_responses_count_.Subtract(1); } std::chrono::milliseconds ResponseDataAccounter::GetAvgRequestTime() const { // TODO: race - auto count = count_.NonNegativeRead(); + auto count = pending_responses_count_.NonNegativeRead(); auto time_sum = std::chrono::milliseconds(time_sum_.NonNegativeRead()); auto now_ms = ToMsFromStart(std::chrono::steady_clock::now()); @@ -83,7 +83,9 @@ void ResponseBase::SetReady(std::chrono::steady_clock::time_point now) { is_ready_ = true; } -bool ResponseBase::IsLimitReached() const { return accounter_.GetCurrentLevel() >= accounter_.GetMaxLevel(); } +bool ResponseBase::IsLimitReached() const { + return accounter_.GetPendingResponsesSizeInBytes() >= accounter_.GetMaxPendingResponsesSizeInBytes(); +} void ResponseBase::SetSendFailed(std::chrono::steady_clock::time_point failure_time) { SetSent(0, failure_time); } diff --git a/core/src/server/requests_view.cpp b/core/src/server/requests_view.cpp index 6691254ea431..756c4741da8c 100644 --- a/core/src/server/requests_view.cpp +++ b/core/src/server/requests_view.cpp @@ -12,7 +12,7 @@ const auto kDequeuePollPeriod = std::chrono::milliseconds(100); namespace server { -RequestsView::RequestsView() : queue_(std::make_shared()), job_requests(kDequeueBulkSize) {} +RequestsView::RequestsView() : queue_(std::make_shared()), job_requests_(kDequeueBulkSize) {} RequestsView::~RequestsView() { StopBackgroundWorker(); } @@ -69,10 +69,10 @@ void RequestsView::GarbageCollect() { void RequestsView::HandleQueue() { const std::lock_guard lock(requests_in_flight_mutex_); for (;;) { - const auto count = queue_->try_dequeue_bulk(job_requests.begin(), job_requests.size()); + const auto count = queue_->try_dequeue_bulk(job_requests_.begin(), job_requests_.size()); if (count == 0) break; - for (size_t i = 0; i < count; ++i) requests_in_flight_.push_back(std::move(job_requests[i])); + for (size_t i = 0; i < count; ++i) requests_in_flight_.push_back(std::move(job_requests_[i])); } } diff --git a/core/src/server/requests_view.hpp b/core/src/server/requests_view.hpp index fb877bba06b0..577ccdc87e2a 100644 --- a/core/src/server/requests_view.hpp +++ b/core/src/server/requests_view.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -37,7 +38,7 @@ class RequestsView final { std::shared_ptr queue_; engine::TaskWithResult job_task_; - std::vector job_requests; + std::vector job_requests_; engine::Mutex requests_in_flight_mutex_; std::list requests_in_flight_; diff --git a/core/src/server/server.cpp b/core/src/server/server.cpp index 76e9e5455f50..3879d5e0f9d2 100644 --- a/core/src/server/server.cpp +++ b/core/src/server/server.cpp @@ -40,10 +40,10 @@ struct PortInfo final { bool IsRunning() const noexcept; - std::optional request_handler_; - std::shared_ptr endpoint_info_; - request::ResponseDataAccounter data_accounter_; - std::vector listeners_; + std::optional request_handler; + std::shared_ptr endpoint_info; + request::ResponseDataAccounter data_accounter; + std::vector listeners; }; void PortInfo::Init( @@ -58,44 +58,44 @@ void PortInfo::Init( ? component_context.GetTaskProcessor(*listener_config.task_processor) : engine::current_task::GetTaskProcessor(); - request_handler_.emplace( + request_handler.emplace( component_context, config.logger_access, config.logger_access_tskv, is_monitor, config.server_name ); - endpoint_info_ = std::make_shared(listener_config, *request_handler_); + endpoint_info = std::make_shared(listener_config, *request_handler); const auto& event_thread_pool = task_processor.EventThreadPool(); size_t listener_shards = listener_config.shards ? *listener_config.shards : event_thread_pool.GetSize(); - listeners_.reserve(listener_shards); + listeners.reserve(listener_shards); while (listener_shards--) { - listeners_.emplace_back(endpoint_info_, task_processor, data_accounter_); + listeners.emplace_back(endpoint_info, task_processor, data_accounter); } } void PortInfo::Start() { - UASSERT(request_handler_); - request_handler_->DisableAddHandler(); - for (auto& listener : listeners_) { + UASSERT(request_handler); + request_handler->DisableAddHandler(); + for (auto& listener : listeners) { listener.Start(); } } void PortInfo::Stop() { LOG_TRACE() << "Stopping listeners"; - listeners_.clear(); + listeners.clear(); LOG_TRACE() << "Stopped listeners"; - if (endpoint_info_) { - UASSERT_MSG(endpoint_info_->connection_count == 0, "Not all the connections were closed"); + if (endpoint_info) { + UASSERT_MSG(endpoint_info->connection_count == 0, "Not all the connections were closed"); } LOG_TRACE() << "Stopping request handlers"; - request_handler_.reset(); + request_handler.reset(); LOG_TRACE() << "Stopped request handlers"; } -bool PortInfo::IsRunning() const noexcept { return request_handler_ && request_handler_->IsAddHandlerDisabled(); } +bool PortInfo::IsRunning() const noexcept { return request_handler && request_handler->IsAddHandlerDisabled(); } void WriteRateAndLegacyMetrics(utils::statistics::Writer&& writer, utils::statistics::Rate metric) { writer = metric.value; @@ -156,11 +156,14 @@ ServerImpl::ServerImpl( : config_(std::move(config)) { LOG_DEBUG() << "Creating server"; - for (auto& port : config_.listener.ports) port.ReadTlsSettings(secdist); + for (auto& port : config_.listener.ports) { + port.ReadTlsSettings(secdist); + port.InitSslCtx(); + } main_port_info_.Init(config_, config_.listener, component_context, false); if (config_.max_response_size_in_flight) { - main_port_info_.data_accounter_.SetMaxLevel(*config_.max_response_size_in_flight); + main_port_info_.data_accounter.SetMaxPendingResponsesSizeInBytes(*config_.max_response_size_in_flight); } if (config_.monitor_listener) { monitor_port_info_.Init(config_, *config_.monitor_listener, component_context, true); @@ -175,20 +178,20 @@ ServerImpl::ServerImpl( ServerImpl::~ServerImpl() { Stop(); } void ServerImpl::StartPortInfos() { - UASSERT(main_port_info_.request_handler_); + UASSERT(main_port_info_.request_handler); if (has_requests_view_watchers_.load()) { auto queue = requests_view_.GetQueue(); requests_view_.StartBackgroundWorker(); auto hook = [queue](std::shared_ptr request) mutable { queue->enqueue(std::move(request)); }; - main_port_info_.request_handler_->SetNewRequestHook(hook); - if (monitor_port_info_.request_handler_) { - monitor_port_info_.request_handler_->SetNewRequestHook(hook); + main_port_info_.request_handler->SetNewRequestHook(hook); + if (monitor_port_info_.request_handler) { + monitor_port_info_.request_handler->SetNewRequestHook(hook); } } main_port_info_.Start(); - if (monitor_port_info_.request_handler_) { + if (monitor_port_info_.request_handler) { monitor_port_info_.Start(); } else { LOG_WARNING() << "No 'listener-monitor' in 'server' component"; @@ -211,7 +214,7 @@ void ServerImpl::Stop() { void ServerImpl::AddHandler(const handlers::HttpHandlerBase& handler, engine::TaskProcessor& task_processor) { UASSERT(!main_port_info_.IsRunning()); - if (handler.IsMonitor() && !monitor_port_info_.request_handler_) { + if (handler.IsMonitor() && !monitor_port_info_.request_handler) { throw std::logic_error( "Attempt to register a handler for 'listener-monitor' that was not " "configured in 'server' section of the component config" @@ -220,14 +223,14 @@ void ServerImpl::AddHandler(const handlers::HttpHandlerBase& handler, engine::Ta if (handler.IsMonitor()) { UINVARIANT( - monitor_port_info_.request_handler_, + monitor_port_info_.request_handler, "Attempt to register monitor handler while the server has no " "'listener-monitor'" ); - monitor_port_info_.request_handler_->AddHandler(handler, task_processor); + monitor_port_info_.request_handler->AddHandler(handler, task_processor); } else { - UASSERT(main_port_info_.request_handler_); - main_port_info_.request_handler_->AddHandler(handler, task_processor); + UASSERT(main_port_info_.request_handler); + main_port_info_.request_handler->AddHandler(handler, task_processor); } if (!handler.IsMonitor()) { @@ -243,17 +246,17 @@ std::size_t ServerImpl::GetThrottlableHandlersCount() const { } std::chrono::milliseconds ServerImpl::GetAvgRequestTimeMs() const { - return main_port_info_.data_accounter_.GetAvgRequestTime(); + return main_port_info_.data_accounter.GetAvgRequestTime(); } const http::HttpRequestHandler& ServerImpl::GetHttpRequestHandler(bool is_monitor) const { if (is_monitor) { - UASSERT(monitor_port_info_.request_handler_); - return *monitor_port_info_.request_handler_; + UASSERT(monitor_port_info_.request_handler); + return *monitor_port_info_.request_handler; } - UASSERT(main_port_info_.request_handler_); - return *main_port_info_.request_handler_; + UASSERT(main_port_info_.request_handler); + return *main_port_info_.request_handler; } net::StatsAggregation ServerImpl::GetServerStats() const { @@ -261,7 +264,7 @@ net::StatsAggregation ServerImpl::GetServerStats() const { const std::shared_lock lock{on_stop_mutex_}; if (is_stopping_) return summary; - for (const auto& listener : main_port_info_.listeners_) { + for (const auto& listener : main_port_info_.listeners) { summary += listener.GetStats(); } @@ -289,8 +292,8 @@ void ServerImpl::WriteTotalHandlerStatistics(utils::statistics::Writer& writer) return; } - UASSERT(main_port_info_.request_handler_); - const auto& handlers = main_port_info_.request_handler_->GetHandlerInfoIndex().GetHandlers(); + UASSERT(main_port_info_.request_handler); + const auto& handlers = main_port_info_.request_handler->GetHandlerInfoIndex().GetHandlers(); for (const auto handler_ptr : handlers) { for (const auto method : handler_ptr->GetAllowedMethods()) { @@ -304,13 +307,13 @@ void ServerImpl::WriteTotalHandlerStatistics(utils::statistics::Writer& writer) } void ServerImpl::SetRpsRatelimitStatusCode(http::HttpStatus status_code) { - UASSERT(main_port_info_.request_handler_); - main_port_info_.request_handler_->SetRpsRatelimitStatusCode(status_code); + UASSERT(main_port_info_.request_handler); + main_port_info_.request_handler->SetRpsRatelimitStatusCode(status_code); } void ServerImpl::SetRpsRatelimit(std::optional rps) { - UASSERT(main_port_info_.request_handler_); - main_port_info_.request_handler_->SetRpsRatelimit(rps); + UASSERT(main_port_info_.request_handler); + main_port_info_.request_handler->SetRpsRatelimit(rps); } std::uint64_t ServerImpl::GetTotalRequests() const { @@ -323,16 +326,16 @@ Server::Server( const storages::secdist::SecdistConfig& secdist, const components::ComponentContext& component_context ) - : pimpl(std::make_unique(std::move(config), secdist, component_context)) {} + : pimpl_(std::make_unique(std::move(config), secdist, component_context)) {} Server::~Server() = default; -const ServerConfig& Server::GetConfig() const { return pimpl->GetServerConfig(); } +const ServerConfig& Server::GetConfig() const { return pimpl_->GetServerConfig(); } -std::vector Server::GetCommonMiddlewares() const { return pimpl->GetMiddlewares(); } +std::vector Server::GetCommonMiddlewares() const { return pimpl_->GetMiddlewares(); } void Server::WriteMonitorData(utils::statistics::Writer& writer) const { - const auto server_stats = pimpl->GetServerStats(); + const auto server_stats = pimpl_->GetServerStats(); if (auto conn_stats = writer["connections"]) { conn_stats["active"] = server_stats.active_connections; WriteRateAndLegacyMetrics(conn_stats["opened"], server_stats.connections_created); @@ -341,7 +344,7 @@ void Server::WriteMonitorData(utils::statistics::Writer& writer) const { if (auto request_stats = writer["requests"]) { request_stats["active"] = server_stats.active_request_count; - request_stats["avg-lifetime-ms"] = pimpl->GetAvgRequestTimeMs().count(); + request_stats["avg-lifetime-ms"] = pimpl_->GetAvgRequestTimeMs().count(); WriteRateAndLegacyMetrics(request_stats["processed"], server_stats.requests_processed_count); request_stats["parsing"] = server_stats.parser_stats.parsing_request_count; @@ -356,36 +359,36 @@ void Server::WriteMonitorData(utils::statistics::Writer& writer) const { } void Server::WriteTotalHandlerStatistics(utils::statistics::Writer& writer) const { - pimpl->WriteTotalHandlerStatistics(writer); + pimpl_->WriteTotalHandlerStatistics(writer); } -net::StatsAggregation Server::GetServerStats() const { return pimpl->GetServerStats(); } +net::StatsAggregation Server::GetServerStats() const { return pimpl_->GetServerStats(); } void Server::AddHandler(const handlers::HttpHandlerBase& handler, engine::TaskProcessor& task_processor) { - pimpl->AddHandler(handler, task_processor); + pimpl_->AddHandler(handler, task_processor); } -size_t Server::GetThrottlableHandlersCount() const { return pimpl->GetThrottlableHandlersCount(); } +size_t Server::GetThrottlableHandlersCount() const { return pimpl_->GetThrottlableHandlersCount(); } const http::HttpRequestHandler& Server::GetHttpRequestHandler(bool is_monitor) const { - return pimpl->GetHttpRequestHandler(is_monitor); + return pimpl_->GetHttpRequestHandler(is_monitor); } void Server::Start() { LOG_INFO() << "Starting server"; - pimpl->StartPortInfos(); + pimpl_->StartPortInfos(); LOG_INFO() << "Server is started"; } -void Server::Stop() { pimpl->Stop(); } +void Server::Stop() { pimpl_->Stop(); } -RequestsView& Server::GetRequestsView() { return pimpl->GetRequestsView(); } +RequestsView& Server::GetRequestsView() { return pimpl_->GetRequestsView(); } -void Server::SetRpsRatelimit(std::optional rps) { pimpl->SetRpsRatelimit(rps); } +void Server::SetRpsRatelimit(std::optional rps) { pimpl_->SetRpsRatelimit(rps); } -void Server::SetRpsRatelimitStatusCode(http::HttpStatus status_code) { pimpl->SetRpsRatelimitStatusCode(status_code); } +void Server::SetRpsRatelimitStatusCode(http::HttpStatus status_code) { pimpl_->SetRpsRatelimitStatusCode(status_code); } -std::uint64_t Server::GetTotalRequests() const { return pimpl->GetTotalRequests(); } +std::uint64_t Server::GetTotalRequests() const { return pimpl_->GetTotalRequests(); } } // namespace server diff --git a/core/src/server/websocket/protocol.cpp b/core/src/server/websocket/protocol.cpp index 875e267ba89f..617c5d281d9f 100644 --- a/core/src/server/websocket/protocol.cpp +++ b/core/src/server/websocket/protocol.cpp @@ -56,13 +56,23 @@ union Mask32 { }; void XorMaskInplace(uint8_t* dest, size_t len, Mask32 mask) { - auto* dest32 = reinterpret_cast(dest); - while (len >= sizeof(uint32_t)) { - *(dest32++) ^= mask.mask32; - len -= sizeof(uint32_t); + // Check if the pointer is aligned for uint32_t operations + const auto alignment = reinterpret_cast(dest) % sizeof(uint32_t); + + if (alignment == 0 && len >= sizeof(uint32_t)) { + // Pointer is aligned, we can safely use uint32_t operations + auto* dest32 = reinterpret_cast(dest); + while (len >= sizeof(uint32_t)) { + *(dest32++) ^= mask.mask32; + len -= sizeof(uint32_t); + } + dest = reinterpret_cast(dest32); + } + + // Process remaining bytes (or all bytes if unaligned) using byte operations + for (size_t i = 0; i < len; ++i) { + dest[i] ^= mask.mask8[i % 4]; } - auto* dest8 = reinterpret_cast(dest32); - for (unsigned i = 0; i < len; ++i) *(dest8++) ^= mask.mask8[i]; } template @@ -89,12 +99,12 @@ DataFrameHeader(utils::span data, bool is_text, Continuation is if (is_continuation == Continuation::kYes) hdr->bits.opcode = kContinuation; if (data.size() <= 125) { - hdr->bits.payloadLen = data.size(); + hdr->bits.payload_len = data.size(); } else if (data.size() <= INT16_MAX) { - hdr->bits.payloadLen = 126; + hdr->bits.payload_len = 126; PushRaw(boost::endian::native_to_big(static_cast(data.size())), frame); } else { - hdr->bits.payloadLen = 127; + hdr->bits.payload_len = 127; PushRaw(boost::endian::native_to_big(data.size()), frame); } return frame; @@ -106,7 +116,7 @@ std::string CloseFrame(CloseStatusInt status_code) { auto* hdr = reinterpret_cast(frame.data()); hdr->bits.fin = 1; hdr->bits.opcode = kClose; - hdr->bits.payloadLen = sizeof(status_code); + hdr->bits.payload_len = sizeof(status_code); auto* payload = reinterpret_cast(&frame[sizeof(WSHeader)]); *payload = boost::endian::native_to_big(status_code); @@ -121,7 +131,7 @@ std::array MakeControlFrame(WSOpcodes opcode, utils::spa hdr->bits.fin = 1; hdr->bits.opcode = opcode; - hdr->bits.payloadLen = data.size(); + hdr->bits.payload_len = data.size(); return frame; } @@ -165,9 +175,9 @@ CloseStatus ReadWSFrameImpl( if (engine::current_task::ShouldCancel()) return CloseStatus::kGoingAway; const bool isDataFrame = (hdr.bits.opcode & (kText | kBinary)) || hdr.bits.opcode == kContinuation; - if (hdr.bits.payloadLen <= 125) { - payload_len = hdr.bits.payloadLen; - } else if (hdr.bits.payloadLen == 126) { + if (hdr.bits.payload_len <= 125) { + payload_len = hdr.bits.payload_len; + } else if (hdr.bits.payload_len == 126) { uint16_t payloadLen16 = 0; RecvExactly(io, AsWritableBytes(MakeSpan(&payloadLen16, 1)), {}); payload_len = boost::endian::big_to_native(payloadLen16); @@ -179,7 +189,7 @@ CloseStatus ReadWSFrameImpl( } if (engine::current_task::ShouldCancel()) return CloseStatus::kGoingAway; - if (!isDataFrame && hdr.bits.payloadLen > 125) { + if (!isDataFrame && hdr.bits.payload_len > 125) { // control frame should not have extended payload return CloseStatus::kProtocolError; } @@ -203,7 +213,9 @@ CloseStatus ReadWSFrameImpl( RecvExactly(io, MakeSpan(frame.payload->data() + newPayloadOffset, payload_len), {}); if (engine::current_task::ShouldCancel()) return CloseStatus::kGoingAway; - if (mask.mask32) XorMaskInplace(reinterpret_cast(frame.payload->data()), frame.payload->size(), mask); + // Apply masking only to the current frame's data, not to the entire buffer + if (mask.mask32) + XorMaskInplace(reinterpret_cast(frame.payload->data() + newPayloadOffset), payload_len, mask); } const char opcode = hdr.bits.opcode; const char fin = hdr.bits.fin; diff --git a/core/src/server/websocket/protocol.hpp b/core/src/server/websocket/protocol.hpp index 8430cdbe5b2e..9a6bfffe433d 100644 --- a/core/src/server/websocket/protocol.hpp +++ b/core/src/server/websocket/protocol.hpp @@ -53,7 +53,7 @@ union WSHeader { unsigned char reserved : 3; unsigned char fin : 1; - unsigned char payloadLen : 7; + unsigned char payload_len : 7; unsigned char mask : 1; } bits; uint16_t bytes = 0; diff --git a/core/src/server/websocket/server.cpp b/core/src/server/websocket/server.cpp index e99f2cfa315e..6534be53a71a 100644 --- a/core/src/server/websocket/server.cpp +++ b/core/src/server/websocket/server.cpp @@ -39,7 +39,7 @@ Config Parse(const yaml_config::YamlConfig& config, formats::parse::To) class WebSocketConnectionImpl final : public WebSocketConnection { public: private: - std::unique_ptr io; + std::unique_ptr io_; struct MessageExtended final { utils::span data; @@ -59,17 +59,17 @@ class WebSocketConnectionImpl final : public WebSocketConnection { // only a single task calling Recv(). impl::FrameParserState frame_; - Config config; + Config config_; std::atomic ping_pending_count_{0}; public: WebSocketConnectionImpl( - std::unique_ptr io_, + std::unique_ptr io, const engine::io::Sockaddr& remote_addr, const Config& server_config ) - : io(std::move(io_)), remote_addr_(remote_addr), config(server_config) {} + : io_(std::move(io)), remote_addr_(remote_addr), config_(server_config) {} ~WebSocketConnectionImpl() override { LOG_TRACE() << "Websocket connection closed"; } @@ -81,31 +81,31 @@ class WebSocketConnectionImpl final : public WebSocketConnection { LOG_TRACE() << "Write message " << message.data.size() << " bytes"; if (message.opcode == impl::WSOpcodes::kPing) { - SendExactly(*io, impl::frames::PingFrame(), {}); + SendExactly(*io_, impl::frames::PingFrame(), {}); } else if (message.opcode == impl::WSOpcodes::kPong) { const auto control_frame = impl::frames::MakeControlFrame(impl::WSOpcodes::kPong, message.data); - SendExactly(*io, control_frame, message.data); + SendExactly(*io_, control_frame, message.data); } else if (message.close_status.has_value()) { const auto close_frame = impl::frames::CloseFrame(static_cast(message.close_status.value())); - SendExactly(*io, close_frame, {}); + SendExactly(*io_, close_frame, {}); } else if (!message.data.empty()) { utils::span data_to_send{message.data}; auto continuation = impl::frames::Continuation::kNo; - while (data_to_send.size() > config.fragment_size && config.fragment_size > 0) { + while (data_to_send.size() > config_.fragment_size && config_.fragment_size > 0) { const auto data_frame_header = impl::frames::DataFrameHeader( - data_to_send.first(config.fragment_size), + data_to_send.first(config_.fragment_size), message.opcode == impl::WSOpcodes::kText, continuation, impl::frames::Final::kNo ); - SendExactly(*io, data_frame_header, data_to_send.first(config.fragment_size)); + SendExactly(*io_, data_frame_header, data_to_send.first(config_.fragment_size)); continuation = impl::frames::Continuation::kYes; - data_to_send = data_to_send.last(data_to_send.size() - config.fragment_size); + data_to_send = data_to_send.last(data_to_send.size() - config_.fragment_size); } const auto data_frame_header = impl::frames::DataFrameHeader( data_to_send, message.opcode == impl::WSOpcodes::kText, continuation, impl::frames::Final::kYes ); - SendExactly(*io, data_frame_header, data_to_send); + SendExactly(*io_, data_frame_header, data_to_send); } } @@ -146,12 +146,12 @@ class WebSocketConnectionImpl final : public WebSocketConnection { if (do_not_wait_for_message_header) { const auto opt_status_raw = - ReadWSFrameDontWaitForHeader(frame_, *io, config.max_remote_payload, payload_len); + ReadWSFrameDontWaitForHeader(frame_, *io_, config_.max_remote_payload, payload_len); if (!opt_status_raw) return false; status_raw = *opt_status_raw; } else { // ReadWSFrame() returns kGoingAway in case of task cancellation - status_raw = ReadWSFrame(frame_, *io, config.max_remote_payload, payload_len); + status_raw = ReadWSFrame(frame_, *io_, config_.max_remote_payload, payload_len); } const auto status = static_cast(status_raw); diff --git a/core/src/storages/query.cpp b/core/src/storages/query.cpp index d4a77d4c54f0..b05fae6986ff 100644 --- a/core/src/storages/query.cpp +++ b/core/src/storages/query.cpp @@ -6,11 +6,11 @@ namespace storages { struct Query::NameViewVisitor { std::optional operator()(const DynamicStrings& x) const noexcept { - return (x.name_ ? std::optional{x.name_->GetUnderlying()} : std::nullopt); + return (x.name ? std::optional{x.name->GetUnderlying()} : std::nullopt); } std::optional operator()(const StaticStrings& x) const noexcept { - return std::optional{x.name_}; + return std::optional{x.name}; } }; @@ -19,7 +19,7 @@ std::optional Query::GetOptionalNameView() const noexcept { } utils::zstring_view Query::GetStatementView() const noexcept { - return std::visit([](const auto& x) { return utils::zstring_view{x.statement_}; }, data_); + return std::visit([](const auto& x) { return utils::zstring_view{x.statement}; }, data_); } } // namespace storages diff --git a/core/src/testing_test.cpp b/core/src/testing_test.cpp index cbc4e04efd21..40674dfdeab7 100644 --- a/core/src/testing_test.cpp +++ b/core/src/testing_test.cpp @@ -16,7 +16,7 @@ using utest::SimpleServer; const std::string kOkRequest = "OK"; const std::string kOkResponse = "OK RESPONSE DATA"; -SimpleServer::Response assert_received_ok(const SimpleServer::Request& r) { +SimpleServer::Response AssertReceivedOk(const SimpleServer::Request& r) { EXPECT_EQ(r, kOkRequest) << "SimpleServer received: " << r; return {kOkResponse, SimpleServer::Response::kWriteAndClose}; } @@ -24,7 +24,7 @@ SimpleServer::Response assert_received_ok(const SimpleServer::Request& r) { } // namespace UTEST(SimpleServer, ExampleTcpIpV4) { - const SimpleServer s(assert_received_ok); + const SimpleServer s(AssertReceivedOk); // ... invoke code that sends "OK" to localhost auto addr = engine::io::Sockaddr::MakeIPv4LoopbackAddress(); @@ -51,7 +51,7 @@ UTEST(SimpleServer, NothingReceived) { } UTEST(SimpleServer, ExampleTcpIpV6) { - const SimpleServer s(assert_received_ok, SimpleServer::kTcpIpV6); + const SimpleServer s(AssertReceivedOk, SimpleServer::kTcpIpV6); // ... invoke code that sends "OK" to localhost:8080 or localhost:8042. auto addr = engine::io::Sockaddr::MakeLoopbackAddress(); diff --git a/core/src/testsuite/impl/actions/control.cpp b/core/src/testsuite/impl/actions/control.cpp index aa688ab95cfc..fcc196fb5867 100644 --- a/core/src/testsuite/impl/actions/control.cpp +++ b/core/src/testsuite/impl/actions/control.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/core/src/testsuite/testpoint.cpp b/core/src/testsuite/testpoint.cpp index 8995e15aa535..145bd4a998e7 100644 --- a/core/src/testsuite/testpoint.cpp +++ b/core/src/testsuite/testpoint.cpp @@ -11,6 +11,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -76,6 +77,7 @@ void ExecuteTestpointCoro( const TestpointScope tp_scope; if (!tp_scope) return; + utils::trx_tracker::CheckDisabler disabler; tp_scope.GetClient().Execute(name, json, callback); } diff --git a/core/src/tracing/any_value.cpp b/core/src/tracing/any_value.cpp index 7d7b7ff843c7..9f69d24db609 100644 --- a/core/src/tracing/any_value.cpp +++ b/core/src/tracing/any_value.cpp @@ -37,11 +37,11 @@ utils::StringLiteral ToString(ValueIndex index) { UINVARIANT(false, fmt::format("Unexpected index: {}", static_cast(index))); } -template +template void CheckIndex(std::uint8_t actual) { UASSERT_MSG( - actual == expected, - fmt::format("Expected: '{}', Actual: '{}'", ToString(expected), ToString(static_cast(actual))) + actual == Expected, + fmt::format("Expected: '{}', Actual: '{}'", ToString(Expected), ToString(static_cast(actual))) ); } diff --git a/core/src/tracing/span.cpp b/core/src/tracing/span.cpp index ee6d0de8b0f9..cd9e258650c0 100644 --- a/core/src/tracing/span.cpp +++ b/core/src/tracing/span.cpp @@ -68,17 +68,17 @@ TsBuffer StartTsToString(std::chrono::system_clock::time_point start) { // Maintain coro-local span stack to identify "current span" in O(1). engine::TaskLocalVariable task_local_spans; -template -utils::SmallString GenerateId() { +template +utils::SmallString GenerateId() { using Chunk = utils::RandomBase::result_type; static constexpr std::size_t kHexChunkSize = utils::encoding::LengthInHexForm(sizeof(Chunk)); - static_assert(kIdSize % kHexChunkSize == 0); + static_assert(IdSize % kHexChunkSize == 0); static_assert(utils::RandomBase::min() == std::numeric_limits::min()); static_assert(utils::RandomBase::max() == std::numeric_limits::max()); - utils::SmallString result; + utils::SmallString result; - result.resize_and_overwrite(kIdSize, [](char* data, std::size_t size) { + result.resize_and_overwrite(IdSize, [](char* data, std::size_t size) { utils::WithDefaultRandom([&](auto& rnd) { for (std::size_t pos = 0; pos < size; pos += kHexChunkSize) { const auto random_value = rnd(); @@ -232,7 +232,7 @@ std::optional Span::Impl::GetSpanIdForChildLogs() const { } void Span::OptionalDeleter::operator()(Span::Impl* impl) const noexcept { - if (do_delete) { + if (do_delete_) { std::default_delete{}(impl); } } diff --git a/core/src/tracing/tracing_benchmark.cpp b/core/src/tracing/tracing_benchmark.cpp index 415a37a835cb..b0d2dcedf109 100644 --- a/core/src/tracing/tracing_benchmark.cpp +++ b/core/src/tracing/tracing_benchmark.cpp @@ -16,7 +16,7 @@ class NoopLogger : public logging::impl::TextLogger { void Flush() override {} }; -void tracing_noop_ctr(benchmark::State& state) { +void TracingNoopCtr(benchmark::State& state) { engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { tracing::Span tmp = tracing::Span::MakeRootSpan("name"); @@ -25,9 +25,9 @@ void tracing_noop_ctr(benchmark::State& state) { } }); } -BENCHMARK(tracing_noop_ctr); +BENCHMARK(TracingNoopCtr); -void tracing_happy_log(benchmark::State& state) { +void TracingHappyLog(benchmark::State& state) { const logging::DefaultLoggerGuard guard{std::make_shared()}; engine::RunStandalone([&] { @@ -37,7 +37,7 @@ void tracing_happy_log(benchmark::State& state) { } }); } -BENCHMARK(tracing_happy_log); +BENCHMARK(TracingHappyLog); tracing::Span GetSpanWithOpentracingHttpTags() { auto span = tracing::Span::MakeRootSpan("name"); @@ -47,7 +47,7 @@ tracing::Span GetSpanWithOpentracingHttpTags() { return span; } -void tracing_opentracing_ctr(benchmark::State& state) { +void TracingOpentracingCtr(benchmark::State& state) { auto logger = logging::MakeNullLogger(); engine::RunStandalone([&] { for ([[maybe_unused]] auto _ : state) { @@ -56,7 +56,7 @@ void tracing_opentracing_ctr(benchmark::State& state) { } }); } -BENCHMARK(tracing_opentracing_ctr); +BENCHMARK(TracingOpentracingCtr); } // namespace diff --git a/core/src/utils/async.cpp b/core/src/utils/async.cpp index fd80821c9e06..afe2ea31e9d0 100644 --- a/core/src/utils/async.cpp +++ b/core/src/utils/async.cpp @@ -13,19 +13,19 @@ namespace utils::impl { struct SpanWrapCall::Impl { explicit Impl(std::string&& name, InheritVariables inherit_variables, const SourceLocation& location); - tracing::InPlaceSpan span_; - engine::impl::task_local::Storage storage_; + tracing::InPlaceSpan span; + engine::impl::task_local::Storage storage; }; SpanWrapCall::Impl::Impl(std::string&& name, InheritVariables inherit_variables, const SourceLocation& location) - : span_(std::move(name), tracing::InPlaceSpan::DetachedTag{}, location) { + : span(std::move(name), tracing::InPlaceSpan::DetachedTag{}, location) { if (!engine::current_task::IsTaskProcessorThread()) { return; } if (inherit_variables == InheritVariables::kYes) { - storage_.InheritFrom(engine::impl::task_local::GetCurrentStorage()); + storage.InheritFrom(engine::impl::task_local::GetCurrentStorage()); } else { - baggage::kInheritedBaggage.InheritTo(storage_, engine::impl::task_local::InternalTag{}); + baggage::kInheritedBaggage.InheritTo(storage, engine::impl::task_local::InternalTag{}); } } @@ -33,8 +33,8 @@ SpanWrapCall::SpanWrapCall(std::string&& name, InheritVariables inherit_variable : pimpl_(std::move(name), inherit_variables, location) {} void SpanWrapCall::DoBeforeInvoke() { - engine::impl::task_local::GetCurrentStorage().InitializeFrom(std::move(pimpl_->storage_)); - pimpl_->span_.Get().AttachToCoroStack(); + engine::impl::task_local::GetCurrentStorage().InitializeFrom(std::move(pimpl_->storage)); + pimpl_->span.Get().AttachToCoroStack(); } SpanWrapCall::~SpanWrapCall() = default; diff --git a/core/src/utils/datetime.cpp b/core/src/utils/datetime.cpp index f3709007819e..8f5cd675070f 100644 --- a/core/src/utils/datetime.cpp +++ b/core/src/utils/datetime.cpp @@ -24,7 +24,7 @@ namespace { std::optional DoGetOptionalTimezone(const std::string& tzname) { #if defined(BSD) && !defined(__APPLE__) - if (tzname == "GMT") return GetOptionalTimezone("UTC"); + if (tzname == "GMT") return DoGetOptionalTimezone("UTC"); #endif cctz::time_zone tz; if (!load_time_zone(tzname, &tz)) { @@ -33,23 +33,6 @@ std::optional DoGetOptionalTimezone(const std::string& tzname) return tz; } -std::optional GetOptionalTimezone(const std::string& tzname) { - if (engine::current_task::IsTaskProcessorThread()) { - static rcu::RcuMap> map; - auto it = map.Get(tzname); - if (it) return *it; - - // DoGetOptionalTimezone() may access filesystem, run it in blocking task processor - auto [value, _] = - map.Emplace(tzname, engine::AsyncNoSpan(engine::current_task::GetBlockingTaskProcessor(), [&tzname] { - return DoGetOptionalTimezone(tzname); - }).Get()); - return *value; - } else { - return DoGetOptionalTimezone(tzname); - } -} - cctz::time_zone GetTimezone(const std::string& tzname) { auto tz = GetOptionalTimezone(tzname); if (!tz.has_value()) { @@ -57,6 +40,7 @@ cctz::time_zone GetTimezone(const std::string& tzname) { } return *tz; } + } // namespace std::string @@ -99,6 +83,23 @@ time_t Unlocalize(const cctz::civil_second& local_tp, const std::string& timezon return Timestamp(cctz::convert(local_tp, GetTimezone(timezone))); } +std::optional GetOptionalTimezone(const std::string& tzname) { + if (engine::current_task::IsTaskProcessorThread()) { + static rcu::RcuMap> map; + auto it = map.Get(tzname); + if (it) return *it; + + // DoGetOptionalTimezone() may access filesystem, run it in blocking task processor + auto [value, _] = + map.Emplace(tzname, engine::AsyncNoSpan(engine::current_task::GetBlockingTaskProcessor(), [&tzname] { + return DoGetOptionalTimezone(tzname); + }).Get()); + return *value; + } else { + return DoGetOptionalTimezone(tzname); + } +} + } // namespace utils::datetime USERVER_NAMESPACE_END diff --git a/core/src/utils/hedged_request_test.cpp b/core/src/utils/hedged_request_test.cpp index 43c88a3f8228..5168e657fa27 100644 --- a/core/src/utils/hedged_request_test.cpp +++ b/core/src/utils/hedged_request_test.cpp @@ -87,20 +87,20 @@ class TestStrategy { } std::optional ProcessReply(RequestType&& request) { - reply_ = std::move(request).Get(); + reply = std::move(request).Get(); event_log.push_back({Event::ProcessReply, request.attempt}); if (attempt_program.size() <= request.attempt) return std::nullopt; return attempt_program[request.attempt].process_reply_result; } - std::optional ExtractReply() { return reply_; } + std::optional ExtractReply() { return reply; } void Finish(RequestType&& request) { event_log.push_back({Event::Finish, request.attempt}); if (request.task.IsValid()) request.task.SyncCancel(); } - std::optional reply_; + std::optional reply; AttemptProgram attempt_program; EventLog& event_log; }; diff --git a/core/src/utils/jemalloc.cpp b/core/src/utils/jemalloc.cpp index 4fb8316bc01c..c451f4990e02 100644 --- a/core/src/utils/jemalloc.cpp +++ b/core/src/utils/jemalloc.cpp @@ -16,8 +16,11 @@ namespace utils::jemalloc { namespace { #ifndef USERVER_FEATURE_JEMALLOC_ENABLED + +// NOLINTNEXTLINE(readability-identifier-naming) int mallctl(const char*, void*, size_t*, void*, size_t) { return ENOTSUP; } +// NOLINTNEXTLINE(readability-identifier-naming) void malloc_stats_print(void (*write_cb)(void*, const char*), void* je_cbopaque, const char*) { write_cb(je_cbopaque, "(libjemalloc support is disabled)"); } diff --git a/core/src/utils/periodic_task.cpp b/core/src/utils/periodic_task.cpp index bced8cb9b43c..0d47d4f61c82 100644 --- a/core/src/utils/periodic_task.cpp +++ b/core/src/utils/periodic_task.cpp @@ -6,9 +6,13 @@ #include #include +#include +#include #include #include +#include #include +#include #include #include #include @@ -28,6 +32,38 @@ auto TieSettings(const PeriodicTask::Settings& settings) { } // namespace +class PeriodicTask::Impl final { +public: + enum class SuspendState { kRunning, kSuspended }; + + std::string name; + std::atomic is_name_set{false}; + PeriodicTask::Callback callback; + engine::TaskWithResult task; + rcu::Variable settings; + engine::SingleConsumerEvent changed_event; + std::atomic should_force_step{false}; + std::optional mutate_period_random; + + // For kNow only + engine::Mutex step_mutex; + std::atomic suspend_state; + + std::optional registration_holder; + + Impl() : settings(std::chrono::seconds(1)), suspend_state(SuspendState::kRunning) {} + + void DoStart(); + void Run(); + bool Step(); + bool StepDebug(bool preserve_span); + bool DoStep(); + std::chrono::milliseconds MutatePeriod(std::chrono::milliseconds period); + std::string_view GetName() const noexcept; + void SuspendDebug(); + void ResumeDebug(); +}; + bool PeriodicTask::Settings::operator==(const Settings& other) const noexcept { return TieSettings(*this) == TieSettings(other); } @@ -36,58 +72,59 @@ bool PeriodicTask::Settings::operator!=(const Settings& other) const noexcept { return TieSettings(*this) != TieSettings(other); } -PeriodicTask::PeriodicTask() : settings_(std::chrono::seconds(1)), suspend_state_(SuspendState::kRunning) {} +PeriodicTask::PeriodicTask() : impl_() {} -PeriodicTask::PeriodicTask(std::string name, Settings settings, Callback callback) - : name_(std::move(name)), - is_name_set_(true), - callback_(std::move(callback)), - settings_(std::move(settings)), - suspend_state_(SuspendState::kRunning) { - DoStart(); +PeriodicTask::PeriodicTask(std::string name, Settings settings, Callback callback) : impl_() { + impl_->name = std::move(name); + impl_->is_name_set = true; + impl_->callback = std::move(callback); + impl_->settings.Assign(std::move(settings)); + impl_->suspend_state = Impl::SuspendState::kRunning; + impl_->DoStart(); } PeriodicTask::~PeriodicTask() { - registration_holder_ = std::nullopt; + impl_->registration_holder = std::nullopt; Stop(); } void PeriodicTask::Start(std::string name, Settings settings, Callback callback) { UASSERT_MSG(!name.empty(), "Periodic task must have a name"); - auto settings_ptr = settings_.StartWriteEmplace(std::move(settings)); + auto settings_ptr = impl_->settings.StartWriteEmplace(std::move(settings)); // Set name_ under the 'settings_' mutex. - if (!is_name_set_) { - name_ = std::move(name); - is_name_set_ = true; + if (!impl_->is_name_set) { + impl_->name = std::move(name); + impl_->is_name_set = true; } else { UINVARIANT( - name_ == name, fmt::format("PeriodicTask name must not be changed on the fly, old={}, new={}", name_, name) + impl_->name == name, + fmt::format("PeriodicTask name must not be changed on the fly, old={}, new={}", impl_->name, name) ); } // Stop here so that if the invariant above fails, the task is not affected. Stop(); - callback_ = std::move(callback); + impl_->callback = std::move(callback); settings_ptr.Commit(); - DoStart(); + impl_->DoStart(); } -void PeriodicTask::DoStart() { +void PeriodicTask::Impl::DoStart() { LOG_DEBUG() << "Starting PeriodicTask with name=" << GetName(); - auto settings_ptr = settings_.Read(); + auto settings_ptr = settings.Read(); auto& task_processor = settings_ptr->task_processor ? *settings_ptr->task_processor : engine::current_task::GetTaskProcessor(); - task_ = engine::CriticalAsyncNoSpan(task_processor, &PeriodicTask::Run, this); + task = engine::CriticalAsyncNoSpan(task_processor, &PeriodicTask::Impl::Run, this); } void PeriodicTask::Stop() noexcept { - const auto name = GetName(); + const auto name = impl_->GetName(); try { if (IsRunning()) { LOG_INFO() << "Stopping PeriodicTask with name=" << name; - task_.SyncCancel(); - changed_event_.Reset(); - should_force_step_ = false; - task_ = engine::TaskWithResult(); + impl_->task.SyncCancel(); + impl_->changed_event.Reset(); + impl_->should_force_step = false; + impl_->task = engine::TaskWithResult(); LOG_INFO() << "Stopped PeriodicTask with name=" << name; } } catch (std::exception& e) { @@ -100,7 +137,7 @@ void PeriodicTask::Stop() noexcept { void PeriodicTask::SetSettings(Settings settings) { bool should_notify_task{}; { - auto writer = settings_.StartWrite(); + auto writer = impl_->settings.StartWrite(); if (settings == *writer) { // Updating an RCU is slow, better to avoid it when possible. return; @@ -112,14 +149,14 @@ void PeriodicTask::SetSettings(Settings settings) { } if (should_notify_task) { - LOG_DEBUG() << "periodic task settings have changed, signalling name=" << GetName(); - changed_event_.Send(); + LOG_DEBUG() << "periodic task settings have changed, signalling name=" << impl_->GetName(); + impl_->changed_event.Send(); } } void PeriodicTask::ForceStepAsync() { - should_force_step_ = true; - changed_event_.Send(); + impl_->should_force_step = true; + impl_->changed_event.Send(); } bool PeriodicTask::SynchronizeDebug(bool preserve_span) { @@ -127,16 +164,16 @@ bool PeriodicTask::SynchronizeDebug(bool preserve_span) { return false; } - return StepDebug(preserve_span); + return impl_->StepDebug(preserve_span); } -bool PeriodicTask::IsRunning() const { return task_.IsValid(); } +bool PeriodicTask::IsRunning() const { return impl_->task.IsValid(); } -void PeriodicTask::Run() { +void PeriodicTask::Impl::Run() { bool skip_step = false; { - auto settings = settings_.Read(); - if (!(settings->flags & Flags::kNow)) { + auto l_settings = settings.Read(); + if (!(l_settings->flags & Flags::kNow)) { skip_step = true; } } @@ -149,40 +186,40 @@ void PeriodicTask::Run() { no_exception = Step(); } - const auto settings = settings_.Read(); - auto period = settings->period; - const auto exception_period = settings->exception_period.value_or(period); + const auto l_settings = settings.Read(); + auto period = l_settings->period; + const auto exception_period = l_settings->exception_period.value_or(period); if (!no_exception) period = exception_period; std::chrono::steady_clock::time_point start; - if (settings->flags & Flags::kStrong) { + if (l_settings->flags & Flags::kStrong) { start = before; } else { start = std::chrono::steady_clock::now(); } - while (changed_event_.WaitForEventUntil(start + MutatePeriod(period))) { - if (should_force_step_.exchange(false)) { + while (changed_event.WaitForEventUntil(start + MutatePeriod(period))) { + if (should_force_step.exchange(false)) { break; } // The config variable value has been changed, reload - const auto settings = settings_.Read(); - period = settings->period; - const auto exception_period = settings->exception_period.value_or(period); + const auto l_settings = settings.Read(); + period = l_settings->period; + const auto exception_period = l_settings->exception_period.value_or(period); if (!no_exception) period = exception_period; } } } -bool PeriodicTask::DoStep() { - auto settings_ptr = settings_.Read(); +bool PeriodicTask::Impl::DoStep() { + auto settings_ptr = settings.Read(); const auto span_log_level = settings_ptr->span_level; const auto name = GetName(); tracing::Span span(std::string{name}); span.SetLogLevel(span_log_level); try { - callback_(); + callback(); return true; } catch (const std::exception& e) { LOG_ERROR() << "Exception in PeriodicTask with name=" << name << ": " << e; @@ -190,10 +227,10 @@ bool PeriodicTask::DoStep() { } } -bool PeriodicTask::Step() { - const std::lock_guard lock_step(step_mutex_); +bool PeriodicTask::Impl::Step() { + const std::lock_guard lock_step(step_mutex); - if (suspend_state_.load() == SuspendState::kSuspended) { + if (suspend_state.load() == SuspendState::kSuspended) { LOG_INFO() << "Skipping suspended PeriodicTask with name=" << GetName(); return true; } @@ -201,8 +238,8 @@ bool PeriodicTask::Step() { return DoStep(); } -bool PeriodicTask::StepDebug(bool preserve_span) { - const std::lock_guard lock_step(step_mutex_); +bool PeriodicTask::Impl::StepDebug(bool preserve_span) { + const std::lock_guard lock_step(step_mutex); std::optional testsuite_oneshot_span; if (preserve_span) { @@ -213,49 +250,53 @@ bool PeriodicTask::StepDebug(bool preserve_span) { return DoStep(); } -std::chrono::milliseconds PeriodicTask::MutatePeriod(std::chrono::milliseconds period) { - auto settings_ptr = settings_.Read(); +std::chrono::milliseconds PeriodicTask::Impl::MutatePeriod(std::chrono::milliseconds period) { + auto settings_ptr = settings.Read(); if (!(settings_ptr->flags & Flags::kChaotic)) return period; - if (!mutate_period_random_) { - mutate_period_random_.emplace( + if (!mutate_period_random) { + mutate_period_random.emplace( utils::WithDefaultRandom(std::uniform_int_distribution{}) ); } const auto jitter = settings_ptr->distribution; std::uniform_int_distribution distribution{(period - jitter).count(), (period + jitter).count()}; - const auto ms = distribution(*mutate_period_random_); + const auto ms = distribution(*mutate_period_random); return std::chrono::milliseconds(ms); } -std::string_view PeriodicTask::GetName() const noexcept { - return is_name_set_ ? std::string_view{name_} : ""; +std::string_view PeriodicTask::Impl::GetName() const noexcept { + return is_name_set ? std::string_view{name} : ""; } -void PeriodicTask::SuspendDebug() { +void PeriodicTask::SuspendDebug() { impl_->SuspendDebug(); } + +void PeriodicTask::ResumeDebug() { impl_->ResumeDebug(); } + +void PeriodicTask::Impl::SuspendDebug() { // step_mutex_ waits, for a potentially long time, for Step() call completion - const std::lock_guard lock_step(step_mutex_); - auto prior_state = suspend_state_.exchange(SuspendState::kSuspended); + const std::lock_guard lock_step(step_mutex); + auto prior_state = suspend_state.exchange(SuspendState::kSuspended); if (prior_state != SuspendState::kSuspended) { LOG_DEBUG() << "Periodic task " << GetName() << " suspended"; } } -void PeriodicTask::ResumeDebug() { - auto prior_state = suspend_state_.exchange(SuspendState::kRunning); +void PeriodicTask::Impl::ResumeDebug() { + auto prior_state = suspend_state.exchange(SuspendState::kRunning); if (prior_state != SuspendState::kRunning) { LOG_DEBUG() << "Periodic task " << GetName() << " resumed"; } } void PeriodicTask::RegisterInTestsuite(testsuite::PeriodicTaskControl& periodic_task_control) { - UINVARIANT(is_name_set_, "PeriodicTask::RegisterInTestsuite should be called after Start"); - registration_holder_.emplace(periodic_task_control, std::string{GetName()}, *this); + UINVARIANT(impl_->is_name_set, "PeriodicTask::RegisterInTestsuite should be called after Start"); + impl_->registration_holder.emplace(periodic_task_control, std::string{impl_->GetName()}, *this); } PeriodicTask::Settings PeriodicTask::GetCurrentSettings() const { - auto settings_ptr = settings_.Read(); + auto settings_ptr = impl_->settings.Read(); return *settings_ptr; } diff --git a/core/src/utils/periodic_task_test.cpp b/core/src/utils/periodic_task_test.cpp index ca1d58d83067..8a4c09308103 100644 --- a/core/src/utils/periodic_task_test.cpp +++ b/core/src/utils/periodic_task_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include using namespace std::chrono_literals; diff --git a/core/src/utils/statistics/busy.cpp b/core/src/utils/statistics/busy.cpp index 6e50749a836e..8b0b0fcfec94 100644 --- a/core/src/utils/statistics/busy.cpp +++ b/core/src/utils/statistics/busy.cpp @@ -49,30 +49,30 @@ struct BusyStorage::Impl { }; BusyStorage::BusyStorage(Duration epoch_duration, Duration history_period) - : pimpl(std::make_unique(epoch_duration, history_period)) {} + : pimpl_(std::make_unique(epoch_duration, history_period)) {} BusyStorage::~BusyStorage() { UASSERT_MSG(!IsAlreadyStarted(), "BusyStorage was not stopped before destruction"); } double BusyStorage::GetCurrentLoad() const { - pimpl->recent_period.UpdateEpochIfOld(); + pimpl_->recent_period.UpdateEpochIfOld(); const Duration current_load_duration = GetNotCommittedLoad(); - const auto result = pimpl->recent_period.GetStatsForPeriod(Timer::duration::min(), true); + const auto result = pimpl_->recent_period.GetStatsForPeriod(Timer::duration::min(), true); auto duration = result.Total(); - if (duration.count() == 0) duration = pimpl->recent_period.GetMaxDuration(); + if (duration.count() == 0) duration = pimpl_->recent_period.GetMaxDuration(); auto load_duration = result.Get() + std::min(current_load_duration, duration); return static_cast(load_duration.count()) / duration.count(); } -bool BusyStorage::IsAlreadyStarted() const noexcept { return pimpl->refcount.load() != 0; } +bool BusyStorage::IsAlreadyStarted() const noexcept { return pimpl_->refcount.load() != 0; } void BusyStorage::StartWork() { - if (++pimpl->refcount != 1) { + if (++pimpl_->refcount != 1) { #ifndef NDEBUG UASSERT_MSG( - pimpl->thread_id.load() == std::this_thread::get_id(), + pimpl_->thread_id.load() == std::this_thread::get_id(), "BusyStorage should be started and stopped at the same thread" ); #endif @@ -80,19 +80,19 @@ void BusyStorage::StartWork() { } #ifndef NDEBUG - pimpl->thread_id = std::this_thread::get_id(); + pimpl_->thread_id = std::this_thread::get_id(); #endif - pimpl->start_work = utils::datetime::SteadyNow(); + pimpl_->start_work = utils::datetime::SteadyNow(); } void BusyStorage::StopWork() noexcept { - const auto refcount_snapshot = pimpl->refcount--; + const auto refcount_snapshot = pimpl_->refcount--; UASSERT_MSG(refcount_snapshot != 0, "Stop was called more times than start"); if (refcount_snapshot != 1) { #ifndef NDEBUG UASSERT_MSG( - pimpl->thread_id.load() == std::this_thread::get_id(), + pimpl_->thread_id.load() == std::this_thread::get_id(), "BusyStorage should be started and stopped at the same thread" ); #endif @@ -100,14 +100,14 @@ void BusyStorage::StopWork() noexcept { } auto not_committed_load = GetNotCommittedLoad(); - auto& value = pimpl->recent_period.GetCurrentCounter().value; + auto& value = pimpl_->recent_period.GetCurrentCounter().value; value = value.load() + not_committed_load; - pimpl->start_work = Timer::time_point{}; + pimpl_->start_work = Timer::time_point{}; } Duration BusyStorage::GetNotCommittedLoad() const noexcept { - const auto start_work = pimpl->start_work.load(); + const auto start_work = pimpl_->start_work.load(); if (start_work == Timer::time_point{}) { return Duration(0); } else { diff --git a/core/src/utils/statistics/min_max_avg_test.cpp b/core/src/utils/statistics/min_max_avg_test.cpp index 4e94aba3c5dc..c3f97c78cf21 100644 --- a/core/src/utils/statistics/min_max_avg_test.cpp +++ b/core/src/utils/statistics/min_max_avg_test.cpp @@ -18,10 +18,10 @@ constexpr auto kStressTestDuration = std::chrono::milliseconds{500}; static_assert(utils::statistics::kHasWriterSupport>); -template +template auto GetFilledMma() { utils::statistics::MinMaxAvg mma; - (mma.Account(values), ...); + (mma.Account(Values), ...); return mma; } diff --git a/core/src/utils/statistics/visitation.cpp b/core/src/utils/statistics/visitation.cpp index 034873171af8..cd8fd9e1bea0 100644 --- a/core/src/utils/statistics/visitation.cpp +++ b/core/src/utils/statistics/visitation.cpp @@ -116,16 +116,11 @@ class DfsLabelsBag { }; struct DfsNode { - DfsNode( - const formats::json::Value& next, - std::string&& path_node_, - Label&& label_, - std::string&& children_label_name_ - ) + DfsNode(const formats::json::Value& next, std::string&& path_node, Label&& label, std::string&& children_label_name) : current(next), - path_node(std::move(path_node_)), - label(std::move(label_)), - children_label_name(std::move(children_label_name_)) {} + path_node(std::move(path_node)), + label(std::move(label)), + children_label_name(std::move(children_label_name)) {} bool is_in_subtree{false}; const formats::json::Value current; diff --git a/core/src/utils/trx_tracker.cpp b/core/src/utils/trx_tracker.cpp new file mode 100644 index 000000000000..9b59f9317223 --- /dev/null +++ b/core/src/utils/trx_tracker.cpp @@ -0,0 +1,161 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace utils::trx_tracker { + +namespace { + +compiler::ThreadLocal local_task_counter = []() -> std::uint64_t { return 0; }; + +struct TransactionTracker final { + std::uint32_t trx_count = 0; + std::uint32_t disabler_count = 0; + impl::TaskId task_id; +}; + +struct TransactionTrackerStatisticsInternal final { + utils::statistics::RateCounter triggers{0}; +}; + +engine::TaskLocalVariable transaction_tracker{}; +TransactionTrackerStatisticsInternal transaction_tracker_statistics{}; + +void CheckNoTransactions(utils::function_ref get_location) { + if (!impl::IsEnabled()) { + return; + } + + auto* tracker = transaction_tracker.GetOptional(); + if (tracker && !tracker->disabler_count && tracker->trx_count > 0) { + const std::string location = get_location(); + logging::LogExtra log_extra; + log_extra.Extend("location", location); + LOG_LIMITED_WARNING() << "Long call while having active transactions" << std::move(log_extra); + ++transaction_tracker_statistics.triggers; + + TESTPOINT("long_call_in_trx", [&location] { + formats::json::ValueBuilder builder; + builder["location"] = location; + return builder.ExtractValue(); + }()); + } +} + +std::optional StartTransaction() { + if (impl::IsEnabled()) { + ++transaction_tracker->trx_count; + return transaction_tracker->task_id; + } + return std::nullopt; +} + +bool EndTransaction(const impl::TaskId& task_id) noexcept { + if (impl::IsEnabled()) { + if (transaction_tracker.GetOptional() == nullptr || transaction_tracker->task_id != task_id) { + // EndTransaction is called in different task than StartTransaction + return false; + } + --transaction_tracker->trx_count; + return true; + } + return true; +} + +bool trx_tracker_enabled = false; +bool trx_tracker_enabler_exists = false; + +} // namespace + +namespace impl { + +GlobalEnabler::GlobalEnabler(bool enable) { + UASSERT(!trx_tracker_enabler_exists); + trx_tracker_enabled = enable; + trx_tracker_enabler_exists = true; +} + +GlobalEnabler::~GlobalEnabler() { + trx_tracker_enabled = false; + trx_tracker_enabler_exists = false; +} + +bool IsEnabled() noexcept { return trx_tracker_enabled; } + +TaskId::TaskId() : created_thread_id_{std::this_thread::get_id()} { + auto counter_scope{local_task_counter.Use()}; + thread_local_counter_ = (*counter_scope)++; +} + +bool TaskId::operator==(const TaskId& other) const { + return created_thread_id_ == other.created_thread_id_ && thread_local_counter_ == other.thread_local_counter_; +} + +bool TaskId::operator!=(const TaskId& other) const { return !(*this == other); } + +} // namespace impl + +TransactionLock::~TransactionLock() { Unlock(); } + +void TransactionLock::Lock() noexcept { + if (!task_id_.has_value()) { + task_id_ = StartTransaction(); + } +} + +void TransactionLock::Unlock() noexcept { + if (task_id_.has_value() && EndTransaction(task_id_.value())) { + task_id_ = std::nullopt; + } +} + +TransactionLock::TransactionLock(TransactionLock&& other) noexcept + : task_id_(std::exchange(other.task_id_, std::nullopt)) {} + +TransactionLock& TransactionLock::operator=(TransactionLock&& other) noexcept { + if (&other == this) { + return *this; + } + + Unlock(); + task_id_ = std::exchange(other.task_id_, std::nullopt); + return *this; +} + +void CheckNoTransactions(utils::impl::SourceLocation location) { + CheckNoTransactions([&location]() { return utils::impl::ToString(location); }); +} + +void CheckNoTransactions(std::string_view location) { + CheckNoTransactions([&location]() { return std::string(location); }); +} + +TransactionTrackerStatistics GetStatistics() noexcept { + return TransactionTrackerStatistics{transaction_tracker_statistics.triggers.Load()}; +} + +void ResetStatistics() { utils::statistics::ResetMetric(transaction_tracker_statistics.triggers); } + +CheckDisabler::CheckDisabler() { ++transaction_tracker->disabler_count; } + +CheckDisabler::~CheckDisabler() { Reenable(); } + +void CheckDisabler::Reenable() noexcept { + if (!reenabled_) { + --transaction_tracker->disabler_count; + reenabled_ = true; + } +} + +} // namespace utils::trx_tracker + +USERVER_NAMESPACE_END diff --git a/core/src/utils/trx_tracker_test.cpp b/core/src/utils/trx_tracker_test.cpp new file mode 100644 index 000000000000..2b94392045e1 --- /dev/null +++ b/core/src/utils/trx_tracker_test.cpp @@ -0,0 +1,308 @@ +#include + +#include + +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +utils::statistics::Rate GetTriggers() { return utils::trx_tracker::GetStatistics().triggers; } + +using TrxTrackerFixture = utest::LogCaptureFixture<>; + +} // namespace + +UTEST_F(TrxTrackerFixture, AssertInTransaction) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + /// [Sample TransactionTracker usage] + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + + EXPECT_EQ(utils::trx_tracker::GetStatistics().triggers, 1); + EXPECT_THAT(GetLogCapture().Filter("Long call while having active transactions"), testing::SizeIs(1)); + /// [Sample TransactionTracker usage] +} + +UTEST(TrxTracker, AssertTwoInTransaction) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::CheckNoTransactions(); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + + EXPECT_EQ(GetTriggers(), 2); +} + +UTEST(TrxTracker, AssertOutOfTransaction) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + trx.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 0); +} + +UTEST(TrxTracker, UnlockOnDestruction) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + { + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::CheckNoTransactions(); + } + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, AssertNestedTransactions) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx_1{}; + trx_1.Lock(); + utils::trx_tracker::TransactionLock trx_2{}; + trx_2.Lock(); + trx_2.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + trx_1.Unlock(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, CopyConstructorLocked) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::TransactionLock trx_copy{std::move(trx)}; + utils::trx_tracker::CheckNoTransactions(); + trx_copy.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, CopyConstructorUnlocked) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + utils::trx_tracker::TransactionLock trx_copy{std::move(trx)}; + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, CopyAssignmentLockedToLocked) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx_1{}; + trx_1.Lock(); + utils::trx_tracker::TransactionLock trx_2{}; + trx_2.Lock(); + utils::trx_tracker::CheckNoTransactions(); + trx_2 = std::move(trx_1); + utils::trx_tracker::CheckNoTransactions(); + trx_2.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 2); +} + +UTEST(TrxTracker, CopyAssignmentLockedToUnlocked) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx_1{}; + trx_1.Lock(); + utils::trx_tracker::TransactionLock trx_2{}; + utils::trx_tracker::CheckNoTransactions(); + trx_2 = std::move(trx_1); + utils::trx_tracker::CheckNoTransactions(); + trx_2.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 2); +} + +UTEST(TrxTracker, CopyAssignmentUnlockedToLocked) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx_1{}; + utils::trx_tracker::TransactionLock trx_2{}; + trx_2.Lock(); + utils::trx_tracker::CheckNoTransactions(); + trx_2 = std::move(trx_1); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, CopyAssignmentUnlockedToUnlocked) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx_1{}; + utils::trx_tracker::TransactionLock trx_2{}; + trx_2 = std::move(trx_1); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 0); +} + +UTEST(TrxTracker, CopyAssignmentSelf) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + auto& trx_ref = trx; + trx = std::move(trx_ref); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, AssertWithDisabler) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + const utils::trx_tracker::CheckDisabler disabler; + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + + EXPECT_EQ(GetTriggers(), 0); +} + +UTEST(TrxTracker, AssertDisablerReenabled) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::CheckDisabler disabler; + utils::trx_tracker::CheckNoTransactions(); + disabler.Reenable(); + utils::trx_tracker::CheckNoTransactions(utils::impl::SourceLocation::Current()); + trx.Unlock(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, AssertDisablerDestroyed) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + { + const utils::trx_tracker::CheckDisabler disabler; + utils::trx_tracker::CheckNoTransactions(); + } + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, AssertMultipleDisablers) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + const utils::trx_tracker::CheckDisabler disabler; + { + const utils::trx_tracker::CheckDisabler disabler; + utils::trx_tracker::CheckNoTransactions(); + } + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + + EXPECT_EQ(GetTriggers(), 0); +} + +UTEST(TrxTracker, NoGlobalEnabler) { + utils::trx_tracker::ResetStatistics(); + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + + EXPECT_EQ(utils::trx_tracker::GetStatistics().triggers, 0); +} + +UTEST(TrxTracker, GlobalEnablerFalse) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler(false); + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + + EXPECT_EQ(utils::trx_tracker::GetStatistics().triggers, 0); +} + +UTEST(TrxTracker, UnlockInAsync) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + auto task = utils::Async("", [&trx]() { trx.Unlock(); }); + task.Get(); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 1); +} + +UTEST(TrxTracker, LockInAsync) { + utils::trx_tracker::ResetStatistics(); + const utils::trx_tracker::impl::GlobalEnabler enabler; + + auto task = utils::Async("", []() { + utils::trx_tracker::TransactionLock trx{}; + trx.Lock(); + return trx; + }); + auto trx = task.Get(); + utils::trx_tracker::CheckNoTransactions(); + trx.Unlock(); + utils::trx_tracker::CheckNoTransactions(); + + EXPECT_EQ(GetTriggers(), 0); +} + +USERVER_NAMESPACE_END diff --git a/core/utest/src/utest/test_case_macros.cpp b/core/utest/src/utest/test_case_macros.cpp index df98a19e47a2..ef2b9ff5cfe6 100644 --- a/core/utest/src/utest/test_case_macros.cpp +++ b/core/utest/src/utest/test_case_macros.cpp @@ -89,14 +89,6 @@ void DoRunTest( }); } -bool IsStackUsageMonitorEnabled() { - // NOLINTNEXTLINE(concurrency-mt-unsafe) - auto* enable = std::getenv("USERVER_GTEST_ENABLE_STACK_USAGE_MONITOR"); - if (!enable) return true; - if (std::string_view(enable) == "0") return false; - return true; -} - } // namespace void DoRunTest( @@ -115,7 +107,6 @@ void DoRunTest( // work with gtest's `waitpid()` calls. config.ev_default_loop_disabled = true; } - config.is_stack_usage_monitor_enabled = IsStackUsageMonitorEnabled(); return DoRunTest(thread_count, config, std::move(factory)); } diff --git a/grpc/CMakeLists.txt b/grpc/CMakeLists.txt index 4c8fe7b6f342..ee9d170823c5 100644 --- a/grpc/CMakeLists.txt +++ b/grpc/CMakeLists.txt @@ -197,3 +197,5 @@ _userver_directory_install( FILES "${USERVER_ROOT_DIR}/cmake/modules/FindUserverGrpc.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver/modules ) + +_userver_install_component(MODULE grpc DEPENDS core) diff --git a/grpc/benchmarks/format_log_message.cpp b/grpc/benchmarks/format_log_message.cpp index 5b3cfb1e4a5b..03d7b1447d9e 100644 --- a/grpc/benchmarks/format_log_message.cpp +++ b/grpc/benchmarks/format_log_message.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -20,25 +21,28 @@ void FormatLogMessage(benchmark::State& state) { // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) for (auto _ : state) { auto result = - ugrpc::server::impl::FormatLogMessage(metadata, peer, start_time, call_name, grpc::StatusCode::OK); + ugrpc::server::impl::FormatLogMessage(metadata, peer, start_time, call_name, grpc::StatusCode::OK, nullptr); benchmark::DoNotOptimize(result); } - auto result = ugrpc::server::impl::FormatLogMessage(metadata, peer, start_time, call_name, grpc::StatusCode::OK); + auto result = + ugrpc::server::impl::FormatLogMessage(metadata, peer, start_time, call_name, grpc::StatusCode::OK, nullptr); - UINVARIANT(utils::text::StartsWith(result, "tskv\ttimestamp=1971-05-"), "Fail"); - UINVARIANT(result.find("\ttimezone=") != std::string::npos, "Fail 1"); + const std::string_view log_line_view{result.ExtractTextLogItem().log_line}; // Convert SmallString to string_view + + UINVARIANT(utils::text::StartsWith(log_line_view, "tskv\ttimestamp=1971-05-"), "Fail"); + UINVARIANT(log_line_view.find("\ttimezone=") != std::string::npos, "Fail 1"); UINVARIANT( - result.find("\tuser_agent=grpc-go/1.45.0\t" - "ip=2a02:aaaa:aaaa:aaaa::1:1f\t" - "x_real_ip=2a02:aaaa:aaaa:aaaa::1:1f\t" - "request=hello.HelloService/SayHello\t" - "request_time=") != std::string::npos, + log_line_view.find("\tuser_agent=grpc-go/1.45.0\t" + "ip=2a02:aaaa:aaaa:aaaa::1:1f\t" + "x_real_ip=2a02:aaaa:aaaa:aaaa::1:1f\t" + "request=hello.HelloService/SayHello\t" + "request_time=") != std::string::npos, "Fail 2" ); - UINVARIANT(result.find("\tgrpc_status=0\t") != std::string::npos, "Fail 3"); - UINVARIANT(result.find("\tgrpc_status_code=OK\n") != std::string::npos, "Fail 4"); + UINVARIANT(log_line_view.find("\tgrpc_status=0\t") != std::string::npos, "Fail 3"); + UINVARIANT(log_line_view.find("\tgrpc_status_code=OK\n") != std::string::npos, "Fail 4"); } BENCHMARK(FormatLogMessage); diff --git a/grpc/benchmarks/logging.cpp b/grpc/benchmarks/logging.cpp index b5ebe6b97227..3e13cb8397bd 100644 --- a/grpc/benchmarks/logging.cpp +++ b/grpc/benchmarks/logging.cpp @@ -1,9 +1,8 @@ #include +#include #include -#include - #include USERVER_NAMESPACE_BEGIN @@ -53,7 +52,7 @@ void BenchCustomLimit(benchmark::State& state) { // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) for (auto _ : state) { { - auto log = impl::ToLimitedDebugString(kMessage, 1024); + auto log = ugrpc::ToLimitedDebugString(kMessage, 1024); benchmark::DoNotOptimize(log); } } diff --git a/grpc/dynamic_configs/EGRESS_GRPC_PROXY_ENABLED.yaml b/grpc/dynamic_configs/EGRESS_GRPC_PROXY_ENABLED.yaml new file mode 100644 index 000000000000..35ff566e5c41 --- /dev/null +++ b/grpc/dynamic_configs/EGRESS_GRPC_PROXY_ENABLED.yaml @@ -0,0 +1,4 @@ +default: false +description: '' +schema: + type: boolean diff --git a/grpc/functional_tests/basic_chaos/client.hpp b/grpc/functional_tests/basic_chaos/client.hpp index 41574e798b58..0a9f9e17d40b 100644 --- a/grpc/functional_tests/basic_chaos/client.hpp +++ b/grpc/functional_tests/basic_chaos/client.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -31,7 +32,7 @@ class GreeterClient final : public components::ComponentBase { // Tests dedicated-channel-count from SimpleClientComponent auto& client = context.FindComponent>("greeter-client-component").GetClient(); - const auto& data = ugrpc::client::impl::GetClientData(client); + const auto& data = ugrpc::client::impl::ClientDataAccessor::GetClientData(client); const auto stub_state = data.GetStubState(); UASSERT(stub_state->dedicated_stubs[0].Size() == 3); UASSERT(stub_state->dedicated_stubs[1].Size() == 0); diff --git a/grpc/functional_tests/basic_chaos/tests-grpcserver/requests_server.py b/grpc/functional_tests/basic_chaos/tests-grpcserver/requests_server.py index 8deb47c67435..adbca613484b 100644 --- a/grpc/functional_tests/basic_chaos/tests-grpcserver/requests_server.py +++ b/grpc/functional_tests/basic_chaos/tests-grpcserver/requests_server.py @@ -2,7 +2,6 @@ import logging import grpc -import pytest import samples.greeter_pb2 as greeter_pb2 # noqa: E402, E501 @@ -26,35 +25,6 @@ async def _say_hello(grpc_client, gate): assert gate.connections_count() > 0 -@pytest.mark.parametrize( - 'metadata,logs,', - [ - [(('x-yatraceid', 'traceid'),), {'trace_id': 'traceid'}], - [ - (('x-yatraceid', 'traceid'), ('x-yaspanid', 'span')), - {'trace_id': 'traceid', 'parent_id': 'span'}, - ], - ], -) -async def test_tracing_metadata( - grpc_client, - service_client, - gate, - metadata, - logs, -): - request = greeter_pb2.GreetingRequest(name='Python') - async with service_client.capture_logs() as capture: - response = await grpc_client.SayHello( - request, - wait_for_ready=True, - metadata=metadata, - ) - assert response.greeting == 'Hello, Python!' - - assert capture.select(**logs) - - async def _say_hello_response_stream(grpc_client, gate): request = greeter_pb2.GreetingRequest(name='Python') reference = '!' diff --git a/grpc/functional_tests/basic_chaos/tests-grpcserver/test_tracing_metadata.py b/grpc/functional_tests/basic_chaos/tests-grpcserver/test_tracing_metadata.py new file mode 100644 index 000000000000..20a0f3fb2e4d --- /dev/null +++ b/grpc/functional_tests/basic_chaos/tests-grpcserver/test_tracing_metadata.py @@ -0,0 +1,32 @@ +import pytest + +import samples.greeter_pb2 as greeter_pb2 # noqa: E402, E501 + + +@pytest.mark.parametrize( + 'metadata,logs,', + [ + [(('x-yatraceid', 'traceid'),), {'trace_id': 'traceid'}], + [ + (('x-yatraceid', 'traceid'), ('x-yaspanid', 'span')), + {'trace_id': 'traceid', 'parent_id': 'span'}, + ], + ], +) +async def test_tracing_metadata( + grpc_client, + service_client, + gate, + metadata, + logs, +): + request = greeter_pb2.GreetingRequest(name='Python') + async with service_client.capture_logs() as capture: + response = await grpc_client.SayHello( + request, + wait_for_ready=True, + metadata=metadata, + ) + assert response.greeting == 'Hello, Python!' + + assert capture.select(**logs) diff --git a/grpc/functional_tests/basic_server/CMakeLists.txt b/grpc/functional_tests/basic_server/CMakeLists.txt index 9c50d101bf5b..e35b2f8642b8 100644 --- a/grpc/functional_tests/basic_server/CMakeLists.txt +++ b/grpc/functional_tests/basic_server/CMakeLists.txt @@ -5,7 +5,8 @@ target_link_libraries(${PROJECT_NAME} userver::core) include(UserverGrpcTargets) userver_add_grpc_library(${PROJECT_NAME}-proto PROTOS samples/greeter.proto) -target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}-proto) +add_subdirectory(unused_proto) # Test for https://github.com/userver-framework/userver/pull/1014 +target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}-proto ${PROJECT_NAME}-unused-proto) userver_chaos_testsuite_add(TESTS_DIRECTORY tests-tls) userver_chaos_testsuite_add(TESTS_DIRECTORY tests-unix-socket) diff --git a/grpc/functional_tests/basic_server/unused_proto/CMakeLists.txt b/grpc/functional_tests/basic_server/unused_proto/CMakeLists.txt new file mode 100644 index 000000000000..08c23af9686d --- /dev/null +++ b/grpc/functional_tests/basic_server/unused_proto/CMakeLists.txt @@ -0,0 +1,2 @@ +include(UserverGrpcTargets) +userver_add_grpc_library(${PROJECT_NAME}-unused-proto PROTOS samples/unused.proto) diff --git a/grpc/functional_tests/basic_server/unused_proto/proto/samples/unused.proto b/grpc/functional_tests/basic_server/unused_proto/proto/samples/unused.proto new file mode 100644 index 000000000000..32e5c311976d --- /dev/null +++ b/grpc/functional_tests/basic_server/unused_proto/proto/samples/unused.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package unused.api; + +service GreeterService { + rpc SayHello(GreetingRequest) returns(GreetingResponse) {} + rpc CallEchoNobody(GreetingRequest) returns(GreetingResponse) {} +} + +message GreetingRequest { + string name = 1; +} + +message GreetingResponse { + string greeting = 1; +} diff --git a/grpc/functional_tests/basic_server/unused_proto/proto/ya.make b/grpc/functional_tests/basic_server/unused_proto/proto/ya.make new file mode 100644 index 000000000000..338c9f9ddbb7 --- /dev/null +++ b/grpc/functional_tests/basic_server/unused_proto/proto/ya.make @@ -0,0 +1,35 @@ +PROTO_LIBRARY(userver-functional-test-service-grpc-unused-proto) + +SUBSCRIBER(g:taxi-common) + +EXCLUDE_TAGS(GO_PROTO JAVA_PROTO) + +SRCDIR(taxi/uservices/userver/grpc/functional_tests/basic_server/unused_proto/proto) + +SRCS( + samples/unused.proto +) + +PROTO_NAMESPACE(taxi/uservices/userver/grpc/functional_tests/basic_server/unused_proto/proto) +PY_NAMESPACE(.) + +GRPC() + +CPP_PROTO_PLUGIN2( + usrv_service + taxi/uservices/userver-arc-utils/scripts/grpc/arc-generate-service + _service.usrv.pb.hpp + _service.usrv.pb.cpp + DEPS taxi/uservices/userver/grpc +) +CPP_PROTO_PLUGIN2( + usrv_client + taxi/uservices/userver-arc-utils/scripts/grpc/arc-generate-client + _client.usrv.pb.hpp + _client.usrv.pb.cpp + DEPS taxi/uservices/userver/grpc +) + +END() + + diff --git a/grpc/functional_tests/lowlevel/static_config.yaml b/grpc/functional_tests/lowlevel/static_config.yaml index 11d157282917..4c7dba72a631 100644 --- a/grpc/functional_tests/lowlevel/static_config.yaml +++ b/grpc/functional_tests/lowlevel/static_config.yaml @@ -15,6 +15,8 @@ components_manager: # The TaskProcessor for blocking connection initiation blocking-task-processor: grpc-blocking-task-processor + # /// [retry configuration] + # yaml # Creates gRPC clients grpc-client-factory: disable-all-pipeline-middlewares: true @@ -27,6 +29,7 @@ components_manager: # https://grpc.github.io/grpc/core/group__grpc__arg__keys.html channel-args: {} + # Transparent retries configuration default-service-config: > { "methodConfig": [{ @@ -46,6 +49,7 @@ components_manager: } }] } + # /// [retry configuration] grpc-client-middlewares-pipeline: {} diff --git a/grpc/functional_tests/middleware_client/static_config.yaml b/grpc/functional_tests/middleware_client/static_config.yaml index 1149d5238243..b6a34e164686 100644 --- a/grpc/functional_tests/middleware_client/static_config.yaml +++ b/grpc/functional_tests/middleware_client/static_config.yaml @@ -26,11 +26,16 @@ components_manager: grpc-client-headers-propagator: skip-headers: skipped-header: true + # /// [grpc client factory] + # yaml grpc-client-factory: middlewares: grpc-client-deadline-propagation: enabled: false + retry-config: + attempts: 2 channel-args: {} + # /// [grpc client factory] greeter-client: endpoint: $greeter-client-endpoint diff --git a/grpc/functional_tests/middleware_client/tests/test_error_status.py b/grpc/functional_tests/middleware_client/tests/test_error_status.py index 73d28dd13008..fe13ebf474f9 100644 --- a/grpc/functional_tests/middleware_client/tests/test_error_status.py +++ b/grpc/functional_tests/middleware_client/tests/test_error_status.py @@ -16,13 +16,17 @@ async def mock_say_hello(request, context: grpc.aio.ServicerContext): with pytest.raises(pytest_userver.client.TestsuiteTaskFailed) as ex_info: await service_client.run_task('call-say-hello') ex_info.match("'samples.api.GreeterService/SayHello' failed: code=UNAVAILABLE, message='Greeter is down'") - assert mock_say_hello.times_called == 1 + assert mock_say_hello.times_called == 2 # /// [Mocked error status] async def test_error_status_via_result(service_client, grpc_mockserver): @grpc_mockserver(greeter_services.GreeterServiceServicer.SayHello) async def mock_say_hello(request, context: grpc.aio.ServicerContext): + # Be aware of using `context.set_code()`! + # Maybe you should use `await context.abort()` instead + # e.g. using `context.set_code()` as an unexpected side effect disables grpc-core client retries + # but does not disable userver grpc retries context.set_code(grpc.StatusCode.UNAVAILABLE) context.set_details('Greeter is down') return greeter_protos.GreetingResponse() # Message is ignored. @@ -30,4 +34,5 @@ async def mock_say_hello(request, context: grpc.aio.ServicerContext): with pytest.raises(pytest_userver.client.TestsuiteTaskFailed) as ex_info: await service_client.run_task('call-say-hello') ex_info.match("'samples.api.GreeterService/SayHello' failed: code=UNAVAILABLE, message='Greeter is down'") - assert mock_say_hello.times_called == 1 + # using `context.set_code()` does not disable userver grpc retries + assert mock_say_hello.times_called == 2 diff --git a/grpc/functional_tests/middleware_client/tests/test_mocked_fail.py b/grpc/functional_tests/middleware_client/tests/test_mocked_fail.py index 4197f674479a..6105622ee37a 100644 --- a/grpc/functional_tests/middleware_client/tests/test_mocked_fail.py +++ b/grpc/functional_tests/middleware_client/tests/test_mocked_fail.py @@ -14,6 +14,7 @@ async def mock_say_hello(request, context): with pytest.raises(pytest_userver.client.TestsuiteTaskFailed) as ex_info: await service_client.run_task('call-say-hello') ex_info.match("'samples.api.GreeterService/SayHello' failed: interrupted at Testsuite network") + assert mock_say_hello.times_called == 1 # /// [Mocked network failure] @@ -26,4 +27,5 @@ async def mock_say_hello(request, context): with pytest.raises(pytest_userver.client.TestsuiteTaskFailed) as ex_info: await service_client.run_task('call-say-hello') ex_info.match("'samples.api.GreeterService/SayHello' failed: interrupted at Testsuite timeout") + assert mock_say_hello.times_called == 1 # /// [Mocked timeout failure] diff --git a/grpc/handlers/CMakeLists.txt b/grpc/handlers/CMakeLists.txt index bbd226db2744..696c854e3731 100644 --- a/grpc/handlers/CMakeLists.txt +++ b/grpc/handlers/CMakeLists.txt @@ -6,6 +6,7 @@ userver_module( INSTALL_COMPONENT grpc LINK_LIBRARIES userver-grpc-internal UTEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*_test.cpp" + DEPENDS grpc ) userver_add_grpc_library(${PROJECT_NAME}-proto PROTOS health/v1/health.proto) diff --git a/grpc/include/userver/ugrpc/client/call_context.hpp b/grpc/include/userver/ugrpc/client/call_context.hpp index 78be335fa1c7..4304e3936dd0 100644 --- a/grpc/include/userver/ugrpc/client/call_context.hpp +++ b/grpc/include/userver/ugrpc/client/call_context.hpp @@ -19,10 +19,13 @@ class CallState; /// @brief gRPC call context class CallContext { public: - CallContext(utils::impl::InternalTag, impl::CallState& state); + /// @cond + // For internal use only + CallContext(utils::impl::InternalTag, impl::CallState& state) noexcept; + /// @endcond /// @returns the `ClientContext` used for this RPC - grpc::ClientContext& GetClientContext() noexcept; + grpc::ClientContext& GetClientContext(); /// @returns client name std::string_view GetClientName() const noexcept; diff --git a/grpc/include/userver/ugrpc/client/call_options.hpp b/grpc/include/userver/ugrpc/client/call_options.hpp index ceaeae75a55b..e50de2700362 100644 --- a/grpc/include/userver/ugrpc/client/call_options.hpp +++ b/grpc/include/userver/ugrpc/client/call_options.hpp @@ -24,6 +24,12 @@ class CallOptionsAccessor; /// @brief Options passed to interface calls class CallOptions { public: + /// @{ + /// Set and get retry attempts. Maximum number of retry attempts, including the original attempt. + void SetAttempts(int attempts); + int GetAttempts() const; + /// @} + /// @{ /// Set and get operation timeout. /// @@ -47,6 +53,8 @@ class CallOptions { private: friend class impl::CallOptionsAccessor; + int attempts_{0}; + std::chrono::milliseconds timeout_{std::chrono::milliseconds::max()}; std::vector> metadata_; diff --git a/grpc/include/userver/ugrpc/client/channels.hpp b/grpc/include/userver/ugrpc/client/channels.hpp index 10ee09edc102..a800826427cc 100644 --- a/grpc/include/userver/ugrpc/client/channels.hpp +++ b/grpc/include/userver/ugrpc/client/channels.hpp @@ -9,7 +9,7 @@ #include #include -#include +#include USERVER_NAMESPACE_BEGIN @@ -17,8 +17,11 @@ namespace ugrpc::client { namespace impl { -[[nodiscard]] bool -TryWaitForConnected(ClientData& client_data, engine::Deadline deadline, engine::TaskProcessor& blocking_task_processor); +[[nodiscard]] bool TryWaitForConnected( + const ClientData& client_data, + engine::Deadline deadline, + engine::TaskProcessor& blocking_task_processor +); } // namespace impl @@ -48,7 +51,9 @@ std::shared_ptr MakeChannel( template [[nodiscard]] bool TryWaitForConnected(Client& client, engine::Deadline deadline, engine::TaskProcessor& blocking_task_processor) { - return impl::TryWaitForConnected(impl::GetClientData(client), deadline, blocking_task_processor); + return impl::TryWaitForConnected( + impl::ClientDataAccessor::GetClientData(client), deadline, blocking_task_processor + ); } } // namespace ugrpc::client diff --git a/grpc/include/userver/ugrpc/client/client_factory_component.hpp b/grpc/include/userver/ugrpc/client/client_factory_component.hpp index 3329c19bbeaa..15e210c22909 100644 --- a/grpc/include/userver/ugrpc/client/client_factory_component.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory_component.hpp @@ -53,6 +53,7 @@ using MiddlewareRunnerComponentBase = USERVER_NAMESPACE::middlewares::RunnerComp /// ---- | ----------- | ------------- /// auth-type | authentication method, see @ref grpc_ssl_authentication "Authentication" | - /// ssl-credentials-options | TLS/SSL options, see @ref grpc_ssl_authentication "Authentication" | - +/// retry-config | retry configuration for outgoing RPCs | {} /// channel-args | a map of channel arguments, see gRPC Core docs | {} /// default-service-config | default service config, see above | - /// channel-count | Number of underlying grpc::Channel objects | 1 diff --git a/grpc/include/userver/ugrpc/client/client_factory_settings.hpp b/grpc/include/userver/ugrpc/client/client_factory_settings.hpp index beee7465360c..b10ad12c5a13 100644 --- a/grpc/include/userver/ugrpc/client/client_factory_settings.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory_settings.hpp @@ -11,6 +11,8 @@ #include #include +#include + USERVER_NAMESPACE_BEGIN namespace ugrpc::client { @@ -22,7 +24,10 @@ struct ClientFactorySettings final { /// gRPC channel credentials by client_name. If not set, default `credentials` /// is used instead. - std::unordered_map> client_credentials{}; + std::unordered_map> client_credentials; + + /// Retry configuration for outgoing RPCs + RetryConfig retry_config; /// Optional grpc-core channel args /// @see https://grpc.github.io/grpc/core/group__grpc__arg__keys.html diff --git a/grpc/include/userver/ugrpc/client/generic_client.hpp b/grpc/include/userver/ugrpc/client/generic_client.hpp index 7db93ed8dfe2..792a1203278f 100644 --- a/grpc/include/userver/ugrpc/client/generic_client.hpp +++ b/grpc/include/userver/ugrpc/client/generic_client.hpp @@ -8,9 +8,11 @@ #include +#include + #include #include -#include +#include #include #include @@ -47,8 +49,10 @@ namespace ugrpc::client { /// For a more complete sample, see @ref grpc_generic_api. class GenericClient final { public: - GenericClient(GenericClient&&) noexcept = default; - GenericClient& operator=(GenericClient&&) noexcept = delete; + GenericClient(GenericClient&&) noexcept; + GenericClient& operator=(GenericClient&&) noexcept; + + ~GenericClient(); /// Initiate a `single request -> single response` RPC with the given name. ResponseFuture AsyncUnaryCall( @@ -74,10 +78,9 @@ class GenericClient final { /// @endcond private: - template - friend impl::ClientData& impl::GetClientData(Client& client); + friend class impl::ClientDataAccessor; - impl::ClientData impl_; + utils::Box client_data_; }; } // namespace ugrpc::client diff --git a/grpc/include/userver/ugrpc/client/impl/async_methods.hpp b/grpc/include/userver/ugrpc/client/impl/async_methods.hpp index a9d75d563889..9aaf43569ea4 100644 --- a/grpc/include/userver/ugrpc/client/impl/async_methods.hpp +++ b/grpc/include/userver/ugrpc/client/impl/async_methods.hpp @@ -1,36 +1,13 @@ #pragma once -#include #include #include -#include -#include - -#include -#include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -/// @{ -/// @brief Helper type aliases for low-level asynchronous gRPC streams -/// @see -/// @see -template -using RawResponseReader = std::unique_ptr>; - -template -using RawReader = std::unique_ptr>; - -template -using RawWriter = std::unique_ptr>; - -template -using RawReaderWriter = std::unique_ptr>; -/// @} - template const google::protobuf::Message* ToBaseMessage(const Message* message) { if constexpr (std::is_base_of_v) { @@ -40,25 +17,6 @@ const google::protobuf::Message* ToBaseMessage(const Message* message) { } } -ugrpc::impl::AsyncMethodInvocation::WaitStatus WaitAndTryCancelIfNeeded( - ugrpc::impl::AsyncMethodInvocation& invocation, - engine::Deadline deadline, - grpc::ClientContext& context -) noexcept; - -ugrpc::impl::AsyncMethodInvocation::WaitStatus -WaitAndTryCancelIfNeeded(ugrpc::impl::AsyncMethodInvocation& invocation, grpc::ClientContext& context) noexcept; - -void ProcessFinish(CallState& state, const google::protobuf::Message* final_response); - -void ProcessFinishAbandoned(CallState& state) noexcept; - -void CheckFinishStatus(CallState& state); - -void ProcessCancelled(CallState& state, std::string_view stage) noexcept; - -void ProcessNetworkError(CallState& state, std::string_view stage) noexcept; - } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/async_stream_methods.hpp b/grpc/include/userver/ugrpc/client/impl/async_stream_methods.hpp index 20f18d6f43bd..e342e327b4bb 100644 --- a/grpc/include/userver/ugrpc/client/impl/async_stream_methods.hpp +++ b/grpc/include/userver/ugrpc/client/impl/async_stream_methods.hpp @@ -1,19 +1,48 @@ #pragma once -#include +#include + +#include #include #include #include +#include #include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { +/// @{ +/// @brief Helper type aliases for low-level asynchronous gRPC streams +/// @see +template +using RawReader = std::unique_ptr>; + +template +using RawWriter = std::unique_ptr>; + +template +using RawReaderWriter = std::unique_ptr>; +/// @} + +ugrpc::impl::AsyncMethodInvocation::WaitStatus +WaitAndTryCancelIfNeeded(ugrpc::impl::AsyncMethodInvocation& invocation, grpc::ClientContext& context) noexcept; + void CheckOk(StreamingCallState& state, ugrpc::impl::AsyncMethodInvocation::WaitStatus status, std::string_view stage); +void CheckFinishStatus(CallState& state); + +void ProcessFinish(CallState& state, const google::protobuf::Message* final_response); + +void ProcessFinishAbandoned(CallState& state) noexcept; + +void ProcessCancelled(CallState& state, std::string_view stage) noexcept; + +void ProcessNetworkError(CallState& state, std::string_view stage) noexcept; + template void StartCall(GrpcStream& stream, StreamingCallState& state) { ugrpc::impl::AsyncMethodInvocation start_call; diff --git a/grpc/include/userver/ugrpc/client/impl/async_unary_call_adapter.hpp b/grpc/include/userver/ugrpc/client/impl/async_unary_call_adapter.hpp new file mode 100644 index 000000000000..ecc3f28c17ee --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/async_unary_call_adapter.hpp @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +template +class AsyncUnaryCallAdapter final : public ResponseFutureImplBase { +public: + AsyncUnaryCallAdapter( + CallParams&& params, + PrepareUnaryCallProxy&& prepare_unary_call, + const Request& request + ) + : request_{request}, + unary_call_{std::move(params), std::move(prepare_unary_call), request_}, + perform_task_{utils::CriticalAsync( + "async-unary-call-perform", + [this] { + // TODO: get rid of span for perform_task + // we use `utils::CriticalAsync` to inherit TaskInheritedVariable, but it create span, + // so set LogLevel=None + tracing::Span::CurrentSpan().SetLogLevel(logging::Level::kNone); + + unary_call_.Perform(); + return unary_call_.ExtractResponse(); + } + )}, + cancellation_token_{perform_task_} {} + + ~AsyncUnaryCallAdapter() override { unary_call_.Abandon(); } + + AsyncUnaryCallAdapter(AsyncUnaryCallAdapter&&) = delete; + AsyncUnaryCallAdapter& operator=(AsyncUnaryCallAdapter&&) = delete; + + CallContext& GetContext() noexcept override { return unary_call_.GetContext(); } + const CallContext& GetContext() const noexcept override { return unary_call_.GetContext(); } + + bool IsReady() const override { return perform_task_.IsFinished(); } + + engine::FutureStatus WaitUntil(engine::Deadline deadline) const noexcept override { + utils::trx_tracker::CheckNoTransactions(unary_call_.GetContext().GetCallName()); + return perform_task_.WaitNothrowUntil(deadline); + } + + Response Get() override { + const auto status = WaitUntil(engine::Deadline{}); + if (status != engine::FutureStatus::kReady) { + perform_task_.RequestCancel(); + } + const engine::TaskCancellationBlocker cancel_blocker; + return perform_task_.Get(); + } + + void Cancel() override { cancellation_token_.RequestCancel(); } + + engine::impl::ContextAccessor* TryGetContextAccessor() noexcept override { + return perform_task_.TryGetContextAccessor(); + } + +private: + Request request_; + UnaryCall unary_call_; + engine::TaskWithResult perform_task_; + engine::TaskCancellationToken cancellation_token_; +}; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/call_params.hpp b/grpc/include/userver/ugrpc/client/impl/call_params.hpp index f4755aa19586..69cb82972a83 100644 --- a/grpc/include/userver/ugrpc/client/impl/call_params.hpp +++ b/grpc/include/userver/ugrpc/client/impl/call_params.hpp @@ -29,6 +29,7 @@ struct CallParams { StubHandle stub; const Middlewares& middlewares; ugrpc::impl::MethodStatistics& statistics; + const testsuite::GrpcControl& testsuite_grpc; }; CallParams CreateCallParams(const ClientData& client_data, std::size_t method_id, CallOptions&& call_options); diff --git a/grpc/include/userver/ugrpc/client/impl/call_state.hpp b/grpc/include/userver/ugrpc/client/impl/call_state.hpp index 190a2d3dc274..4b3367f0a820 100644 --- a/grpc/include/userver/ugrpc/client/impl/call_state.hpp +++ b/grpc/include/userver/ugrpc/client/impl/call_state.hpp @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -21,6 +20,10 @@ USERVER_NAMESPACE_BEGIN +namespace ugrpc::client { +class CallOptions; +} // namespace ugrpc::client + namespace ugrpc::client::impl { struct CallParams; @@ -42,6 +45,8 @@ class CallState { StubHandle& GetStub() noexcept; + void SetClientContext(std::unique_ptr client_context) noexcept; + const grpc::ClientContext& GetClientContext() const noexcept; grpc::ClientContext& GetClientContext() noexcept; @@ -58,6 +63,8 @@ class CallState { const MiddlewarePipeline& GetMiddlewarePipeline() const noexcept; + const testsuite::GrpcControl& GetTestsuiteControl() const noexcept; + CallKind GetCallKind() const noexcept; void ResetSpan() noexcept; @@ -70,6 +77,10 @@ class CallState { grpc::Status& GetStatus() noexcept; + void Commit() noexcept; + + grpc::ClientContext& GetClientContextCommitted(); + private: StubHandle stub_; @@ -90,45 +101,18 @@ class CallState { MiddlewarePipeline middleware_pipeline_; + const testsuite::GrpcControl& testsuite_grpc_; + CallKind call_kind_{}; grpc::Status status_; -}; - -class UnaryCallState final : public CallState { -public: - explicit UnaryCallState(CallParams&& params) : CallState(std::move(params), CallKind::kUnaryCall) {} - - ~UnaryCallState() noexcept; - - UnaryCallState(UnaryCallState&&) noexcept = delete; - UnaryCallState& operator=(UnaryCallState&&) noexcept = delete; - bool IsFinishProcessed() const noexcept; - void SetFinishProcessed() noexcept; - - bool IsStatusExtracted() const noexcept; - void SetStatusExtracted() noexcept; - - void EmplaceFinishAsyncMethodInvocation(); - - FinishAsyncMethodInvocation& GetFinishAsyncMethodInvocation() noexcept; - -private: - bool finish_processed_{false}; - - bool status_extracted_{false}; - - // In unary call the call is finished as soon as grpc core - // gives us back a response - so for unary call - // We use FinishAsyncMethodInvocation that will correctly close all our - // tracing::Span objects and account everything in statistics. - std::optional invocation_; + std::atomic committed_{false}; }; class StreamingCallState final : public CallState { public: - StreamingCallState(CallParams&& params, CallKind call_kind) : CallState(std::move(params), call_kind) {} + StreamingCallState(CallParams&& params, CallKind call_kind); ~StreamingCallState() noexcept; @@ -192,6 +176,8 @@ bool IsWriteAvailable(const StreamingCallState&) noexcept; bool IsWriteAndCheckAvailable(const StreamingCallState&) noexcept; +void SetupClientContext(CallState& state, const CallOptions& call_options); + void HandleCallStatistics(CallState& state, const grpc::Status& status) noexcept; void RunMiddlewarePipeline(CallState& state, const MiddlewareHooks& hooks); diff --git a/grpc/include/userver/ugrpc/client/impl/channel_argument_utils.hpp b/grpc/include/userver/ugrpc/client/impl/channel_argument_utils.hpp new file mode 100644 index 000000000000..f6b7af72dd28 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/channel_argument_utils.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +grpc::ChannelArguments +BuildChannelArguments(const grpc::ChannelArguments& channel_args, const std::optional& service_config); + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/client_data.hpp b/grpc/include/userver/ugrpc/client/impl/client_data.hpp index e86ffce7a608..04ca8c490256 100644 --- a/grpc/include/userver/ugrpc/client/impl/client_data.hpp +++ b/grpc/include/userver/ugrpc/client/impl/client_data.hpp @@ -13,8 +13,9 @@ #include #include -#include +#include #include +#include #include #include #include @@ -52,9 +53,9 @@ class ClientData final { std::in_place, internals_.channel_args, internals_.default_service_config, + internals_.retry_config, metadata - ), - stub_state_(std::make_unique>()) { + ) { if (internals_.qos) { SubscribeOnConfigUpdate(*internals_.qos); } else { @@ -64,20 +65,19 @@ class ClientData final { template ClientData(ClientInternals&& internals, GenericClientTag, std::in_place_type_t) - : internals_(std::move(internals)), stub_state_(std::make_unique>()) { + : internals_(std::move(internals)) { ConstructStubState(); } ~ClientData(); - ClientData(ClientData&&) noexcept = default; - ClientData& operator=(ClientData&&) = delete; - + ClientData(ClientData&&) noexcept = delete; ClientData(const ClientData&) = delete; + ClientData& operator=(ClientData&&) = delete; ClientData& operator=(const ClientData&) = delete; StubHandle NextStubFromMethodId(std::size_t method_id) const { - auto stub_state = stub_state_->Read(); + auto stub_state = stub_state_.Read(); auto& dedicated_stubs = stub_state->dedicated_stubs[method_id]; auto& stubs = dedicated_stubs.Size() ? dedicated_stubs : stub_state->stubs; auto& stub = stubs.NextStub(); @@ -85,7 +85,7 @@ class ClientData final { } StubHandle NextStub() const { - auto stub_state = stub_state_->Read(); + auto stub_state = stub_state_.Read(); auto& stub = stub_state->stubs.NextStub(); return StubHandle{std::move(stub_state), stub}; } @@ -108,7 +108,9 @@ class ClientData final { const dynamic_config::Key* GetClientQos() const; - rcu::ReadablePtr GetStubState() const { return stub_state_->Read(); } + const RetryConfig& GetRetryConfig() const { return internals_.retry_config; } + + rcu::ReadablePtr GetStubState() const { return stub_state_.Read(); } private: template @@ -159,26 +161,21 @@ class ClientData final { ) : utils::FixedArray{}; - stub_state_->Assign({client_qos, std::move(stubs), std::move(dedicated_stubs)}); + stub_state_.Assign({client_qos, std::move(stubs), std::move(dedicated_stubs)}); } ClientInternals internals_; - std::optional metadata_{std::nullopt}; + std::optional metadata_; ugrpc::impl::ServiceStatistics* service_statistics_{nullptr}; - std::optional channel_arguments_builder_; + std::optional channel_arguments_builder_; - std::unique_ptr> stub_state_; + rcu::Variable stub_state_; // These fields must be the last ones concurrent::AsyncEventSubscriberScope config_subscription_; }; -template -ClientData& GetClientData(Client& client) { - return client.impl_; -} - } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/client_data_accessor.hpp b/grpc/include/userver/ugrpc/client/impl/client_data_accessor.hpp new file mode 100644 index 000000000000..6cbc2f68a991 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/client_data_accessor.hpp @@ -0,0 +1,19 @@ +#pragma once + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +class ClientData; + +class ClientDataAccessor final { +public: + template + static const ClientData& GetClientData(const Client& client) { + return *client.client_data_; + } +}; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/client_internals.hpp b/grpc/include/userver/ugrpc/client/impl/client_internals.hpp index 205bd6a2eabb..e5cbabc6da27 100644 --- a/grpc/include/userver/ugrpc/client/impl/client_internals.hpp +++ b/grpc/include/userver/ugrpc/client/impl/client_internals.hpp @@ -11,6 +11,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -33,7 +34,8 @@ struct ClientInternals final { std::size_t channel_count{}; DedicatedMethodsConfig dedicated_methods_config; ChannelFactory channel_factory; - const grpc::ChannelArguments& channel_args{}; + const RetryConfig& retry_config; + const grpc::ChannelArguments& channel_args; const std::optional& default_service_config; }; diff --git a/grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp b/grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp index b475ed946d0b..ab151e615c26 100644 --- a/grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp +++ b/grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp @@ -5,8 +5,10 @@ // // Do not include this header in your code, use non-impl includes instead! +#include + #include -#include +#include #include #include #include diff --git a/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp b/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp index 69ccfd65f2bd..d534cb9efd44 100644 --- a/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp +++ b/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp @@ -6,9 +6,10 @@ // Do not include this header in your code, use non-impl includes instead! #include -#include +#include #include #include +#include #include #include diff --git a/grpc/include/userver/ugrpc/client/impl/channel_arguments_builder.hpp b/grpc/include/userver/ugrpc/client/impl/compat/channel_arguments_builder.hpp similarity index 86% rename from grpc/include/userver/ugrpc/client/impl/channel_arguments_builder.hpp rename to grpc/include/userver/ugrpc/client/impl/compat/channel_arguments_builder.hpp index 916c815bb3fa..c78e71dc4307 100644 --- a/grpc/include/userver/ugrpc/client/impl/channel_arguments_builder.hpp +++ b/grpc/include/userver/ugrpc/client/impl/compat/channel_arguments_builder.hpp @@ -9,11 +9,12 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN -namespace ugrpc::client::impl { +namespace ugrpc::client::impl::compat { class ServiceConfigBuilder final { public: @@ -25,10 +26,11 @@ class ServiceConfigBuilder final { ServiceConfigBuilder( const ugrpc::impl::StaticServiceMetadata& metadata, + const RetryConfig& retry_config, const std::optional& static_service_config ); - ServiceConfigBuilder(ServiceConfigBuilder&&) noexcept = default; + ServiceConfigBuilder(ServiceConfigBuilder&&) noexcept = delete; ServiceConfigBuilder(const ServiceConfigBuilder&) = delete; ServiceConfigBuilder& operator=(ServiceConfigBuilder&&) = delete; ServiceConfigBuilder& operator=(const ServiceConfigBuilder&) = delete; @@ -40,6 +42,8 @@ class ServiceConfigBuilder final { ugrpc::impl::StaticServiceMetadata metadata_; + RetryConfig retry_config_; + formats::json::Value static_service_config_; PreparedMethodConfigs prepared_method_configs_; @@ -50,10 +54,11 @@ class ChannelArgumentsBuilder final { ChannelArgumentsBuilder( const grpc::ChannelArguments& channel_args, const std::optional& static_service_config, + const RetryConfig& retry_config, const ugrpc::impl::StaticServiceMetadata& metadata ); - ChannelArgumentsBuilder(ChannelArgumentsBuilder&&) noexcept = default; + ChannelArgumentsBuilder(ChannelArgumentsBuilder&&) noexcept = delete; ChannelArgumentsBuilder(const ChannelArgumentsBuilder&) = delete; ChannelArgumentsBuilder& operator=(ChannelArgumentsBuilder&&) = delete; ChannelArgumentsBuilder& operator=(const ChannelArgumentsBuilder&) = delete; @@ -66,9 +71,6 @@ class ChannelArgumentsBuilder final { ServiceConfigBuilder service_config_builder_; }; -grpc::ChannelArguments -BuildChannelArguments(const grpc::ChannelArguments& channel_args, const std::optional& service_config); - -} // namespace ugrpc::client::impl +} // namespace ugrpc::client::impl::compat USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/fwd.hpp b/grpc/include/userver/ugrpc/client/impl/fwd.hpp new file mode 100644 index 000000000000..c55b8d9bae00 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/fwd.hpp @@ -0,0 +1,14 @@ +#pragma once + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +class ClientData; +class ClientDataAccessor; + +struct ClientInternals; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp b/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp index a223a7054429..a61d3b33d260 100644 --- a/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp +++ b/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp @@ -2,7 +2,7 @@ #include -#include +#include USERVER_NAMESPACE_BEGIN @@ -14,8 +14,10 @@ Response PerformUnaryCall( PrepareUnaryCallProxy&& prepare_unary_call, const Request& request ) { + utils::trx_tracker::CheckNoTransactions(params.call_name.Get()); UnaryCall unary_call{std::move(params), std::move(prepare_unary_call), request}; - return unary_call.Finish(); + unary_call.Perform(); + return unary_call.ExtractResponse(); } } // namespace ugrpc::client::impl diff --git a/grpc/include/userver/ugrpc/client/impl/response_future_impl_base.hpp b/grpc/include/userver/ugrpc/client/impl/response_future_impl_base.hpp new file mode 100644 index 000000000000..662be675d3c6 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/response_future_impl_base.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { +class CallContext; +} // namespace ugrpc::client + +namespace ugrpc::client::impl { + +template +class ResponseFutureImplBase { +public: + virtual ~ResponseFutureImplBase() = default; + + virtual bool IsReady() const = 0; + + virtual engine::FutureStatus WaitUntil(engine::Deadline deadline) const noexcept = 0; + + virtual Response Get() = 0; + + virtual void Cancel() = 0; + + virtual CallContext& GetContext() noexcept = 0; + virtual const CallContext& GetContext() const noexcept = 0; + + virtual engine::impl::ContextAccessor* TryGetContextAccessor() noexcept = 0; +}; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/retry_backoff.hpp b/grpc/include/userver/ugrpc/client/impl/retry_backoff.hpp new file mode 100644 index 000000000000..59b0c7e6611c --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/retry_backoff.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +class RetryBackoff final { +public: + RetryBackoff(); + + std::chrono::milliseconds NextAttemptDelay(); + + void Reset(); + +private: + bool initial_{}; + std::int64_t current_backoff_ms_{}; +}; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/retry_policy.hpp b/grpc/include/userver/ugrpc/client/impl/retry_policy.hpp new file mode 100644 index 000000000000..32d426477795 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/retry_policy.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +/* + Retryable Status Codes: + + - grpc::StatusCode::CANCELLED + - grpc::StatusCode::UNKNOWN + - grpc::StatusCode::DEADLINE_EXCEEDED + - grpc::StatusCode::ABORTED + - grpc::StatusCode::INTERNAL + - grpc::StatusCode::UNAVAILABLE + */ +bool IsRetryable(grpc::StatusCode status_code) noexcept; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/rpc.hpp b/grpc/include/userver/ugrpc/client/impl/rpc.hpp index a7b9ab5c83ec..f5700933d365 100644 --- a/grpc/include/userver/ugrpc/client/impl/rpc.hpp +++ b/grpc/include/userver/ugrpc/client/impl/rpc.hpp @@ -3,15 +3,10 @@ /// @file userver/ugrpc/client/impl/rpc.hpp /// @brief Classes representing an outgoing RPC -#include -#include #include #include -#include -#include -#include #include #include @@ -27,81 +22,6 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -// Contains the implementation of UnaryFinishFuture that is not dependent on template parameters. -class UnaryFinishFuture { -public: - UnaryFinishFuture(UnaryCallState& state, const google::protobuf::Message* response) noexcept; - - ~UnaryFinishFuture(); - - UnaryFinishFuture(UnaryFinishFuture&&) = delete; - UnaryFinishFuture& operator=(UnaryFinishFuture&&) = delete; - - [[nodiscard]] bool IsReady() const noexcept; - - [[nodiscard]] engine::FutureStatus WaitUntil(engine::Deadline deadline) const noexcept; - - void Get(); - - engine::impl::ContextAccessor* TryGetContextAccessor() noexcept; - -private: - void Destroy() noexcept; - - UnaryCallState& state_; - const google::protobuf::Message* response_; - mutable std::exception_ptr exception_; -}; - -/// @brief Controls a single request -> single response RPC -/// -/// This class is not thread-safe, it cannot be used from multiple tasks at the same time. -/// -/// The RPC is cancelled on destruction unless the RPC is already finished. In -/// that case the connection is not closed (it will be reused for new RPCs), and -/// the server receives `RpcInterruptedError` immediately. -template -class [[nodiscard]] UnaryCall final { - // Implementation note. For consistency with other RPC objects, UnaryCall should have been exposed to the user - // directly. However, in 90-99% use cases it's more intuitive to treat the RPC as a future (see ResponseFuture). - // If we decide to expose more controls, like lazy Finish or ReadInitialMetadata, then they should be added - // to UnaryCall, and UnaryCall should be exposed via ResponseFuture::GetCall. - -public: - template - UnaryCall( - CallParams&& params, - PrepareUnaryCallProxy&& prepare_unary_call, - const Request& request - ); - - ~UnaryCall() = default; - - UnaryCall(UnaryCall&&) = delete; - UnaryCall& operator=(UnaryCall&&) = delete; - - CallContext& GetContext() noexcept { return context_; } - const CallContext& GetContext() const noexcept { return context_; } - - /// @brief Returns the future created earlier using @ref FinishAsync. - UnaryFinishFuture& GetFinishFuture(); - const UnaryFinishFuture& GetFinishFuture() const; - - Response Finish(); - -private: - // Asynchronously finish the call. - // `FinishAsync` should not be called multiple times for the same RPC. - // Creates the future inside this `UnaryCall`. It can be retrieved using @ref GetFinishFuture. - void FinishAsync(); - - UnaryCallState state_; - CallContext context_; - Response response_; - RawResponseReader reader_; - std::optional finish_future_; -}; - /// @brief Controls a single request -> response stream RPC /// /// This class is not thread-safe except for `GetContext`. @@ -326,51 +246,6 @@ class [[nodiscard]] BidirectionalStream final { RawReaderWriter stream_; }; -template -template -UnaryCall::UnaryCall( - CallParams&& params, - PrepareUnaryCallProxy&& prepare_unary_call, - const Request& request -) - : state_{std::move(params)}, context_{utils::impl::InternalTag{}, state_} { - RunMiddlewarePipeline(state_, StartCallHooks(ToBaseMessage(&request))); - - // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) - reader_ = prepare_unary_call(state_.GetStub(), &state_.GetClientContext(), request, &state_.GetQueue()); - reader_->StartCall(); - - FinishAsync(); -} - -template -UnaryFinishFuture& UnaryCall::GetFinishFuture() { - UASSERT(finish_future_); - return *finish_future_; -} - -template -const UnaryFinishFuture& UnaryCall::GetFinishFuture() const { - UASSERT(finish_future_); - return *finish_future_; -} - -template -Response UnaryCall::Finish() { - GetFinishFuture().Get(); - return std::move(response_); -} - -template -void UnaryCall::FinishAsync() { - state_.EmplaceFinishAsyncMethodInvocation(); - auto& status = state_.GetStatus(); - auto& finish = state_.GetFinishAsyncMethodInvocation(); - UASSERT(reader_); - reader_->Finish(&response_, &status, finish.GetCompletionTag()); - finish_future_.emplace(state_, ToBaseMessage(&response_)); -} - template template InputStream::InputStream( diff --git a/grpc/include/userver/ugrpc/client/impl/unary_call.hpp b/grpc/include/userver/ugrpc/client/impl/unary_call.hpp new file mode 100644 index 000000000000..c41aa62006c1 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/unary_call.hpp @@ -0,0 +1,177 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +template +class UnaryCall final { +public: + using PrepareUnaryCall = PrepareUnaryCallProxy; + + UnaryCall(CallParams&& params, PrepareUnaryCall&& prepare_unary_call, const Request& request) + : call_options_{std::move(params.call_options)}, + state_{std::move(params), CallKind::kUnaryCall}, + context_{utils::impl::InternalTag{}, state_}, + prepare_unary_call_{std::move(prepare_unary_call)}, + request_{request} {} + + ~UnaryCall() = default; + + UnaryCall(UnaryCall&&) = delete; + UnaryCall& operator=(UnaryCall&&) = delete; + + CallContext& GetContext() noexcept { return context_; } + const CallContext& GetContext() const noexcept { return context_; } + + void Perform() { CallWithRetries(); } + + Response&& ExtractResponse() { + if (!done_) { + throw RpcCancelledError{state_.GetCallName(), "UnaryCall"}; + } + if (!status_.ok()) { + ugrpc::client::ThrowErrorWithStatus(state_.GetCallName(), std::move(status_)); + } + return std::move(response_); + } + + void Abandon() { abandoned_ = true; } + +private: + void CallWithRetries() { + const utils::FastScopeGuard commit_state_guard([this]() noexcept { state_.Commit(); }); + + const auto task_deadline = USERVER_NAMESPACE::server::request::GetTaskInheritedDeadline(); + const int max_attempts = call_options_.GetAttempts(); + state_.GetSpan().AddTag(tracing::kMaxAttempts, max_attempts); + + int attempt = 1; + RetryBackoff retry_backoff; + + while (!engine::current_task::ShouldCancel()) { + state_.GetSpan().AddTag(tracing::kAttempts, attempt); + impl::SetupClientContext(state_, call_options_); + + const bool completed = PerformAttempt(); + if (!completed) { + break; + } + + if (status_.ok()) { + OnDone(status_); + return; + } + + if (max_attempts <= attempt || !IsRetryable(status_.error_code())) { + OnDone(status_); + return; + } + + const auto delay = retry_backoff.NextAttemptDelay(); + if (task_deadline.IsReachable() && task_deadline.TimeLeft() <= delay) { + OnDone(status_); + return; + } + + ++attempt; + engine::InterruptibleSleepFor(delay); + } + + OnCancelled(); + } + + std::unique_ptr> StartCall() { + auto call = prepare_unary_call_(state_.GetStub(), &state_.GetClientContext(), request_, &state_.GetQueue()); + call->StartCall(); + return call; + } + + bool PerformAttempt() { + RunStartCallHooks(); + + auto response_reader = StartCall(); + + ugrpc::impl::AsyncMethodInvocation invocation; + response_reader->Finish(&response_, &status_, invocation.GetCompletionTag()); + const auto wait_status = invocation.Wait(); + if (ugrpc::impl::AsyncMethodInvocation::WaitStatus::kCancelled == wait_status) { + state_.GetClientContext().TryCancel(); + return false; + } + + if (status_.ok() && ugrpc::impl::AsyncMethodInvocation::WaitStatus::kError == wait_status) { + // CompletionQueue returned ok=false. For Client-side Finish ok should always be true. + // If a GRPC status has been set despite ok=false, or if it has been set by a previous attempt, keep it. + // Otherwise, propagate ok=false to a failure status. + status_ = grpc::Status{grpc::StatusCode::INTERNAL, "Client-side Finish CompletionQueue status failed"}; + } + + RunFinishHooks(status_); + + return true; + } + + void RunStartCallHooks() { impl::RunMiddlewarePipeline(state_, StartCallHooks(ToBaseMessage(&request_))); } + + void RunFinishHooks(const grpc::Status& status) { + impl::RunMiddlewarePipeline(state_, FinishHooks(status, ToBaseMessage(&response_))); + } + + void OnDone(const grpc::Status& status) { + done_ = true; + impl::HandleCallStatistics(state_, status); + impl::SetStatusForSpan(state_.GetSpan(), status); + } + + void OnCancelled() { + if (abandoned_) { + impl::SetErrorForSpan(state_.GetSpan(), "Call abandoned"); + } else { + state_.GetStatsScope().OnCancelled(); + impl::SetErrorForSpan(state_.GetSpan(), "Call cancelled"); + } + state_.GetStatsScope().Flush(); + } + + CallOptions call_options_; + CallState state_; + CallContext context_; + + PrepareUnaryCall prepare_unary_call_; + const Request& request_; + + Response response_; + grpc::Status status_; + bool done_{false}; + + std::atomic abandoned_{false}; +}; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/qos.hpp b/grpc/include/userver/ugrpc/client/qos.hpp index 148e782130fa..1309beea06fb 100644 --- a/grpc/include/userver/ugrpc/client/qos.hpp +++ b/grpc/include/userver/ugrpc/client/qos.hpp @@ -42,10 +42,6 @@ Qos Parse(const formats::json::Value& value, formats::parse::To); formats::json::Value Serialize(const Qos& qos, formats::serialize::To); -std::optional GetAttempts(const Qos& qos); - -std::optional GetTotalTimeout(const Qos& qos); - } // namespace ugrpc::client USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/response_future.hpp b/grpc/include/userver/ugrpc/client/response_future.hpp index 97ee5d66abdc..b02d5d0020f4 100644 --- a/grpc/include/userver/ugrpc/client/response_future.hpp +++ b/grpc/include/userver/ugrpc/client/response_future.hpp @@ -5,7 +5,7 @@ #include -#include +#include USERVER_NAMESPACE_BEGIN @@ -29,13 +29,13 @@ class [[nodiscard]] ResponseFuture final { /// @brief Checks if the asynchronous call has completed /// Note, that once user gets result, IsReady should not be called /// @return true if result ready - [[nodiscard]] bool IsReady() const noexcept { return call_->GetFinishFuture().IsReady(); } + [[nodiscard]] bool IsReady() const { return impl_->IsReady(); } /// @brief Await response until specified timepoint /// /// @throws ugrpc::client::RpcError on an RPC error - [[nodiscard]] engine::FutureStatus WaitUntil(engine::Deadline deadline) const { - return call_->GetFinishFuture().WaitUntil(deadline); + [[nodiscard]] engine::FutureStatus WaitUntil(engine::Deadline deadline) const noexcept { + return impl_->WaitUntil(deadline); } /// @brief Await and read the response @@ -47,13 +47,16 @@ class [[nodiscard]] ResponseFuture final { /// @returns the response on success /// @throws ugrpc::client::RpcError on an RPC error /// @throws ugrpc::client::RpcCancelledError on task cancellation - Response Get() { return call_->Finish(); } + Response Get() { return impl_->Get(); } + + /// @brief Cancel call + void Cancel() { return impl_->Cancel(); } /// @brief Get call context, useful e.g. for accessing metadata. - CallContext& GetContext() { return call_->GetContext(); } + CallContext& GetContext() { return impl_->GetContext(); } /// @overload - const CallContext& GetContext() const { return call_->GetContext(); } + const CallContext& GetContext() const { return impl_->GetContext(); } /// @cond // For internal use only @@ -63,17 +66,18 @@ class [[nodiscard]] ResponseFuture final { impl::PrepareUnaryCallProxy&& prepare_unary_call, const Request& request ) - : call_(std::make_unique>(std::move(params), std::move(prepare_unary_call), request) - ) {} + : impl_{std::make_unique>( + std::move(params), + std::move(prepare_unary_call), + request + )} {} // For internal use only. - engine::impl::ContextAccessor* TryGetContextAccessor() noexcept { - return call_->GetFinishFuture().TryGetContextAccessor(); - } + engine::impl::ContextAccessor* TryGetContextAccessor() noexcept { return impl_->TryGetContextAccessor(); } /// @endcond private: - std::unique_ptr> call_; + std::unique_ptr> impl_; }; } // namespace ugrpc::client diff --git a/grpc/include/userver/ugrpc/client/retry_config.hpp b/grpc/include/userver/ugrpc/client/retry_config.hpp new file mode 100644 index 000000000000..939c21fac2ed --- /dev/null +++ b/grpc/include/userver/ugrpc/client/retry_config.hpp @@ -0,0 +1,22 @@ +#pragma once + +/// @file userver/ugrpc/client/retry_config.hpp +/// @brief @copybrief ugrpc::client::RetryConfig + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { + +/// Retry configuration for outgoing RPCs +struct RetryConfig final { + /// The maximum number of RPC attempts, including the original attempt + int attempts{1}; +}; + +RetryConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To); + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/simple_client_component.hpp b/grpc/include/userver/ugrpc/client/simple_client_component.hpp index a73e3d3e5c3b..6a00f0ce00dc 100644 --- a/grpc/include/userver/ugrpc/client/simple_client_component.hpp +++ b/grpc/include/userver/ugrpc/client/simple_client_component.hpp @@ -77,7 +77,8 @@ class SimpleClientComponent : public impl::SimpleClientComponentAny { : SimpleClientComponentAny(config, context), client_(FindFactory(config, context).MakeClient(MakeClientSettings(config, nullptr))) {} - /// To use a ClientQos config, + /// To use a ClientQos config, derive from SimpleClientComponent and provide the parameter at construction: + /// @snippet samples/grpc_service/src/greeter_client.hpp component SimpleClientComponent( const components::ComponentConfig& config, const components::ComponentContext& context, @@ -86,7 +87,7 @@ class SimpleClientComponent : public impl::SimpleClientComponentAny { : SimpleClientComponentAny(config, context), client_(FindFactory(config, context).MakeClient(MakeClientSettings(config, &client_qos))) {} - /// @@brief Get gRPC service client + /// @brief Get gRPC service client Client& GetClient() { return client_; } private: diff --git a/grpc/include/userver/ugrpc/client/stream.hpp b/grpc/include/userver/ugrpc/client/stream.hpp index 77f0ee54ac75..91482f573781 100644 --- a/grpc/include/userver/ugrpc/client/stream.hpp +++ b/grpc/include/userver/ugrpc/client/stream.hpp @@ -15,11 +15,11 @@ namespace ugrpc::client { /// /// This class is not thread-safe except for `GetContext`. /// -/// The RPC is cancelled on destruction unless the stream is closed (`Read` has -/// returned `false`). In that case the connection is not closed (it will be -/// reused for new RPCs), and the server receives `RpcInterruptedError` -/// immediately. gRPC provides no way to early-close a server-streaming RPC -/// gracefully. +/// The RPC should be completed by reading until @ref ugrpc::client::Reader::Read returns `false`. +/// If destroyed early, the RPC is cancelled. The server gets @ref ugrpc::client::RpcInterruptedError +/// and the `abandoned-error` metric is incremented. The connection stays open for reuse. +/// gRPC provides no way to early-close a server-streaming RPC gracefully. +/// See @ref ugrpc::client::ReadRemainingAndFinish for graceful completion. template class [[nodiscard]] Reader final { public: @@ -62,9 +62,9 @@ class [[nodiscard]] Reader final { /// /// This class is not thread-safe except for `GetContext`. /// -/// The RPC is cancelled on destruction unless `Finish` has been called. In that -/// case the connection is not closed (it will be reused for new RPCs), and the -/// server receives `RpcInterruptedError` immediately. +/// The RPC should be completed by calling @ref ugrpc::client::Writer::Finish. +/// If destroyed early, the RPC is cancelled. The server gets @ref ugrpc::client::RpcInterruptedError +/// and the `abandoned-error` metric is incremented. When properly finished, the connection stays open for reuse. template class [[nodiscard]] Writer final { public: @@ -141,11 +141,11 @@ class [[nodiscard]] Writer final { /// /// `WriteAndCheck` is NOT thread-safe. /// -/// The RPC is cancelled on destruction unless the stream is closed (`Read` has -/// returned `false`). In that case the connection is not closed (it will be -/// reused for new RPCs), and the server receives `RpcInterruptedError` -/// immediately. gRPC provides no way to early-close a server-streaming RPC -/// gracefully. +/// The RPC should be completed by reading until @ref ugrpc::client::Reader::Read returns `false`. +/// If destroyed early, the RPC is cancelled. The server gets @ref ugrpc::client::RpcInterruptedError +/// and the `abandoned-error` metric is incremented. The connection stays open for reuse. +/// gRPC provides no way to early-close a server-streaming RPC gracefully. +// See @ref ugrpc::client::ReadRemainingAndFinish and @ref ugrpc::client::PingPongFinish for graceful completion. /// /// `Read` and `AsyncRead` can throw if error status is received from server. /// User MUST NOT call `Read` or `AsyncRead` again after failure of any of these diff --git a/grpc/include/userver/ugrpc/impl/rpc_type.hpp b/grpc/include/userver/ugrpc/impl/rpc_type.hpp new file mode 100644 index 000000000000..9d4709ccab22 --- /dev/null +++ b/grpc/include/userver/ugrpc/impl/rpc_type.hpp @@ -0,0 +1,17 @@ +#pragma once + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::impl { + +/// Type categorizes RPCs by unary or streaming type +enum class RpcType { + kUnary, + kClientStreaming, + kServerStreaming, + kBidiStreaming, +}; + +} // namespace ugrpc::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/impl/static_service_metadata.hpp b/grpc/include/userver/ugrpc/impl/static_service_metadata.hpp index 738892f83cbb..0ee74f541e4e 100644 --- a/grpc/include/userver/ugrpc/impl/static_service_metadata.hpp +++ b/grpc/include/userver/ugrpc/impl/static_service_metadata.hpp @@ -5,35 +5,46 @@ #include +#include + USERVER_NAMESPACE_BEGIN namespace ugrpc::impl { +/// Descriptor of an RPC method +struct MethodDescriptor final { + std::string_view method_full_name; + RpcType method_type; +}; + /// Per-gRPC-service statically generated data struct StaticServiceMetadata final { std::string_view service_full_name; - utils::span method_full_names; + utils::span methods; }; template -constexpr StaticServiceMetadata MakeStaticServiceMetadata(utils::span method_full_names -) noexcept { - return {GrpcppService::service_full_name(), method_full_names}; +constexpr StaticServiceMetadata MakeStaticServiceMetadata(utils::span methods) noexcept { + return {GrpcppService::service_full_name(), methods}; } constexpr std::size_t GetMethodsCount(const StaticServiceMetadata& metadata) noexcept { - return metadata.method_full_names.size(); + return metadata.methods.size(); } constexpr std::string_view GetMethodFullName(const StaticServiceMetadata& metadata, std::size_t method_id) { - return metadata.method_full_names[method_id]; + return metadata.methods[method_id].method_full_name; } constexpr std::string_view GetMethodName(const StaticServiceMetadata& metadata, std::size_t method_id) { - const auto& method_full_name = metadata.method_full_names[method_id]; + auto method_full_name = GetMethodFullName(metadata, method_id); return method_full_name.substr(metadata.service_full_name.size() + 1); } +constexpr RpcType GetMethodType(const StaticServiceMetadata& metadata, std::size_t method_id) { + return metadata.methods[method_id].method_type; +} + std::optional FindMethod(const ugrpc::impl::StaticServiceMetadata& metadata, std::string_view method_full_name); diff --git a/grpc/include/userver/ugrpc/protobuf_logging.hpp b/grpc/include/userver/ugrpc/protobuf_logging.hpp new file mode 100644 index 000000000000..15553b113b2e --- /dev/null +++ b/grpc/include/userver/ugrpc/protobuf_logging.hpp @@ -0,0 +1,91 @@ +#pragma once + +/// @file +/// @brief Public API for protobuf logging utilities. + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc { + +inline constexpr std::size_t kDefaultDebugStringLimit = 1024; + +/// @brief Convert protobuf message to limited debug string for logging. +/// +/// Differences from built-in `DebugString`: +/// - Fields marked with `[debug_redact]` option are hidden (`DebugString` only does so since Protobuf v30). +/// - When the character limit is reached, serialization stops immediately (`DebugString` still wastes CPU on +/// serializing the entire message regardless). +/// +/// @param message The protobuf message to convert. +/// @param limit Maximum size of the resulting string. +/// Avoid setting this to very large values as it may cause OOM (Out of Memory) issues. +/// @returns String representation of the message, truncated if necessary. +/// +/// @warning This is a debug representation of protobuf that is unstable and should only be used for diagnostics. +/// The order of keys in maps is unstable; the format itself can change even within a single run. +/// You CANNOT parse back from this debug text representation. +/// You CANNOT use it for equality match with reference values in gtest. +std::string ToLimitedDebugString(const google::protobuf::Message& message, std::size_t limit); + +/// @brief Convert protobuf message to unlimited debug string for logging. +/// +/// Differences from built-in `DebugString`: +/// - Fields marked with `[debug_redact]` option are hidden (`DebugString` only does so since Protobuf v30). +/// +/// @param message The protobuf message to convert. +/// @returns String representation of the message. +/// +/// @warning This is a debug representation of protobuf that is unstable and should only be used for diagnostics. +/// The order of keys in maps is unstable; the format itself can change even within a single run. +/// You CANNOT parse back from this debug text representation. +/// You CANNOT use it for equality match with reference values in gtest. +std::string ToUnlimitedDebugString(const google::protobuf::Message& message); + +/// @brief Get error details from `grpc::Status` for logging with size limit. +/// @param status The `grpc::Status` to extract details from. +/// @param max_size Maximum size of the resulting string. +/// Avoid setting this to very large values as it may cause OOM (Out of Memory) issues. +/// @returns String representation of error details, truncated if necessary. +/// +/// @warning This is a debug representation of protobuf that is unstable and should only be used for diagnostics. +/// The order of keys in maps is unstable; the format itself can change even within a single run. +/// You CANNOT parse back from this debug text representation. +/// You CANNOT use it for equality match with reference values in gtest. +std::string ToLimitedDebugString(const grpc::Status& status, std::size_t max_size = kDefaultDebugStringLimit); + +/// @brief Get error details from `grpc::Status` for logging without size limit. +/// @param status The `grpc::Status` to extract details from. +/// @returns String representation of error details. +/// +/// @warning This is a debug representation of protobuf that is unstable and should only be used for diagnostics. +/// The order of keys in maps is unstable; the format itself can change even within a single run. +/// You CANNOT parse back from this debug text representation. +/// You CANNOT use it for equality match with reference values in gtest. +std::string ToUnlimitedDebugString(const grpc::Status& status); + +} // namespace ugrpc + +USERVER_NAMESPACE_END + +namespace fmt { + +/// @brief `fmt::format` support for protobuf messages +template +struct formatter>, char>> { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template + auto format(const T& message, FormatContext& ctx) const { + return fmt::format_to( + ctx.out(), + "{}", + USERVER_NAMESPACE::ugrpc::ToLimitedDebugString(message, USERVER_NAMESPACE::ugrpc::kDefaultDebugStringLimit) + ); + } +}; + +} // namespace fmt diff --git a/grpc/include/userver/ugrpc/server/impl/call_state.hpp b/grpc/include/userver/ugrpc/server/impl/call_state.hpp index e4775495af41..31e2ab7772e9 100644 --- a/grpc/include/userver/ugrpc/server/impl/call_state.hpp +++ b/grpc/include/userver/ugrpc/server/impl/call_state.hpp @@ -17,6 +17,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -37,6 +38,7 @@ struct CallParams { std::optional& span_storage; const Middlewares& middlewares; const dynamic_config::Source& config_source; + const boost::container::flat_map& status_codes_log_level; }; // Non-templated state of CallProcessor. Can be non-movable. diff --git a/grpc/include/userver/ugrpc/server/impl/service_internals.hpp b/grpc/include/userver/ugrpc/server/impl/service_internals.hpp index 95691cb39d2e..ae47cb5e2f83 100644 --- a/grpc/include/userver/ugrpc/server/impl/service_internals.hpp +++ b/grpc/include/userver/ugrpc/server/impl/service_internals.hpp @@ -1,10 +1,14 @@ #pragma once +#include + #include #include #include +#include #include +#include USERVER_NAMESPACE_BEGIN @@ -23,6 +27,7 @@ struct ServiceInternals final { ugrpc::impl::StatisticsStorage& statistics_storage; Middlewares middlewares; const dynamic_config::Source config_source; + boost::container::flat_map status_codes_log_level; }; } // namespace ugrpc::server::impl diff --git a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp index 4406f8868876..66cc227525ec 100644 --- a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp +++ b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp @@ -170,6 +170,7 @@ class CallData final { span_storage_, method_data_.service_data.internals.middlewares, method_data_.service_data.internals.config_source, + method_data_.service_data.internals.status_codes_log_level, }, raw_responder_, initial_request_, @@ -233,13 +234,13 @@ class ServiceWorkerImpl final : public ServiceWorker { template std::unique_ptr MakeServiceWorker( ServiceInternals&& internals, - const std::array& method_full_names, + const std::array& methods, Service& service, ServiceMethods... service_methods ) { return std::make_unique>( std::move(internals), - ugrpc::impl::MakeStaticServiceMetadata(method_full_names), + ugrpc::impl::MakeStaticServiceMetadata(methods), service, service_methods... ); diff --git a/grpc/include/userver/ugrpc/server/middlewares/access_log/component.hpp b/grpc/include/userver/ugrpc/server/middlewares/access_log/component.hpp index 3d21b8fb2bd1..4f9d0fb3ad3e 100644 --- a/grpc/include/userver/ugrpc/server/middlewares/access_log/component.hpp +++ b/grpc/include/userver/ugrpc/server/middlewares/access_log/component.hpp @@ -14,6 +14,10 @@ USERVER_NAMESPACE_BEGIN /// @see @ref ugrpc::server::middlewares::access_log::Component namespace ugrpc::server::middlewares::access_log { +/// @brief Storage to handle additional fields in access_log +/// @snippet grpc/tests/logging_test.cpp grpc log extra tag +inline const utils::AnyStorageDataTag kLogExtraTag; + /// @ingroup userver_components userver_base_classes /// /// @brief gRPC server access log middleware component. Writes one TSKV log line per handled RPC in a static format. diff --git a/grpc/include/userver/ugrpc/server/middlewares/access_log/log_extra.hpp b/grpc/include/userver/ugrpc/server/middlewares/access_log/log_extra.hpp new file mode 100644 index 000000000000..03b489c548a3 --- /dev/null +++ b/grpc/include/userver/ugrpc/server/middlewares/access_log/log_extra.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server::middlewares::access_log { +/// @brief Adds or extends log extra fields for gRPC access logging +/// @param context Middleware call context containing the storage +/// @param log_extra Additional log fields to add (will be moved from) +/// @snippet grpc/tests/logging_test.cpp grpc log extra tag +/// +/// If log extra fields already exist in the context, extends them with new fields. +/// Otherwise, creates new log extra storage with the provided fields. +void SetAdditionalLogKeys(MiddlewareCallContext& context, logging::LogExtra&& log_extra); +} // namespace ugrpc::server::middlewares::access_log + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/server/middlewares/log/component.hpp b/grpc/include/userver/ugrpc/server/middlewares/log/component.hpp index b23113c26d2d..fa8b850cb92a 100644 --- a/grpc/include/userver/ugrpc/server/middlewares/log/component.hpp +++ b/grpc/include/userver/ugrpc/server/middlewares/log/component.hpp @@ -29,6 +29,7 @@ namespace ugrpc::server::middlewares::log { /// msg-log-level | logging level to use for request and response messages themselves | debug /// msg-size-log-limit | max message size to log, the rest will be truncated | 512 /// local-log-level | local log level of the span for user-provided handler | debug +/// status-codes-log-level | gRPC status code string -> response log level map | {} /// /// ## Static configuration example: /// diff --git a/grpc/include/userver/ugrpc/server/service_base.hpp b/grpc/include/userver/ugrpc/server/service_base.hpp index 95850246ee3d..6b366a8e25d9 100644 --- a/grpc/include/userver/ugrpc/server/service_base.hpp +++ b/grpc/include/userver/ugrpc/server/service_base.hpp @@ -3,11 +3,14 @@ /// @file userver/ugrpc/server/service_base.hpp /// @brief @copybrief ugrpc::server::ServiceBase +#include + #include #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -24,6 +27,10 @@ struct ServiceConfig final { /// Server middlewares to use for the gRPC service. Middlewares middlewares; + + /// map of "status_code": log_level items to override span log level for specific status codes + /// see @ref ugrpc::kStatusCodesMap for available statuses + boost::container::flat_map status_codes_log_level; }; /// @brief The type-erased base class for all gRPC service implementations diff --git a/grpc/include/userver/ugrpc/server/service_component_base.hpp b/grpc/include/userver/ugrpc/server/service_component_base.hpp index 32fb85890b20..b42ce93b241f 100644 --- a/grpc/include/userver/ugrpc/server/service_component_base.hpp +++ b/grpc/include/userver/ugrpc/server/service_component_base.hpp @@ -43,6 +43,7 @@ using MiddlewareRunnerComponentBase = USERVER_NAMESPACE::middlewares::RunnerComp /// disable-user-pipeline-middlewares | flag to disable `groups::User` middlewares from pipeline | false /// disable-all-pipeline-middlewares | flag to disable all middlewares from pipeline | false /// middlewares | middlewares names to use | `{}` (use server defaults) +/// status-codes-log-level | gRPC status code string -> span log level map | {} // clang-format on diff --git a/grpc/include/userver/ugrpc/status_codes.hpp b/grpc/include/userver/ugrpc/status_codes.hpp index 0017ae8bcb66..cb394fa7afe3 100644 --- a/grpc/include/userver/ugrpc/status_codes.hpp +++ b/grpc/include/userver/ugrpc/status_codes.hpp @@ -5,6 +5,9 @@ #include +#include +#include + USERVER_NAMESPACE_BEGIN namespace ugrpc { @@ -34,4 +37,20 @@ bool IsServerError(grpc::StatusCode code) noexcept; } // namespace ugrpc +namespace formats::parse { + +/// @ref yaml_config::YamlConfig parsing support for `grpc::StatusCode`. +grpc::StatusCode Parse(const yaml_config::YamlConfig& value, To); + +/// Support for parsing `grpc::StatusCode` from string. Used for headers and map keys. +grpc::StatusCode Parse(std::string_view value, To); + +} // namespace formats::parse + +namespace formats::serialize { + +formats::json::Value Serialize(const grpc::StatusCode& value, formats::serialize::To); + +} + USERVER_NAMESPACE_END diff --git a/grpc/library.yaml b/grpc/library.yaml index 5a346b676fbe..c5e8c0277f57 100644 --- a/grpc/library.yaml +++ b/grpc/library.yaml @@ -10,5 +10,6 @@ libraries: configs: names: + - EGRESS_GRPC_PROXY_ENABLED - USERVER_GRPC_CLIENT_ENABLE_DEADLINE_PROPAGATION - USERVER_GRPC_SERVER_CANCEL_TASK_BY_DEADLINE diff --git a/grpc/src/ugrpc/client/call_context.cpp b/grpc/src/ugrpc/client/call_context.cpp index dd91907fff08..5d0edb0ec3e0 100644 --- a/grpc/src/ugrpc/client/call_context.cpp +++ b/grpc/src/ugrpc/client/call_context.cpp @@ -8,9 +8,9 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client { -CallContext::CallContext(utils::impl::InternalTag, impl::CallState& state) : state_(state) {} +CallContext::CallContext(utils::impl::InternalTag, impl::CallState& state) noexcept : state_(state) {} -grpc::ClientContext& CallContext::GetClientContext() noexcept { return state_.GetClientContext(); } +grpc::ClientContext& CallContext::GetClientContext() { return state_.GetClientContextCommitted(); } std::string_view CallContext::GetClientName() const noexcept { return state_.GetClientName(); } diff --git a/grpc/src/ugrpc/client/call_options.cpp b/grpc/src/ugrpc/client/call_options.cpp index 3ac05ed2c9d0..ace162e7f8b2 100644 --- a/grpc/src/ugrpc/client/call_options.cpp +++ b/grpc/src/ugrpc/client/call_options.cpp @@ -6,18 +6,18 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client { -void CallOptions::AddMetadata(std::string_view meta_key, std::string_view meta_value) { - metadata_.emplace_back(ugrpc::impl::ToGrpcString(meta_key), ugrpc::impl::ToGrpcString(meta_value)); -} - -// void CallOptions::SetAttempts(int attempts) { attempts_ = attempts; } +void CallOptions::SetAttempts(int attempts) { attempts_ = attempts; } -// int CallOptions::GetAttempts() const { return attempts_; } +int CallOptions::GetAttempts() const { return attempts_; } void CallOptions::SetTimeout(std::chrono::milliseconds timeout) { timeout_ = timeout; } std::chrono::milliseconds CallOptions::GetTimeout() const { return timeout_; } +void CallOptions::AddMetadata(std::string_view meta_key, std::string_view meta_value) { + metadata_.emplace_back(ugrpc::impl::ToGrpcString(meta_key), ugrpc::impl::ToGrpcString(meta_value)); +} + void CallOptions::SetClientContextFactory(ClientContextFactory&& client_context_factory) { client_context_factory_ = std::move(client_context_factory); } diff --git a/grpc/src/ugrpc/client/channels.cpp b/grpc/src/ugrpc/client/channels.cpp index 5227c7a68697..1aee55a60456 100644 --- a/grpc/src/ugrpc/client/channels.cpp +++ b/grpc/src/ugrpc/client/channels.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -37,7 +38,7 @@ DoTryWaitForConnected(grpc::Channel& channel, grpc::CompletionQueue& queue, engi } // namespace [[nodiscard]] bool TryWaitForConnected( - ClientData& client_data, + const ClientData& client_data, engine::Deadline deadline, engine::TaskProcessor& blocking_task_processor ) { diff --git a/grpc/src/ugrpc/client/client_factory.cpp b/grpc/src/ugrpc/client/client_factory.cpp index 10b4c785b1f1..a2365ebec873 100644 --- a/grpc/src/ugrpc/client/client_factory.cpp +++ b/grpc/src/ugrpc/client/client_factory.cpp @@ -56,6 +56,7 @@ impl::ClientInternals ClientFactory::MakeClientInternals( client_factory_settings_.channel_count, std::move(client_settings.dedicated_methods_config), std::move(channel_factory), + client_factory_settings_.retry_config, client_factory_settings_.channel_args, client_factory_settings_.default_service_config, }; diff --git a/grpc/src/ugrpc/client/client_factory_component.cpp b/grpc/src/ugrpc/client/client_factory_component.cpp index ce863439f058..dc6000f41a0c 100644 --- a/grpc/src/ugrpc/client/client_factory_component.cpp +++ b/grpc/src/ugrpc/client/client_factory_component.cpp @@ -3,13 +3,17 @@ #include #include #include +#include #include #include #include -#include #include #include +#include + +#include +#include USERVER_NAMESPACE_BEGIN @@ -24,6 +28,43 @@ const storages::secdist::SecdistConfig* GetSecdist(const components::ComponentCo return nullptr; } +std::shared_ptr +MakeCredentials(impl::AuthType auth_type, const grpc::SslCredentialsOptions& ssl_credentials_options) { + if (auth_type == impl::AuthType::kSsl) { + LOG_INFO() << "GRPC client (SSL) initialized..."; + return grpc::SslCredentials(ssl_credentials_options); + } else { + LOG_INFO() << "GRPC client (non SSL) initialized..."; + return grpc::InsecureChannelCredentials(); + } +} + +ClientFactorySettings +MakeClientFactorySettings(impl::ClientFactoryConfig&& config, const storages::secdist::SecdistConfig* secdist) { + std::shared_ptr credentials = + MakeCredentials(config.auth_type, config.ssl_credentials_options); + std::unordered_map> client_credentials; + + if (secdist) { + const auto& tokens = secdist->Get(); + + for (const auto& [client_name, token] : tokens.tokens) { + client_credentials[client_name] = grpc::CompositeChannelCredentials( + credentials, grpc::AccessTokenCredentials(ugrpc::impl::ToGrpcString(token)) + ); + } + } + + return ClientFactorySettings{ + std::move(credentials), + std::move(client_credentials), + std::move(config.retry_config), + std::move(config.channel_args), + std::move(config.default_service_config), + config.channel_count, + }; +} + } // namespace ClientFactoryComponent::ClientFactoryComponent( @@ -44,7 +85,7 @@ ClientFactoryComponent::ClientFactoryComponent( const auto* secdist = GetSecdist(context); factory_.emplace( - MakeFactorySettings(std::move(factory_config), secdist), + MakeClientFactorySettings(std::move(factory_config), secdist), client_common_component.blocking_task_processor_, *this, // impl::PipelineCreatorInterface& client_common_component.completion_queues_, @@ -87,6 +128,17 @@ additionalProperties: false type: string description: The path to file containing the PEM encoding of the client's certificate chain defaultDescription: absent + retry-config: + type: object + description: Retry configuration for outgoing RPCs + defaultDescription: '{}' + additionalProperties: false + properties: + attempts: + type: integer + description: The maximum number of RPC attempts, including the original attempt + defaultDescription: 1 + minimum: 1 channel-args: type: object description: a map of channel arguments, see gRPC Core docs diff --git a/grpc/src/ugrpc/client/generic_client.cpp b/grpc/src/ugrpc/client/generic_client.cpp index 28c29ce5f4ea..b49860e89c31 100644 --- a/grpc/src/ugrpc/client/generic_client.cpp +++ b/grpc/src/ugrpc/client/generic_client.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -21,12 +22,18 @@ struct GenericService final { } // namespace GenericClient::GenericClient(impl::ClientInternals&& internals) - : impl_(std::move(internals), impl::GenericClientTag{}, std::in_place_type) { + : client_data_(std::move(internals), impl::GenericClientTag{}, std::in_place_type) { // There is no technical reason why QOS configs should be unsupported here. // However, it would be difficult to detect non-existent RPC names in QOS. - UINVARIANT(!impl_.GetClientQos(), "Client QOS configs are unsupported for generic services"); + UINVARIANT(!client_data_->GetClientQos(), "Client QOS configs are unsupported for generic services"); } +GenericClient::GenericClient(GenericClient&&) noexcept = default; + +GenericClient& GenericClient::operator=(GenericClient&&) noexcept = default; + +GenericClient::~GenericClient() = default; + ResponseFuture GenericClient::AsyncUnaryCall( std::string_view call_name, const grpc::ByteBuffer& request, @@ -35,7 +42,7 @@ ResponseFuture GenericClient::AsyncUnaryCall( ) const { auto method_name = utils::StrCat("/", call_name); return { - impl::CreateGenericCallParams(impl_, call_name, std::move(call_options), std::move(generic_options)), + impl::CreateGenericCallParams(*client_data_, call_name, std::move(call_options), std::move(generic_options)), impl::PrepareUnaryCallProxy(&grpc::GenericStub::PrepareUnaryCall, std::move(method_name)), request, }; @@ -49,7 +56,7 @@ grpc::ByteBuffer GenericClient::UnaryCall( ) const { auto method_name = utils::StrCat("/", call_name); return impl::PerformUnaryCall( - impl::CreateGenericCallParams(impl_, call_name, std::move(call_options), std::move(generic_options)), + impl::CreateGenericCallParams(*client_data_, call_name, std::move(call_options), std::move(generic_options)), impl::PrepareUnaryCallProxy(&grpc::GenericStub::PrepareUnaryCall, std::move(method_name)), request ); diff --git a/grpc/src/ugrpc/client/impl/async_methods.cpp b/grpc/src/ugrpc/client/impl/async_methods.cpp deleted file mode 100644 index d4fc019c47bf..000000000000 --- a/grpc/src/ugrpc/client/impl/async_methods.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include - -#include - -#include -#include -#include - -USERVER_NAMESPACE_BEGIN - -namespace ugrpc::client::impl { - -namespace { - -void SetStatusAndResetSpan(CallState& state, const grpc::Status& status) noexcept { - SetStatusForSpan(state.GetSpan(), status); - state.ResetSpan(); -} - -void SetErrorAndResetSpan(CallState& state, std::string_view error_message) noexcept { - SetErrorForSpan(state.GetSpan(), error_message); - state.ResetSpan(); -} - -} // namespace - -ugrpc::impl::AsyncMethodInvocation::WaitStatus WaitAndTryCancelIfNeeded( - ugrpc::impl::AsyncMethodInvocation& invocation, - engine::Deadline deadline, - grpc::ClientContext& context -) noexcept { - const auto wait_status = invocation.WaitUntil(deadline); - if (ugrpc::impl::AsyncMethodInvocation::WaitStatus::kCancelled == wait_status) { - context.TryCancel(); - } - return wait_status; -} - -ugrpc::impl::AsyncMethodInvocation::WaitStatus -WaitAndTryCancelIfNeeded(ugrpc::impl::AsyncMethodInvocation& invocation, grpc::ClientContext& context) noexcept { - return WaitAndTryCancelIfNeeded(invocation, engine::Deadline{}, context); -} - -void ProcessFinish(CallState& state, const google::protobuf::Message* final_response) { - const auto& status = state.GetStatus(); - - HandleCallStatistics(state, status); - - RunMiddlewarePipeline(state, FinishHooks(status, final_response)); - - SetStatusAndResetSpan(state, status); -} - -void ProcessFinishAbandoned(CallState& state) noexcept { SetStatusAndResetSpan(state, state.GetStatus()); } - -void ProcessCancelled(CallState& state, std::string_view stage) noexcept { - state.GetStatsScope().OnCancelled(); - state.GetStatsScope().Flush(); - SetErrorAndResetSpan(state, fmt::format("Task cancellation at '{}'", stage)); -} - -void ProcessNetworkError(CallState& state, std::string_view stage) noexcept { - state.GetStatsScope().OnNetworkError(); - state.GetStatsScope().Flush(); - SetErrorAndResetSpan(state, fmt::format("Network error at '{}'", stage)); -} - -void CheckFinishStatus(CallState& state) { - auto& status = state.GetStatus(); - if (!status.ok()) { - ThrowErrorWithStatus(state.GetCallName(), std::move(status)); - } -} - -} // namespace ugrpc::client::impl - -USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/async_stream_methods.cpp b/grpc/src/ugrpc/client/impl/async_stream_methods.cpp index 0ad61c05f600..4e4be99f7247 100644 --- a/grpc/src/ugrpc/client/impl/async_stream_methods.cpp +++ b/grpc/src/ugrpc/client/impl/async_stream_methods.cpp @@ -1,9 +1,38 @@ #include +#include + +#include +#include +#include + USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { +namespace { + +void SetStatusAndResetSpan(CallState& state, const grpc::Status& status) noexcept { + SetStatusForSpan(state.GetSpan(), status); + state.ResetSpan(); +} + +void SetErrorAndResetSpan(CallState& state, std::string_view error_message) noexcept { + SetErrorForSpan(state.GetSpan(), error_message); + state.ResetSpan(); +} + +} // namespace + +ugrpc::impl::AsyncMethodInvocation::WaitStatus +WaitAndTryCancelIfNeeded(ugrpc::impl::AsyncMethodInvocation& invocation, grpc::ClientContext& context) noexcept { + const auto wait_status = invocation.Wait(); + if (ugrpc::impl::AsyncMethodInvocation::WaitStatus::kCancelled == wait_status) { + context.TryCancel(); + } + return wait_status; +} + void CheckOk(StreamingCallState& state, ugrpc::impl::AsyncMethodInvocation::WaitStatus status, std::string_view stage) { if (status == ugrpc::impl::AsyncMethodInvocation::WaitStatus::kError) { state.SetFinished(); @@ -16,6 +45,37 @@ void CheckOk(StreamingCallState& state, ugrpc::impl::AsyncMethodInvocation::Wait } } +void CheckFinishStatus(CallState& state) { + auto& status = state.GetStatus(); + if (!status.ok()) { + ThrowErrorWithStatus(state.GetCallName(), std::move(status)); + } +} + +void ProcessFinish(CallState& state, const google::protobuf::Message* final_response) { + const auto& status = state.GetStatus(); + + HandleCallStatistics(state, status); + + RunMiddlewarePipeline(state, FinishHooks(status, final_response)); + + SetStatusAndResetSpan(state, status); +} + +void ProcessFinishAbandoned(CallState& state) noexcept { SetStatusAndResetSpan(state, state.GetStatus()); } + +void ProcessCancelled(CallState& state, std::string_view stage) noexcept { + state.GetStatsScope().OnCancelled(); + state.GetStatsScope().Flush(); + SetErrorAndResetSpan(state, fmt::format("Task cancellation at '{}'", stage)); +} + +void ProcessNetworkError(CallState& state, std::string_view stage) noexcept { + state.GetStatsScope().OnNetworkError(); + state.GetStatsScope().Flush(); + SetErrorAndResetSpan(state, fmt::format("Network error at '{}'", stage)); +} + } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/call_options_accessor.cpp b/grpc/src/ugrpc/client/impl/call_options_accessor.cpp index 2331d50b1f6d..89fb2621f44b 100644 --- a/grpc/src/ugrpc/client/impl/call_options_accessor.cpp +++ b/grpc/src/ugrpc/client/impl/call_options_accessor.cpp @@ -10,15 +10,15 @@ std::unique_ptr CallOptionsAccessor::CreateClientContext(co auto client_context = call_options.client_context_factory_ ? call_options.client_context_factory_() : std::make_unique(); - for (const auto& [meta_key, meta_value] : call_options.metadata_) { - client_context->AddMetadata(meta_key, meta_value); - } - const auto timeout = call_options.GetTimeout(); if (std::chrono::milliseconds::max() != timeout) { client_context->set_deadline(ugrpc::DurationToTimespec(timeout)); } + for (const auto& [meta_key, meta_value] : call_options.metadata_) { + client_context->AddMetadata(meta_key, meta_value); + } + return client_context; } diff --git a/grpc/src/ugrpc/client/impl/call_params.cpp b/grpc/src/ugrpc/client/impl/call_params.cpp index 35552bd6f890..e7f7fecef372 100644 --- a/grpc/src/ugrpc/client/impl/call_params.cpp +++ b/grpc/src/ugrpc/client/impl/call_params.cpp @@ -5,9 +5,12 @@ #include #include #include +#include +#include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -25,16 +28,58 @@ void CheckValidCallName(std::string_view call_name) { ); } -void ApplyDynamicConfig(CallOptions& call_options, const Qos& qos, const testsuite::GrpcControl& testsuite_grpc) { - // dynamic config has lowest priority, - // so set timeout from dynamic config only if NOT set +void SetAttempts(CallOptions& call_options, const Qos& qos, const RetryConfig& retry_config) { + if (0 == call_options.GetAttempts()) { + call_options.SetAttempts(qos.attempts.value_or(retry_config.attempts)); + } +} + +void SetTimeout(CallOptions& call_options, const Qos& qos, const testsuite::GrpcControl& testsuite_grpc) { if (std::chrono::milliseconds::max() == call_options.GetTimeout() && qos.timeout.has_value()) { - const auto total_timeout = GetTotalTimeout(qos); - UASSERT(total_timeout.has_value()); - call_options.SetTimeout(testsuite_grpc.MakeTimeout(*total_timeout)); + call_options.SetTimeout(testsuite_grpc.MakeTimeout(*qos.timeout)); + } +} + +void SetTimeoutStreaming( + CallOptions& call_options, + const Qos& qos, + const RetryConfig& retry_config, + const testsuite::GrpcControl& testsuite_grpc +) { + SetTimeout(call_options, qos, testsuite_grpc); + + // if timeout is set, reset it to TotalTimeout, because of grpc-core retries + const auto timeout = call_options.GetTimeout(); + if (std::chrono::milliseconds::max() != timeout) { + const auto attempts = qos.attempts.value_or(retry_config.attempts); + UINVARIANT(0 < attempts, "Qos/RetryConfig attempts value must be greater than 0"); + const auto total_timeout = compat::CalculateTotalTimeout(timeout, utils::numeric_cast(attempts)); + call_options.SetTimeout(total_timeout); } } +void ApplyRetryConfiguration( + CallOptions& call_options, + const Qos& qos, + const RetryConfig& retry_config, + const testsuite::GrpcControl& testsuite_grpc +) { + SetAttempts(call_options, qos, retry_config); + SetTimeout(call_options, qos, testsuite_grpc); +} + +void ApplyRetryConfigurationStreaming( + CallOptions& call_options, + const Qos& qos, + const RetryConfig& retry_config, + const testsuite::GrpcControl& testsuite_grpc +) { + // we use grpc-core retries for streaming-methods, + // so CallOption attempts do not work, no need to set them + + SetTimeoutStreaming(call_options, qos, retry_config, testsuite_grpc); +} + } // namespace CallParams CreateCallParams(const ClientData& client_data, std::size_t method_id, CallOptions&& call_options) { @@ -48,7 +93,14 @@ CallParams CreateCallParams(const ClientData& client_data, std::size_t method_id auto stub = client_data.NextStubFromMethodId(method_id); const auto qos = stub.GetClientQos().methods.GetOptional(call_name).value_or(Qos{}); - ApplyDynamicConfig(call_options, qos, client_data.GetTestsuiteControl()); + ApplyRetryConfiguration(call_options, qos, client_data.GetRetryConfig(), client_data.GetTestsuiteControl()); + if (ugrpc::impl::RpcType::kUnary == GetMethodType(metadata, method_id)) { + ApplyRetryConfiguration(call_options, qos, client_data.GetRetryConfig(), client_data.GetTestsuiteControl()); + } else { + ApplyRetryConfigurationStreaming( + call_options, qos, client_data.GetRetryConfig(), client_data.GetTestsuiteControl() + ); + } return CallParams{ client_data.GetClientName(), @@ -59,6 +111,7 @@ CallParams CreateCallParams(const ClientData& client_data, std::size_t method_id std::move(stub), client_data.GetMiddlewares(), client_data.GetStatistics(method_id), + client_data.GetTestsuiteControl(), }; } @@ -88,6 +141,7 @@ CallParams CreateGenericCallParams( client_data.NextStub(), client_data.GetMiddlewares(), client_data.GetGenericStatistics(generic_options.metrics_call_name.value_or(call_name)), + client_data.GetTestsuiteControl(), }; } diff --git a/grpc/src/ugrpc/client/impl/call_state.cpp b/grpc/src/ugrpc/client/impl/call_state.cpp index ae33153e5fe4..4b184800e300 100644 --- a/grpc/src/ugrpc/client/impl/call_state.cpp +++ b/grpc/src/ugrpc/client/impl/call_state.cpp @@ -59,20 +59,28 @@ CallState::CallState(CallParams&& params, CallKind call_kind) queue_(params.queue), config_values_(params.config), middleware_pipeline_(params.middlewares), + testsuite_grpc_(params.testsuite_grpc), call_kind_(call_kind) { UINVARIANT(!client_name_.empty(), "client name should not be empty"); SetupSpan(span_, call_name_.Get()); - - client_context_ = CallOptionsAccessor::CreateClientContext(params.call_options); - AddTracingMetadata(*client_context_, GetSpan()); } StubHandle& CallState::GetStub() noexcept { return stub_; } -const grpc::ClientContext& CallState::GetClientContext() const noexcept { return *client_context_; } +void CallState::SetClientContext(std::unique_ptr client_context) noexcept { + client_context_ = std::move(client_context); +} -grpc::ClientContext& CallState::GetClientContext() noexcept { return *client_context_; } +const grpc::ClientContext& CallState::GetClientContext() const noexcept { + UASSERT(client_context_); + return *client_context_; +} + +grpc::ClientContext& CallState::GetClientContext() noexcept { + UASSERT(client_context_); + return *client_context_; +} grpc::CompletionQueue& CallState::GetQueue() const noexcept { return queue_; } @@ -80,6 +88,8 @@ const RpcConfigValues& CallState::GetConfigValues() const noexcept { return conf const MiddlewarePipeline& CallState::GetMiddlewarePipeline() const noexcept { return middleware_pipeline_; } +const testsuite::GrpcControl& CallState::GetTestsuiteControl() const noexcept { return testsuite_grpc_; } + std::string_view CallState::GetCallName() const noexcept { return call_name_.Get(); } std::string_view CallState::GetClientName() const noexcept { return client_name_; } @@ -107,30 +117,18 @@ bool CallState::IsDeadlinePropagated() const noexcept { return is_deadline_propa grpc::Status& CallState::GetStatus() noexcept { return status_; } -UnaryCallState::~UnaryCallState() noexcept { invocation_.reset(); } - -bool UnaryCallState::IsFinishProcessed() const noexcept { return finish_processed_; } +void CallState::Commit() noexcept { committed_.store(true, std::memory_order_release); } -void UnaryCallState::SetFinishProcessed() noexcept { - UASSERT(!finish_processed_); - finish_processed_ = true; +grpc::ClientContext& CallState::GetClientContextCommitted() { + UINVARIANT(committed_, "Call state should be committed"); + UINVARIANT(client_context_, "GetClientContext should not be called on cancelled RPC"); + return *client_context_; } -bool UnaryCallState::IsStatusExtracted() const noexcept { return status_extracted_; } - -void UnaryCallState::SetStatusExtracted() noexcept { - UASSERT(!status_extracted_); - status_extracted_ = true; -} - -void UnaryCallState::EmplaceFinishAsyncMethodInvocation() { - UASSERT(!invocation_.has_value()); - invocation_.emplace(); -} - -FinishAsyncMethodInvocation& UnaryCallState::GetFinishAsyncMethodInvocation() noexcept { - UASSERT(invocation_.has_value()); - return *invocation_; +StreamingCallState::StreamingCallState(CallParams&& params, CallKind call_kind) + : CallState(std::move(params), call_kind) { + SetupClientContext(*this, params.call_options); + Commit(); } StreamingCallState::~StreamingCallState() noexcept { @@ -196,6 +194,12 @@ bool IsWriteAndCheckAvailable(const StreamingCallState& state) noexcept { return !state.AreWritesFinished() && !state.IsFinished(); } +void SetupClientContext(CallState& state, const CallOptions& call_options) { + auto client_context = CallOptionsAccessor::CreateClientContext(call_options); + AddTracingMetadata(*client_context, state.GetSpan()); + state.SetClientContext(std::move(client_context)); +} + void HandleCallStatistics(CallState& state, const grpc::Status& status) noexcept { auto& stats = state.GetStatsScope(); stats.OnExplicitFinish(status.error_code()); diff --git a/grpc/src/ugrpc/client/impl/channel_argument_utils.cpp b/grpc/src/ugrpc/client/impl/channel_argument_utils.cpp new file mode 100644 index 000000000000..1b18b05575d0 --- /dev/null +++ b/grpc/src/ugrpc/client/impl/channel_argument_utils.cpp @@ -0,0 +1,25 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +grpc::ChannelArguments +BuildChannelArguments(const grpc::ChannelArguments& channel_args, const std::optional& service_config) { + if (!service_config.has_value()) { + return channel_args; + } + + LOG_INFO() << "Building ChannelArguments, ServiceConfig: " << *service_config; + auto effective_channel_args{channel_args}; + effective_channel_args.SetServiceConfigJSON(ugrpc::impl::ToGrpcString(*service_config)); + return effective_channel_args; +} + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/channel_arguments_builder.cpp b/grpc/src/ugrpc/client/impl/channel_arguments_builder.cpp deleted file mode 100644 index 0738f748b8ce..000000000000 --- a/grpc/src/ugrpc/client/impl/channel_arguments_builder.cpp +++ /dev/null @@ -1,263 +0,0 @@ -#include - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -USERVER_NAMESPACE_BEGIN - -namespace ugrpc::client::impl { - -namespace { - -constexpr bool HasValue(const Qos& qos) noexcept { return qos.attempts.has_value(); } - -void SetName(formats::json::ValueBuilder& method_config, formats::json::Value name) { - method_config["name"] = formats::json::MakeArray(std::move(name)); -} - -void ApplyQosConfig(formats::json::ValueBuilder& method_config, const Qos& qos) { - const auto attempts = GetAttempts(qos); - if (attempts.has_value()) { - method_config.Remove("hedgingPolicy"); - - if (1 == *attempts) { - method_config.Remove("retryPolicy"); - return; - } - - if (!method_config.HasMember("retryPolicy")) { - method_config["retryPolicy"] = ConstructDefaultRetryPolicy(); - } - method_config["retryPolicy"]["maxAttempts"] = *attempts; - if (qos.timeout.has_value()) { - method_config["retryPolicy"]["perAttemptRecvTimeout"] = - ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString( - google::protobuf::util::TimeUtil::MillisecondsToDuration(qos.timeout->count()) - )); - } - } -} - -formats::json::Value ApplyQosConfig(formats::json::Value method_config, const Qos& qos) { - formats::json::ValueBuilder method_config_builder{std::move(method_config)}; - ApplyQosConfig(method_config_builder, qos); - return method_config_builder.ExtractValue(); -} - -formats::json::Value Normalize(const formats::json::Value& method_config, formats::json::Value name) { - formats::json::ValueBuilder method_config_builder{method_config}; - SetName(method_config_builder, std::move(name)); - return method_config_builder.ExtractValue(); -} - -formats::json::Value NormalizeDefault(const formats::json::Value& method_config) { - formats::json::ValueBuilder method_config_builder{method_config}; - SetName(method_config_builder, formats::json::MakeObject()); - return method_config_builder.ExtractValue(); -} - -ServiceConfigBuilder::PreparedMethodConfigs PrepareMethodConfigs( - const formats::json::Value& static_service_config, - const ugrpc::impl::StaticServiceMetadata& metadata -) { - std::unordered_map method_configs; - std::optional default_method_config; - - if (static_service_config.HasMember("methodConfig")) { - for (const auto& method_config : static_service_config["methodConfig"]) { - if (method_config.HasMember("name")) { - for (const auto& name : method_config["name"]) { - const auto service_name = name["service"].As(""); - const auto method_name = name["method"].As(""); - - // - If the 'service' field is empty, the 'method' field must be empty, and - // this MethodConfig specifies the default for all methods (it's the default - // config). - if (service_name.empty()) { - UINVARIANT( - method_name.empty(), "If the 'service' field is empty, the 'method' field must be empty" - ); - if (!default_method_config.has_value()) { - default_method_config = NormalizeDefault(method_config); - } - continue; - } - - if (metadata.service_full_name != service_name) { - throw std::runtime_error{ - fmt::format("Invalid MethodConfig: unknown service name {}", service_name)}; - } - - // - If the 'method' field is empty, this MethodConfig specifies the defaults - // for all methods for the specified service. - if (method_name.empty()) { - default_method_config = NormalizeDefault(method_config); - continue; - } - - const auto method_id = FindMethod(metadata, service_name, method_name); - if (!method_id.has_value()) { - throw std::runtime_error{ - fmt::format("Invalid MethodConfig: unknown method name {}", method_name)}; - } - - const auto [_, inserted] = method_configs.try_emplace(*method_id, Normalize(method_config, name)); - UINVARIANT(inserted, "Each name entry must be unique across the entire ServiceConfig"); - } - } - } - } - - return {std::move(method_configs), std::move(default_method_config)}; -} - -formats::json::Value -BuildMethodConfig(formats::json::Value name, const Qos& qos, std::optional static_method_config) { - formats::json::ValueBuilder method_config{std::move(static_method_config).value_or(formats::json::MakeObject())}; - - SetName(method_config, std::move(name)); - - ApplyQosConfig(method_config, qos); - - return method_config.ExtractValue(); -} - -formats::json::Value BuildMethodConfig( - std::string_view service_name, - std::string_view method_name, - const Qos& qos, - std::optional static_method_config -) { - UINVARIANT( - !service_name.empty() || method_name.empty(), - "If the 'service' field is empty, the 'method' field must be empty" - ); - - auto name = formats::json::MakeObject("service", service_name, "method", method_name); - - return BuildMethodConfig(std::move(name), qos, std::move(static_method_config)); -} - -formats::json::Value -BuildDefaultMethodConfig(const Qos& qos, std::optional static_method_config) { - return BuildMethodConfig(formats::json::MakeObject(), qos, std::move(static_method_config)); -} - -} // namespace - -ServiceConfigBuilder::ServiceConfigBuilder( - const ugrpc::impl::StaticServiceMetadata& metadata, - const std::optional& static_service_config -) - : metadata_{metadata} { - if (static_service_config.has_value()) { - static_service_config_ = formats::json::FromString(*static_service_config); - prepared_method_configs_ = PrepareMethodConfigs(static_service_config_, metadata_); - } -} - -formats::json::Value ServiceConfigBuilder::Build(const ClientQos& client_qos) const { - formats::json::ValueBuilder service_config_builder{static_service_config_}; - - if (auto method_config_array = BuildMethodConfigArray(client_qos); !method_config_array.IsEmpty()) { - service_config_builder["methodConfig"] = std::move(method_config_array); - } - - return service_config_builder.ExtractValue(); -} - -formats::json::Value ServiceConfigBuilder::BuildMethodConfigArray(const ClientQos& client_qos) const { - formats::json::ValueBuilder method_config_array; - - const auto qos_default = - client_qos.methods.HasDefaultValue() ? client_qos.methods.GetDefaultValue() : std::optional{}; - - auto default_method_config = prepared_method_configs_.default_method_config; - if (default_method_config.has_value() && qos_default.has_value()) { - default_method_config = ApplyQosConfig(std::move(*default_method_config), *qos_default); - } - - for (std::size_t method_id = 0; method_id < GetMethodsCount(metadata_); ++method_id) { - const auto method_name = GetMethodName(metadata_, method_id); - const auto method_full_name = GetMethodFullName(metadata_, method_id); - - const auto qos = client_qos.methods.GetOptional(method_full_name); - - auto method_config = utils::FindOptional(prepared_method_configs_.method_configs, method_id); - if (method_config.has_value() && qos_default.has_value()) { - method_config = ApplyQosConfig(std::move(*method_config), *qos_default); - } - - // add MethodConfig - // if method Qos with non empty value exists, - // or `static-service-config` has corresponding MethodConfig entry - if (client_qos.methods.HasValue(method_full_name) && HasValue(*qos)) { - method_config_array.PushBack(BuildMethodConfig( - metadata_.service_full_name, - method_name, - *qos, - method_config.has_value() ? std::move(method_config) : default_method_config - )); - } else if (method_config.has_value()) { - method_config_array.PushBack(std::move(*method_config)); - } - } - - // add default MethodConfig if default Qos with non empty value exists - // or `static-service-config` has default MethodConfig - if (qos_default.has_value() && HasValue(*qos_default)) { - method_config_array.PushBack(BuildDefaultMethodConfig(*qos_default, std::move(default_method_config))); - } else if (default_method_config.has_value()) { - method_config_array.PushBack(std::move(*default_method_config)); - } - - return method_config_array.ExtractValue(); -} - -ChannelArgumentsBuilder::ChannelArgumentsBuilder( - const grpc::ChannelArguments& channel_args, - const std::optional& static_service_config, - const ugrpc::impl::StaticServiceMetadata& metadata -) - : channel_args_{channel_args}, service_config_builder_{metadata, static_service_config} {} - -grpc::ChannelArguments ChannelArgumentsBuilder::Build(const ClientQos& client_qos) const { - const auto service_config = service_config_builder_.Build(client_qos); - if (service_config.IsNull()) { - return channel_args_; - } - return BuildChannelArguments(channel_args_, formats::json::ToString(service_config)); -} - -grpc::ChannelArguments -BuildChannelArguments(const grpc::ChannelArguments& channel_args, const std::optional& service_config) { - if (!service_config.has_value()) { - return channel_args; - } - - LOG_INFO() << "Building ChannelArguments, ServiceConfig: " << *service_config; - auto effective_channel_args{channel_args}; - effective_channel_args.SetServiceConfigJSON(ugrpc::impl::ToGrpcString(*service_config)); -#ifdef GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING - effective_channel_args.SetInt(GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING, 1); -#endif // GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING - return effective_channel_args; -} - -} // namespace ugrpc::client::impl - -USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/client_factory_config.cpp b/grpc/src/ugrpc/client/impl/client_factory_config.cpp index 211d30fdc996..4a8930030174 100644 --- a/grpc/src/ugrpc/client/impl/client_factory_config.cpp +++ b/grpc/src/ugrpc/client/impl/client_factory_config.cpp @@ -1,14 +1,10 @@ #include #include -#include +#include #include #include -#include -#include -#include -#include #include USERVER_NAMESPACE_BEGIN @@ -17,16 +13,6 @@ namespace ugrpc::client::impl { namespace { -std::shared_ptr MakeCredentials(const ClientFactoryConfig& config) { - if (config.auth_type == AuthType::kSsl) { - LOG_INFO() << "GRPC client (SSL) initialized..."; - return grpc::SslCredentials(config.ssl_credentials_options); - } else { - LOG_INFO() << "GRPC client (non SSL) initialized..."; - return grpc::InsecureChannelCredentials(); - } -} - grpc::SslCredentialsOptions MakeCredentialsOptions(const yaml_config::YamlConfig& config) { grpc::SslCredentialsOptions result; if (config.IsMissing()) { @@ -78,36 +64,14 @@ ClientFactoryConfig Parse(const yaml_config::YamlConfig& value, formats::parse:: if (config.auth_type == AuthType::kSsl) { config.ssl_credentials_options = MakeCredentialsOptions(value["ssl-credentials-options"]); } + config.retry_config = value["retry-config"].As(); + LOG_INFO() << "RetryConfig: attempts=" << config.retry_config.attempts; config.channel_args = MakeChannelArgs(value["channel-args"]); config.default_service_config = value["default-service-config"].As>(); config.channel_count = value["channel-count"].As(config.channel_count); return config; } -ClientFactorySettings -MakeFactorySettings(ClientFactoryConfig&& config, const storages::secdist::SecdistConfig* secdist) { - std::shared_ptr credentials = MakeCredentials(config); - std::unordered_map> client_credentials; - - if (secdist) { - const auto& tokens = secdist->Get(); - - for (const auto& [client_name, token] : tokens.tokens) { - client_credentials[client_name] = grpc::CompositeChannelCredentials( - credentials, grpc::AccessTokenCredentials(ugrpc::impl::ToGrpcString(token)) - ); - } - } - - return ClientFactorySettings{ - std::move(credentials), - std::move(client_credentials), - std::move(config.channel_args), - std::move(config.default_service_config), - config.channel_count, - }; -} - } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/client_factory_config.hpp b/grpc/src/ugrpc/client/impl/client_factory_config.hpp index 4537de1734b3..8a62a129f8ad 100644 --- a/grpc/src/ugrpc/client/impl/client_factory_config.hpp +++ b/grpc/src/ugrpc/client/impl/client_factory_config.hpp @@ -1,9 +1,11 @@ #pragma once -#include +#include +#include + #include -#include +#include USERVER_NAMESPACE_BEGIN @@ -20,6 +22,9 @@ struct ClientFactoryConfig final { grpc::SslCredentialsOptions ssl_credentials_options{}; + /// Retry configuration for outgoing RPCs + RetryConfig retry_config; + /// Optional grpc-core channel args /// @see https://grpc.github.io/grpc/core/group__grpc__arg__keys.html grpc::ChannelArguments channel_args{}; @@ -35,9 +40,6 @@ struct ClientFactoryConfig final { ClientFactoryConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To); -ClientFactorySettings -MakeFactorySettings(impl::ClientFactoryConfig&& config, const storages::secdist::SecdistConfig* secdist); - } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/compat/channel_arguments_builder.cpp b/grpc/src/ugrpc/client/impl/compat/channel_arguments_builder.cpp new file mode 100644 index 000000000000..93866aa9824d --- /dev/null +++ b/grpc/src/ugrpc/client/impl/compat/channel_arguments_builder.cpp @@ -0,0 +1,343 @@ +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl::compat { + +namespace { + +void SetName(formats::json::ValueBuilder& method_config, formats::json::Value name) { + method_config["name"] = std::move(name); +} + +formats::json::Value Normalize(const formats::json::Value& name, const formats::json::Value& method_config) { + formats::json::ValueBuilder method_config_builder{method_config}; + SetName(method_config_builder, formats::json::MakeArray(name)); + return method_config_builder.ExtractValue(); +} + +ServiceConfigBuilder::PreparedMethodConfigs PrepareMethodConfigs( + const formats::json::Value& static_service_config, + const ugrpc::impl::StaticServiceMetadata& metadata +) { + std::unordered_map method_configs; + std::optional default_method_config; + + if (static_service_config.HasMember("methodConfig")) { + for (const auto& method_config : static_service_config["methodConfig"]) { + if (method_config.HasMember("name")) { + for (const auto& name : method_config["name"]) { + const auto service_name = name["service"].As(""); + const auto method_name = name["method"].As(""); + + // - If the 'service' field is empty, the 'method' field must be empty, and + // this MethodConfig specifies the default for all methods (it's the default + // config). + if (service_name.empty()) { + UINVARIANT( + method_name.empty(), "If the 'service' field is empty, the 'method' field must be empty" + ); + if (!default_method_config.has_value()) { + default_method_config = Normalize(name, method_config); + } + continue; + } + + if (metadata.service_full_name != service_name) { + throw std::runtime_error{ + fmt::format("Invalid MethodConfig: unknown service name {}", service_name)}; + } + + // - If the 'method' field is empty, this MethodConfig specifies the defaults + // for all methods for the specified service. + if (method_name.empty()) { + default_method_config = Normalize(name, method_config); + continue; + } + + const auto method_id = FindMethod(metadata, service_name, method_name); + if (!method_id.has_value()) { + throw std::runtime_error{ + fmt::format("Invalid MethodConfig: unknown method name {}", method_name)}; + } + + const auto [_, inserted] = method_configs.try_emplace(*method_id, Normalize(name, method_config)); + UINVARIANT(inserted, "Each name entry must be unique across the entire ServiceConfig"); + } + } + } + } + + return {std::move(method_configs), std::move(default_method_config)}; +} + +class RetryConfiguration final { +public: + RetryConfiguration( + const ugrpc::impl::StaticServiceMetadata& metadata, + const RetryConfig& retry_config, + const ServiceConfigBuilder::PreparedMethodConfigs& prepared_method_configs, + const ClientQos& client_qos + ) + : metadata_{metadata}, + retry_config_{retry_config}, + prepared_method_configs_{prepared_method_configs}, + client_qos_{client_qos} {} + + bool HasMethodConfiguration(size_t method_id) { + // method Qos exists and has non empty 'attempts' value + const auto method_full_name = GetMethodFullName(metadata_, method_id); + if (client_qos_.methods.HasValue(method_full_name)) { + const auto& qos = client_qos_.methods.Get(method_full_name); + if (qos.attempts.has_value()) { + return true; + } + } + + // or `static-service-config` has corresponding MethodConfig entry + return prepared_method_configs_.method_configs.count(method_id); + } + + bool HasDefaultConfiguration() { return GetDefaultAttempts().has_value(); } + + std::optional GetAttempts(size_t method_id) { + const auto method_full_name = GetMethodFullName(metadata_, method_id); + const auto qos = client_qos_.methods.GetOptional(method_full_name); + if (qos.has_value() && qos->attempts.has_value()) { + return qos->attempts; + } + + return GetDefaultAttempts(); + } + + std::optional GetDefaultAttempts() { + if (client_qos_.methods.HasDefaultValue()) { + const auto qos_default = client_qos_.methods.GetDefaultValue(); + if (qos_default.attempts.has_value()) { + return qos_default.attempts; + } + } + + return GetStaticConfigAttempts(); + } + + std::optional GetMethodConfig(size_t method_id) { + auto method_config = utils::FindOptional(prepared_method_configs_.method_configs, method_id); + return method_config.has_value() ? method_config : GetDefaultMethodConfig(); + } + + std::optional GetDefaultMethodConfig() { + return prepared_method_configs_.default_method_config; + } + +private: + std::optional GetStaticConfigAttempts() const { + // for RetryConfig 'attempts == 1' means stay ServiceConfig AS IS + if (1 == retry_config_.attempts) { + return std::nullopt; + } + + return retry_config_.attempts; + } + + const ugrpc::impl::StaticServiceMetadata& metadata_; + const RetryConfig& retry_config_; + const ServiceConfigBuilder::PreparedMethodConfigs& prepared_method_configs_; + const ClientQos& client_qos_; +}; + +void ClearRetryPolicy(formats::json::ValueBuilder& method_config) { + method_config.Remove("retryPolicy"); + method_config.Remove("hedgingPolicy"); +} + +struct RetryPolicy { + std::uint32_t max_attempts{}; + std::optional per_attempt_recv_timeout; +}; +void SetRetryPolicy(formats::json::ValueBuilder& method_config, const RetryPolicy& retry_policy) { + UINVARIANT(1 < retry_policy.max_attempts, "RetryPolicy MaxAttempts must be greater than 1"); + + // Only one of "retryPolicy" or "hedgingPolicy" may be set + if (method_config.HasMember("hedgingPolicy")) { + method_config.Remove("hedgingPolicy"); + } + + if (!method_config.HasMember("retryPolicy")) { + method_config["retryPolicy"] = ConstructDefaultRetryPolicy(); + } + method_config["retryPolicy"]["maxAttempts"] = retry_policy.max_attempts; + + // do not set "perAttemptRecvTimeout" for now + // https://github.com/grpc/grpc/issues/39935 + // "Client request may hangs forever when set 'perAttemptRecvTimeout'" + // + // if (retry_policy.per_attempt_recv_timeout.has_value()) { + // method_config["retryPolicy"]["perAttemptRecvTimeout"] = + // ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString( + // google::protobuf::util::TimeUtil::MillisecondsToDuration(retry_policy.per_attempt_recv_timeout->count()) + // )); + // } +} + +void ApplyAttempts(formats::json::ValueBuilder& method_config, int attempts) { + UINVARIANT(0 < attempts, "'attempts' value must be greater than 0"); + if (1 == attempts) { + ClearRetryPolicy(method_config); + } else { + SetRetryPolicy(method_config, RetryPolicy{utils::numeric_cast(attempts), std::nullopt}); + } +} + +class MethodConfigBuilder final { +public: + void SetMethodConfig(std::optional method_config) { + method_config_ = std::move(method_config); + } + + void SetName(std::string_view service, std::string_view method) { SetName(service, std::vector{method}); } + + void SetName(std::string_view service, const std::vector& methods) { + formats::json::ValueBuilder name{}; + for (const auto& method : methods) { + UINVARIANT( + !service.empty() || method.empty(), "If the 'service' field is empty, the 'method' field must be empty" + ); + name.PushBack(formats::json::MakeObject("service", service, "method", method)); + } + name_ = name.ExtractValue(); + } + + void SetAttempts(std::optional attempts) { attempts_ = attempts; } + + formats::json::Value Build() && { + formats::json::ValueBuilder method_config_builder{ + std::move(method_config_).value_or(formats::json::MakeObject())}; + + if (name_.has_value()) { + impl::compat::SetName(method_config_builder, std::move(*name_)); + } + + if (attempts_.has_value()) { + ApplyAttempts(method_config_builder, *attempts_); + } + + return method_config_builder.ExtractValue(); + } + +private: + std::optional method_config_; + std::optional name_; + std::optional attempts_; +}; + +} // namespace + +ServiceConfigBuilder::ServiceConfigBuilder( + const ugrpc::impl::StaticServiceMetadata& metadata, + const RetryConfig& retry_config, + const std::optional& static_service_config +) + : metadata_{metadata}, retry_config_{retry_config} { + if (static_service_config.has_value()) { + static_service_config_ = formats::json::FromString(*static_service_config); + prepared_method_configs_ = PrepareMethodConfigs(static_service_config_, metadata_); + } +} + +formats::json::Value ServiceConfigBuilder::Build(const ClientQos& client_qos) const { + formats::json::ValueBuilder service_config_builder{static_service_config_}; + + if (auto method_config_array = BuildMethodConfigArray(client_qos); !method_config_array.IsEmpty()) { + service_config_builder["methodConfig"] = std::move(method_config_array); + } + + return service_config_builder.ExtractValue(); +} + +formats::json::Value ServiceConfigBuilder::BuildMethodConfigArray(const ClientQos& client_qos) const { + formats::json::ValueBuilder method_config_array; + + // push unary method configs AS IS + for (const auto& [method_id, method_config] : prepared_method_configs_.method_configs) { + if (ugrpc::impl::RpcType::kUnary == GetMethodType(metadata_, method_id)) { + method_config_array.PushBack(method_config); + } + } + + RetryConfiguration retry_configuration{metadata_, retry_config_, prepared_method_configs_, client_qos}; + + std::vector default_stream_methods; + + for (std::size_t method_id = 0; method_id < GetMethodsCount(metadata_); ++method_id) { + if (ugrpc::impl::RpcType::kUnary == GetMethodType(metadata_, method_id)) { + continue; + } + + if (retry_configuration.HasMethodConfiguration(method_id)) { + MethodConfigBuilder builder; + builder.SetMethodConfig(retry_configuration.GetMethodConfig(method_id)); + builder.SetName(metadata_.service_full_name, GetMethodName(metadata_, method_id)); + builder.SetAttempts(retry_configuration.GetAttempts(method_id)); + method_config_array.PushBack(std::move(builder).Build()); + continue; + } + + default_stream_methods.push_back(GetMethodName(metadata_, method_id)); + } + + // add default MethodConfig for streaming methods + if (!default_stream_methods.empty() && retry_configuration.HasDefaultConfiguration()) { + MethodConfigBuilder builder; + builder.SetMethodConfig(retry_configuration.GetDefaultMethodConfig()); + builder.SetName(metadata_.service_full_name, default_stream_methods); + builder.SetAttempts(retry_configuration.GetDefaultAttempts()); + method_config_array.PushBack(std::move(builder).Build()); + } + + // push default MethodConfig AS IS + if (prepared_method_configs_.default_method_config.has_value()) { + method_config_array.PushBack(prepared_method_configs_.default_method_config); + } + + return method_config_array.ExtractValue(); +} + +ChannelArgumentsBuilder::ChannelArgumentsBuilder( + const grpc::ChannelArguments& channel_args, + const std::optional& static_service_config, + const RetryConfig& retry_config, + const ugrpc::impl::StaticServiceMetadata& metadata +) + : channel_args_{channel_args}, service_config_builder_{metadata, retry_config, static_service_config} {} + +grpc::ChannelArguments ChannelArgumentsBuilder::Build(const ClientQos& client_qos) const { + const auto service_config = service_config_builder_.Build(client_qos); + if (service_config.IsNull()) { + return channel_args_; + } + return BuildChannelArguments(channel_args_, formats::json::ToString(service_config)); +} + +} // namespace ugrpc::client::impl::compat + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/compat/retry_policy.cpp b/grpc/src/ugrpc/client/impl/compat/retry_policy.cpp new file mode 100644 index 000000000000..f53afc814c96 --- /dev/null +++ b/grpc/src/ugrpc/client/impl/compat/retry_policy.cpp @@ -0,0 +1,72 @@ +#include + +#include +#include + +#include + +#include // for GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl::compat { + +namespace { + +constexpr std::uint32_t kMaxAttempts = 5; + +constexpr std::int64_t kInitialBackoffMs = 10; +constexpr std::int64_t kMaxBackoffMs = 300; + +constexpr float kBackoffMultiplier = 2.0; + +} // namespace + +formats::json::Value ConstructDefaultRetryPolicy() { + formats::json::ValueBuilder retry_policy; + retry_policy["maxAttempts"] = kMaxAttempts; + retry_policy["initialBackoff"] = ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString( + google::protobuf::util::TimeUtil::MillisecondsToDuration(kInitialBackoffMs) + )); + retry_policy["maxBackoff"] = ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString( + google::protobuf::util::TimeUtil::MillisecondsToDuration(kMaxBackoffMs) + )); + retry_policy["backoffMultiplier"] = kBackoffMultiplier; + retry_policy["retryableStatusCodes"] = + formats::json::MakeArray("CANCELLED", "UNKNOWN", "DEADLINE_EXCEEDED", "ABORTED", "INTERNAL", "UNAVAILABLE"); + return retry_policy.ExtractValue(); +} + +std::chrono::milliseconds CalculateTotalTimeout(std::chrono::milliseconds timeout, std::uint32_t attempts) { + // Values greater than 5 are treated as 5 without being considered a validation error. + attempts = std::min(attempts, kMaxAttempts); + + const std::int64_t timeout_ms = timeout.count(); + + std::int64_t total_timeout_ms = timeout_ms; +#ifdef GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING + for (std::uint32_t n = 1; n < attempts; ++n) { + // https://github.com/grpc/proposal/blob/master/A6-client-retries.md + // The initial retry attempt will occur after initialBackoff * random(0.8, 1.2). + // After that, the n-th attempt will occur after min(initialBackoff*backoffMultiplier**(n-1), maxBackoff) * + // random(0.8, 1.2)). + const double backoff = std::min( + kInitialBackoffMs * std::pow(static_cast(kBackoffMultiplier), n - 1), + static_cast(kMaxBackoffMs) + ); + // Jitter of plus or minus 0.2 is applied to the backoff delay + constexpr double kBackoffJitter = 0.2; + total_timeout_ms += timeout_ms + static_cast(std::ceil(backoff * (1 + kBackoffJitter))); + } +#endif // GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING + return std::chrono::milliseconds{total_timeout_ms}; +} + +} // namespace ugrpc::client::impl::compat + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/retry_policy.hpp b/grpc/src/ugrpc/client/impl/compat/retry_policy.hpp similarity index 77% rename from grpc/src/ugrpc/client/impl/retry_policy.hpp rename to grpc/src/ugrpc/client/impl/compat/retry_policy.hpp index 2a9bd496ca88..4b3f12e117e0 100644 --- a/grpc/src/ugrpc/client/impl/retry_policy.hpp +++ b/grpc/src/ugrpc/client/impl/compat/retry_policy.hpp @@ -6,12 +6,12 @@ USERVER_NAMESPACE_BEGIN -namespace ugrpc::client::impl { +namespace ugrpc::client::impl::compat { formats::json::Value ConstructDefaultRetryPolicy(); std::chrono::milliseconds CalculateTotalTimeout(std::chrono::milliseconds timeout, std::uint32_t attempts); -} // namespace ugrpc::client::impl +} // namespace ugrpc::client::impl::compat USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/retry_backoff.cpp b/grpc/src/ugrpc/client/impl/retry_backoff.cpp new file mode 100644 index 000000000000..41339108e328 --- /dev/null +++ b/grpc/src/ugrpc/client/impl/retry_backoff.cpp @@ -0,0 +1,42 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +namespace { + +constexpr std::int64_t kInitialBackoffMs = 10; +constexpr std::int64_t kMaxBackoffMs = 300; + +constexpr float kBackoffMultiplier = 2.0; + +constexpr double kJitter = 0.2; + +} // namespace + +RetryBackoff::RetryBackoff() { Reset(); } + +std::chrono::milliseconds RetryBackoff::NextAttemptDelay() { + if (initial_) { + initial_ = false; + } else { + current_backoff_ms_ *= kBackoffMultiplier; + } + current_backoff_ms_ = std::min(current_backoff_ms_, kMaxBackoffMs); + const double jitter = utils::RandRange(1 - kJitter, 1 + kJitter); + return std::chrono::milliseconds{std::llround(current_backoff_ms_ * jitter)}; +} + +void RetryBackoff::Reset() { + current_backoff_ms_ = kInitialBackoffMs; + initial_ = true; +} + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/retry_policy.cpp b/grpc/src/ugrpc/client/impl/retry_policy.cpp index b424dc48b542..08aec9f14813 100644 --- a/grpc/src/ugrpc/client/impl/retry_policy.cpp +++ b/grpc/src/ugrpc/client/impl/retry_policy.cpp @@ -1,70 +1,25 @@ -#include - -#include -#include - -#include - -#include // for GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING - -#include -#include - -#include +#include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -namespace { - -constexpr std::uint32_t kMaxAttempts = 5; - -constexpr std::int64_t kInitialBackoffMs = 10; -constexpr std::int64_t kMaxBackoffMs = 300; - -constexpr float kBackoffMultiplier = 2.0; - -} // namespace - -formats::json::Value ConstructDefaultRetryPolicy() { - formats::json::ValueBuilder retry_policy; - retry_policy["maxAttempts"] = kMaxAttempts; - retry_policy["initialBackoff"] = ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString( - google::protobuf::util::TimeUtil::MillisecondsToDuration(kInitialBackoffMs) - )); - retry_policy["maxBackoff"] = ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString( - google::protobuf::util::TimeUtil::MillisecondsToDuration(kMaxBackoffMs) - )); - retry_policy["backoffMultiplier"] = kBackoffMultiplier; - retry_policy["retryableStatusCodes"] = formats::json::MakeArray("UNAVAILABLE"); - return retry_policy.ExtractValue(); -} - -std::chrono::milliseconds CalculateTotalTimeout(std::chrono::milliseconds timeout, std::uint32_t attempts) { - // Values greater than 5 are treated as 5 without being considered a validation error. - attempts = std::min(attempts, kMaxAttempts); - - const std::int64_t timeout_ms = timeout.count(); - - std::int64_t total_timeout_ms = timeout_ms; -#ifdef GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING - for (std::uint32_t n = 1; n < attempts; ++n) { - // https://github.com/grpc/proposal/blob/master/A6-client-retries.md - // The initial retry attempt will occur after initialBackoff * random(0.8, 1.2). - // After that, the n-th attempt will occur after min(initialBackoff*backoffMultiplier**(n-1), maxBackoff) * - // random(0.8, 1.2)). - const double backoff = std::min( - kInitialBackoffMs * std::pow(static_cast(kBackoffMultiplier), n - 1), - static_cast(kMaxBackoffMs) - ); - // Jitter of plus or minus 0.2 is applied to the backoff delay - constexpr double kBackoffJitter = 0.2; - total_timeout_ms += timeout_ms + static_cast(std::ceil(backoff * (1 + kBackoffJitter))); +/// [retryable] +bool IsRetryable(grpc::StatusCode status_code) noexcept { + switch (status_code) { + case grpc::StatusCode::CANCELLED: + case grpc::StatusCode::UNKNOWN: + case grpc::StatusCode::DEADLINE_EXCEEDED: + case grpc::StatusCode::ABORTED: + case grpc::StatusCode::INTERNAL: + case grpc::StatusCode::UNAVAILABLE: + return true; + + default: + return false; } -#endif // GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING - return std::chrono::milliseconds{total_timeout_ms}; } +/// [retryable] } // namespace ugrpc::client::impl diff --git a/grpc/src/ugrpc/client/impl/rpc.cpp b/grpc/src/ugrpc/client/impl/rpc.cpp deleted file mode 100644 index adc895149c1d..000000000000 --- a/grpc/src/ugrpc/client/impl/rpc.cpp +++ /dev/null @@ -1,116 +0,0 @@ -#include - -#include - -#include - -USERVER_NAMESPACE_BEGIN - -namespace ugrpc::client::impl { - -UnaryFinishFuture::UnaryFinishFuture(UnaryCallState& state, const google::protobuf::Message* response) noexcept - : state_{state}, response_{response} {} - -UnaryFinishFuture::~UnaryFinishFuture() { Destroy(); } - -void UnaryFinishFuture::Destroy() noexcept try { - if (state_.IsFinishProcessed()) { - return; - } - state_.SetFinishProcessed(); - state_.GetClientContext().TryCancel(); - auto& finish = state_.GetFinishAsyncMethodInvocation(); - - const engine::TaskCancellationBlocker cancel_blocker; - const auto wait_status = finish.Wait(); - - state_.GetStatsScope().SetFinishTime(finish.GetFinishTime()); - - switch (wait_status) { - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kOk: - ProcessFinishAbandoned(state_); - break; - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kError: - ProcessNetworkError(state_, "Finish"); - break; - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kCancelled: - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kDeadline: - utils::AbortWithStacktrace("unreachable"); - } -} catch (const std::exception& ex) { - LOG_WARNING() << "There is a caught exception in 'UnaryFinishFuture::Destroy': " << ex; -} - -bool UnaryFinishFuture::IsReady() const noexcept { - auto& finish = state_.GetFinishAsyncMethodInvocation(); - return finish.IsReady(); -} - -engine::FutureStatus UnaryFinishFuture::WaitUntil(engine::Deadline deadline) const noexcept { - if (state_.IsFinishProcessed()) return engine::FutureStatus::kReady; - - auto& finish = state_.GetFinishAsyncMethodInvocation(); - const auto wait_status = impl::WaitAndTryCancelIfNeeded(finish, deadline, state_.GetClientContext()); - switch (wait_status) { - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kOk: - state_.SetFinishProcessed(); - state_.GetStatsScope().SetFinishTime(finish.GetFinishTime()); - try { - ProcessFinish(state_, response_); - } catch (...) { - exception_ = std::current_exception(); - } - return engine::FutureStatus::kReady; - - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kError: - state_.SetFinishProcessed(); - state_.GetStatsScope().SetFinishTime(finish.GetFinishTime()); - ProcessNetworkError(state_, "Finish"); - exception_ = std::make_exception_ptr(RpcInterruptedError(state_.GetCallName(), "Finish")); - return engine::FutureStatus::kReady; - - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kCancelled: - state_.GetStatsScope().OnCancelled(); - return engine::FutureStatus::kCancelled; - - case ugrpc::impl::AsyncMethodInvocation::WaitStatus::kDeadline: - return engine::FutureStatus::kTimeout; - } - - utils::AbortWithStacktrace("should be unreachable"); -} - -void UnaryFinishFuture::Get() { - UINVARIANT(!state_.IsStatusExtracted(), "'Get' called multiple times on the same future"); - state_.SetStatusExtracted(); - - const auto future_status = WaitUntil(engine::Deadline{}); - - if (future_status == engine::FutureStatus::kCancelled) { - throw RpcCancelledError(state_.GetCallName(), "UnaryFuture::Get"); - } - UASSERT(state_.IsFinishProcessed()); - - if (exception_) { - std::rethrow_exception(std::exchange(exception_, {})); - } - - CheckFinishStatus(state_); -} - -engine::impl::ContextAccessor* UnaryFinishFuture::TryGetContextAccessor() noexcept { - // Unfortunately, we can't require that TryGetContextAccessor is not called - // after future is finished - it doesn't match pattern usage of WaitAny - // Instead we should return nullptr - if (state_.IsStatusExtracted()) { - return nullptr; - } - - // if state exists, then FinishAsyncMethodInvocation also exists - auto& finish = state_.GetFinishAsyncMethodInvocation(); - return finish.TryGetContextAccessor(); -} - -} // namespace ugrpc::client::impl - -USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/middlewares/deadline_propagation/middleware.cpp b/grpc/src/ugrpc/client/middlewares/deadline_propagation/middleware.cpp index c0715e1b0324..b364553d73b6 100644 --- a/grpc/src/ugrpc/client/middlewares/deadline_propagation/middleware.cpp +++ b/grpc/src/ugrpc/client/middlewares/deadline_propagation/middleware.cpp @@ -19,15 +19,17 @@ void AddTimeoutMsToSpan(tracing::Span& span, Duration d) { span.AddTag(tracing::kTimeoutMs, ms.count()); } -void UpdateDeadline(impl::CallState& state) { +void UpdateDeadline(MiddlewareCallContext& context) { + auto& state = context.GetState(utils::impl::InternalTag{}); + // Disable by config if (!state.GetConfigValues().enforce_task_deadline) { return; } - auto& context = state.GetClientContext(); + auto& client_context = context.GetClientContext(); - const auto context_time_left = ugrpc::TimespecToDuration(context.raw_deadline()); + const auto context_time_left = ugrpc::TimespecToDuration(client_context.raw_deadline()); const engine::Deadline task_deadline = USERVER_NAMESPACE::server::request::GetTaskInheritedDeadline(); const auto client_deadline_reachable = (context_time_left != engine::Deadline::Duration::max()); @@ -36,7 +38,7 @@ void UpdateDeadline(impl::CallState& state) { return; } - auto& span = state.GetSpan(); + auto& span = context.GetSpan(); if (!task_deadline.IsReachable() && client_deadline_reachable) { AddTimeoutMsToSpan(span, context_time_left); return; @@ -49,7 +51,7 @@ void UpdateDeadline(impl::CallState& state) { span.AddTag("deadline_updated", true); state.SetDeadlinePropagated(); - context.set_deadline(ugrpc::DurationToTimespec(task_time_left)); + client_context.set_deadline(ugrpc::DurationToTimespec(task_time_left)); AddTimeoutMsToSpan(span, task_time_left); } else { @@ -59,9 +61,7 @@ void UpdateDeadline(impl::CallState& state) { } // namespace -void Middleware::PreStartCall(MiddlewareCallContext& context) const { - UpdateDeadline(context.GetState(utils::impl::InternalTag{})); -} +void Middleware::PreStartCall(MiddlewareCallContext& context) const { UpdateDeadline(context); } } // namespace ugrpc::client::middlewares::deadline_propagation diff --git a/grpc/src/ugrpc/client/middlewares/log/middleware.cpp b/grpc/src/ugrpc/client/middlewares/log/middleware.cpp index c804b7b51f30..3ae3de5c7a88 100644 --- a/grpc/src/ugrpc/client/middlewares/log/middleware.cpp +++ b/grpc/src/ugrpc/client/middlewares/log/middleware.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -17,7 +18,7 @@ std::string GetMessageForLogging(const google::protobuf::Message& message, const if (!logging::ShouldLog(settings.msg_log_level)) { return ""; } - return ugrpc::impl::GetMessageForLogging(message, settings.max_msg_size); + return ugrpc::ToLimitedDebugString(message, settings.max_msg_size); } class SpanLogger { @@ -91,7 +92,7 @@ void Middleware::PostFinish(MiddlewareCallContext& context, const grpc::Status& logger.Log(settings_.msg_log_level, "gRPC response stream finished", logging::LogExtra{}); } } else { - auto error_details = ugrpc::impl::GetErrorDetailsForLogging(status); + auto error_details = ugrpc::ToUnlimitedDebugString(status); logging::LogExtra extra{ {ugrpc::impl::kTypeTag, "error_status"}, {ugrpc::impl::kCodeTag, ugrpc::ToString(status.error_code())}, diff --git a/grpc/src/ugrpc/client/qos.cpp b/grpc/src/ugrpc/client/qos.cpp index f8b05b28156f..245a4364b773 100644 --- a/grpc/src/ugrpc/client/qos.cpp +++ b/grpc/src/ugrpc/client/qos.cpp @@ -10,10 +10,6 @@ #include #include #include -#include -#include - -#include USERVER_NAMESPACE_BEGIN @@ -52,25 +48,6 @@ formats::json::Value Serialize(const Qos& qos, formats::serialize::To GetAttempts(const Qos& qos) { - if (qos.attempts.has_value()) { - UINVARIANT(0 < *qos.attempts, "Qos attempts value must be greater than 0"); - return utils::numeric_cast(*qos.attempts); - } - return std::nullopt; -} - -std::optional GetTotalTimeout(const Qos& qos) { - if (qos.timeout.has_value()) { - const auto attempts = GetAttempts(qos); - if (attempts.has_value()) { - return impl::CalculateTotalTimeout(*qos.timeout, *attempts); - } - return *qos.timeout; - } - return std::nullopt; -} - } // namespace ugrpc::client USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/retry_config.cpp b/grpc/src/ugrpc/client/retry_config.cpp new file mode 100644 index 000000000000..86fbf03ced36 --- /dev/null +++ b/grpc/src/ugrpc/client/retry_config.cpp @@ -0,0 +1,17 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { + +RetryConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To) { + RetryConfig retry_config; + retry_config.attempts = value["attempts"].As(retry_config.attempts); + return retry_config; +} + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/secdist.hpp b/grpc/src/ugrpc/client/secdist.hpp index 1dd3db3e7a1c..b9b5e02330ab 100644 --- a/grpc/src/ugrpc/client/secdist.hpp +++ b/grpc/src/ugrpc/client/secdist.hpp @@ -1,5 +1,10 @@ #pragma once +#include +#include + +#include + USERVER_NAMESPACE_BEGIN namespace ugrpc::client { diff --git a/grpc/src/ugrpc/impl/logging.cpp b/grpc/src/ugrpc/impl/logging.cpp index 60ea723616f5..1ea755627ae9 100644 --- a/grpc/src/ugrpc/impl/logging.cpp +++ b/grpc/src/ugrpc/impl/logging.cpp @@ -1,14 +1,5 @@ #include -#include - -#include - -#include -#include - -#include - USERVER_NAMESPACE_BEGIN namespace ugrpc::impl { @@ -19,28 +10,6 @@ const std::string kComponentTag{"grpc_component"}; const std::string kMessageMarshalledLenTag{"grpc_message_marshalled_len"}; const std::string kTypeTag{"grpc_type"}; -std::string GetMessageForLogging(const google::protobuf::Message& message, std::size_t max_size) { - return ugrpc::impl::ToLimitedDebugString(message, max_size); -} - -std::string GetErrorDetailsForLogging(const grpc::Status& status) { - if (status.ok()) { - return ""; - } - - const auto gstatus = ugrpc::ToGoogleRpcStatus(status); - return gstatus.has_value() - ? fmt::format( - "code: {}, error message: {}\nerror details:\n{}", - ugrpc::ToString(status.error_code()), - status.error_message(), - ugrpc::GetGStatusLimitedMessage(*gstatus) - ) - : fmt::format( - "code: {}, error message: {}", ugrpc::ToString(status.error_code()), status.error_message() - ); -} - } // namespace ugrpc::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/impl/logging.hpp b/grpc/src/ugrpc/impl/logging.hpp index 7803afea4f54..1e7ab449d530 100644 --- a/grpc/src/ugrpc/impl/logging.hpp +++ b/grpc/src/ugrpc/impl/logging.hpp @@ -1,10 +1,6 @@ #pragma once - #include -#include -#include - USERVER_NAMESPACE_BEGIN namespace ugrpc::impl { @@ -15,10 +11,6 @@ extern const std::string kComponentTag; extern const std::string kMessageMarshalledLenTag; extern const std::string kTypeTag; -std::string GetMessageForLogging(const google::protobuf::Message& message, std::size_t max_size); - -std::string GetErrorDetailsForLogging(const grpc::Status& status); - } // namespace ugrpc::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/impl/protobuf_utils.hpp b/grpc/src/ugrpc/impl/protobuf_utils.hpp deleted file mode 100644 index 233e73644db5..000000000000 --- a/grpc/src/ugrpc/impl/protobuf_utils.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include - -USERVER_NAMESPACE_BEGIN - -namespace ugrpc::impl { - -std::string ToLimitedDebugString(const google::protobuf::Message& message, std::size_t limit); - -} // namespace ugrpc::impl - -USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/impl/statistics.cpp b/grpc/src/ugrpc/impl/statistics.cpp index 6aca768c1c49..a9f1d719fe99 100644 --- a/grpc/src/ugrpc/impl/statistics.cpp +++ b/grpc/src/ugrpc/impl/statistics.cpp @@ -209,10 +209,12 @@ void ServiceStatistics::DumpAndCountTotal( std::optional client_name, MethodStatisticsSnapshot& total ) const { - for (const auto& [i, method_full_name] : utils::enumerate(metadata_.method_full_names)) { + for (const auto& [i, method_descriptor] : utils::enumerate(metadata_.methods)) { const MethodStatisticsSnapshot snapshot{method_statistics_[i]}; total.Add(snapshot); - DumpMetricWithLabels(writer, snapshot, client_name, method_full_name, metadata_.service_full_name); + DumpMetricWithLabels( + writer, snapshot, client_name, method_descriptor.method_full_name, metadata_.service_full_name + ); } } diff --git a/grpc/src/ugrpc/proto_json.cpp b/grpc/src/ugrpc/proto_json.cpp index 583d42da9015..6da37724fe40 100644 --- a/grpc/src/ugrpc/proto_json.cpp +++ b/grpc/src/ugrpc/proto_json.cpp @@ -82,74 +82,74 @@ class ResultStackFrame final { : ResultStackFrame(ParseType(iter), iter->GetSize(), GetName(iter, previous_type)) {} void SetStructField(std::string_view field_name, google::protobuf::Value&& field) { - UINVARIANT(type == Type::kStruct, "invalid type"); + UINVARIANT(type_ == Type::kStruct, "invalid type"); #if GOOGLE_PROTOBUF_VERSION >= 4022000 - (*value.mutable_struct_value()->mutable_fields())[field_name] = std::move(field); + (*value_.mutable_struct_value()->mutable_fields())[field_name] = std::move(field); #else // No transparent comparisons till // https://github.com/protocolbuffers/protobuf/commit/38d6de1eef8163342084fe - (*value.mutable_struct_value()->mutable_fields())[std::string{field_name}] = std::move(field); + (*value_.mutable_struct_value()->mutable_fields())[std::string{field_name}] = std::move(field); #endif - --elements_await; + --elements_await_; } void AddListElement(google::protobuf::Value&& field) { - UINVARIANT(type == Type::kArray, "invalid type"); - *(value.mutable_list_value()->mutable_values()->Add()) = std::move(field); - --elements_await; + UINVARIANT(type_ == Type::kArray, "invalid type"); + *(value_.mutable_list_value()->mutable_values()->Add()) = std::move(field); + --elements_await_; } - bool IsStruct() const { return type == Type::kStruct; } + bool IsStruct() const { return type_ == Type::kStruct; } - bool IsArray() const { return type == Type::kArray; } + bool IsArray() const { return type_ == Type::kArray; } - Type GetType() const { return type; } + Type GetType() const { return type_; } - bool AwaitElements() const { return elements_await != 0; } + bool AwaitElements() const { return elements_await_ != 0; } - std::string_view GetOuterFieldName() const { return outer_field_name; } + std::string_view GetOuterFieldName() const { return outer_field_name_; } - google::protobuf::Value GetValue() { return google::protobuf::Value(std::move(value)); } + google::protobuf::Value GetValue() { return google::protobuf::Value(std::move(value_)); } private: ResultStackFrame(const Type type, std::size_t elements_await, std::string&& outer_field_name) - : type(type), elements_await(elements_await), outer_field_name(outer_field_name) { + : type_(type), elements_await_(elements_await), outer_field_name_(outer_field_name) { if (type == Type::kStruct) { - value.mutable_struct_value(); + value_.mutable_struct_value(); } else { - value.mutable_list_value(); + value_.mutable_list_value(); } } private: - Type type; - std::size_t elements_await; - std::string outer_field_name; - google::protobuf::Value value{}; + Type type_; + std::size_t elements_await_; + std::string outer_field_name_; + google::protobuf::Value value_{}; }; class StackFrame { public: using Iterator = formats::json::Value::const_iterator; - StackFrame(Iterator begin, Iterator end) : cur(begin), end(end) {} + StackFrame(Iterator begin, Iterator end) : cur_(begin), end_(end) {} - bool IsTrivial() const { return !(cur->IsObject() || cur->IsArray()); } + bool IsTrivial() const { return !(cur_->IsObject() || cur_->IsArray()); } - std::size_t GetSize() const { return cur->GetSize(); } + std::size_t GetSize() const { return cur_->GetSize(); } - Iterator GetIter() const { return cur; } + Iterator GetIter() const { return cur_; } - std::string GetName() const { return cur.GetName(); } + std::string GetName() const { return cur_.GetName(); } - void Advance() { ++cur; } + void Advance() { ++cur_; } - bool IsValid() const { return cur != end; } + bool IsValid() const { return cur_ != end_; } private: - Iterator cur; - Iterator end; + Iterator cur_; + Iterator end_; }; static constexpr std::size_t kInitialStackDepth = 32; diff --git a/grpc/src/ugrpc/impl/protobuf_utils.cpp b/grpc/src/ugrpc/protobuf_logging.cpp similarity index 74% rename from grpc/src/ugrpc/impl/protobuf_utils.cpp rename to grpc/src/ugrpc/protobuf_logging.cpp index daf8679272bc..4c1430da1e2c 100644 --- a/grpc/src/ugrpc/impl/protobuf_utils.cpp +++ b/grpc/src/ugrpc/protobuf_logging.cpp @@ -1,16 +1,12 @@ -#include - -#include -#include -#include +#include #include -#include #include -#include #include #include +#include +#include #include #include @@ -97,11 +93,11 @@ class DebugRedactFieldValuePrinter final : public google::protobuf::TextFormat:: const google::protobuf::Message& /*message*/, int /*fieldIndex*/, int /*fieldCount*/, - bool singleLineMode, + bool single_line_mode, BaseTextGenerator* generator ) const override { PrintRedacted(generator); - if (singleLineMode) { + if (single_line_mode) { generator->PrintLiteral(" "); } else { generator->PrintLiteral("\n"); @@ -189,31 +185,89 @@ compiler::ThreadLocal kDebugStringPrinter = [] { return DebugStringPrinter{}; }; } // namespace +void Print(const google::protobuf::Message& message, google::protobuf::io::ZeroCopyOutputStream& output_stream) { + auto printer = kDebugStringPrinter.Use(); + printer->RegisterDebugRedactPrinters(*message.GetDescriptor()); + printer->Print(message, output_stream); +} + +} // namespace ugrpc::impl + +namespace ugrpc { + std::string ToLimitedDebugString(const google::protobuf::Message& message, std::size_t limit) { boost::container::small_vector output_buffer{limit, boost::container::default_init}; google::protobuf::io::ArrayOutputStream output_stream{output_buffer.data(), utils::numeric_cast(limit)}; - auto printer = kDebugStringPrinter.Use(); + auto printer = impl::kDebugStringPrinter.Use(); printer->RegisterDebugRedactPrinters(*message.GetDescriptor()); #if defined(ARCADIA_ROOT) || GOOGLE_PROTOBUF_VERSION >= 6031002 // Throw `LimitReachedException` on limit reached to stop printing immediately, otherwise TextFormat will continue // to walk the whole message and apply noop printing. - LimitingOutputStream limiting_output_stream{output_stream}; + impl::LimitingOutputStream limiting_output_stream{output_stream}; try { - printer->Print(message, limiting_output_stream); - } catch (const LimitingOutputStream::LimitReachedException& /*ex*/) { + impl::Print(message, limiting_output_stream); + } catch (const impl::LimitingOutputStream::LimitReachedException& /*ex*/) { // Buffer limit has been reached. } #else // For old protobuf, we cannot apply hard limits when printing messages, because its TextFormat is not // exception-safe. https://github.com/protocolbuffers/protobuf/commit/be875d0aaf37dbe6948717ea621278e75e89c9c7 - printer->Print(message, output_stream); + impl::Print(message, output_stream); #endif + std::string returned_str = std::string{output_buffer.data(), static_cast(output_stream.ByteCount())}; + if (returned_str.empty() && limit >= 7) return ""; + return returned_str; +} - return std::string{output_buffer.data(), static_cast(output_stream.ByteCount())}; +std::string ToUnlimitedDebugString(const google::protobuf::Message& message) { + grpc::string result; + google::protobuf::io::StringOutputStream output_stream(&result); + impl::Print(message, output_stream); + std::string returned_str = std::string(result); + if (returned_str.empty()) return ""; + return returned_str; } -} // namespace ugrpc::impl +std::string ToLimitedDebugString(const grpc::Status& status, std::size_t max_size) { + if (status.ok()) { + return "OK"; + } + + const auto gstatus = ugrpc::ToGoogleRpcStatus(status); + if (gstatus.has_value()) { + const std::string details_string = ugrpc::ToLimitedDebugString(*gstatus, max_size); + return fmt::format( + "code: {}, error message: {}\nerror details:\n{}", + ugrpc::ToString(status.error_code()), + status.error_message(), + details_string + ); + } else { + return fmt::format("code: {}, error message: {}", ugrpc::ToString(status.error_code()), status.error_message()); + } +} + +std::string ToUnlimitedDebugString(const grpc::Status& status) { + if (status.ok()) { + return "OK"; + } + + const auto gstatus = ugrpc::ToGoogleRpcStatus(status); + if (gstatus.has_value()) { + const std::string details_string = ugrpc::ToUnlimitedDebugString(*gstatus); + return fmt::format( + "code: {}, error message: {}\nerror details:\n{}", + ugrpc::ToString(status.error_code()), + status.error_message(), + details_string + ); + } else { + return fmt::format("code: {}, error message: {}", ugrpc::ToString(status.error_code()), status.error_message()); + } +} + +} // namespace ugrpc USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/protobuf_visit.cpp b/grpc/src/ugrpc/protobuf_visit.cpp index 1bc10c8999db..daab1985cf58 100644 --- a/grpc/src/ugrpc/protobuf_visit.cpp +++ b/grpc/src/ugrpc/protobuf_visit.cpp @@ -7,9 +7,8 @@ #include #include -#include #include -#include +#include USERVER_NAMESPACE_BEGIN diff --git a/grpc/src/ugrpc/server/impl/call_processor.cpp b/grpc/src/ugrpc/server/impl/call_processor.cpp index e1a19a2cf860..1a7aab73c5d6 100644 --- a/grpc/src/ugrpc/server/impl/call_processor.cpp +++ b/grpc/src/ugrpc/server/impl/call_processor.cpp @@ -2,6 +2,8 @@ #include +#include + #include #include #include @@ -20,20 +22,21 @@ namespace ugrpc::server::impl { namespace { -void ReportFinishSuccess( - const grpc::Status& status, - tracing::Span& span, - ugrpc::impl::RpcStatisticsScope& statistics_scope -) noexcept { +void ReportFinishSuccess(const grpc::Status& status, CallState& state) noexcept { try { const auto status_code = status.error_code(); - statistics_scope.OnExplicitFinish(status_code); + state.statistics_scope.OnExplicitFinish(status_code); + auto& span = state.GetSpan(); span.AddNonInheritableTag("grpc_code", ugrpc::ToString(status_code)); if (!status.ok()) { span.AddNonInheritableTag(tracing::kErrorFlag, true); span.AddNonInheritableTag(tracing::kErrorMessage, status.error_message()); - span.SetLogLevel(IsServerError(status_code) ? logging::Level::kError : logging::Level::kWarning); + const auto default_error_log_level = + IsServerError(status.error_code()) ? logging::Level::kError : logging::Level::kWarning; + const auto error_log_level = + utils::FindOrDefault(state.status_codes_log_level, status.error_code(), default_error_log_level); + span.SetLogLevel(error_log_level); } } catch (const std::exception& ex) { LOG_ERROR() << "Error in ReportFinishSuccess: " << ex; @@ -143,7 +146,7 @@ ReportCustomError(const USERVER_NAMESPACE::server::handlers::CustomHandlerExcept void CheckFinishStatus(bool finish_op_succeeded, const grpc::Status& status, CallState& state) noexcept { if (finish_op_succeeded) { - ReportFinishSuccess(status, state.GetSpan(), state.statistics_scope); + ReportFinishSuccess(status, state); } else { ReportRpcInterruptedError(state); } diff --git a/grpc/src/ugrpc/server/impl/format_log_message.cpp b/grpc/src/ugrpc/server/impl/format_log_message.cpp index 3d7aec140bfb..c0927aa730c2 100644 --- a/grpc/src/ugrpc/server/impl/format_log_message.cpp +++ b/grpc/src/ugrpc/server/impl/format_log_message.cpp @@ -5,9 +5,11 @@ #include #include +#include #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -61,17 +63,26 @@ std::string_view GetCurrentTimeString(std::chrono::system_clock::time_point star ); cache->cached_time = rounded_now; } + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-stack-address" +#endif return std::string_view{cache->cached_time_string, kTimeTemplate.size()}; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif } } // namespace -std::string FormatLogMessage( +logging::impl::LogExtraTskvFormatter FormatLogMessage( const std::multimap& metadata, std::string_view peer, std::chrono::system_clock::time_point start_time, std::string_view call_name, - grpc::StatusCode code + grpc::StatusCode code, + const logging::LogExtra* log_extra ) { static const auto timezone = utils::datetime::LocalTimezoneTimestring(start_time, "%z"); @@ -88,9 +99,10 @@ std::string FormatLogMessage( const auto request_time_seconds = std::chrono::duration_cast(request_time); const auto request_time_milliseconds = request_time - request_time_seconds; - // FMT_COMPILE makes it slower - return fmt::format( - "tskv" + logging::impl::LogExtraTskvFormatter formatter(logging::Format::kRaw); + + fmt::format_to( + std::back_inserter(formatter.GetTextLogItem().log_line), "\ttimestamp={}" "\ttimezone={}" "\tuser_agent={}" @@ -100,16 +112,13 @@ std::string FormatLogMessage( "\trequest_time={}.{:0>3}" "\tupstream_response_time={}.{:0>3}" "\tgrpc_status={}" - "\tgrpc_status_code={}\n", + "\tgrpc_status_code={}", GetCurrentTimeString(start_time), timezone, EscapeForAccessTskvLog(user_agent), ip, ip, EscapeForAccessTskvLog(call_name), - // request_time should represent the time from the first byte received to the last byte sent. - // We are currently being inexact here by not including the time for initial request deserialization - // and the time for the final response serialization. request_time_seconds.count(), request_time_milliseconds.count(), // TODO remove, this is for safe migration from old access log parsers. @@ -118,6 +127,12 @@ std::string FormatLogMessage( static_cast(code), ToString(code) ); + + if (log_extra) { + formatter.Append(*log_extra); + } + + return formatter; } } // namespace ugrpc::server::impl diff --git a/grpc/src/ugrpc/server/impl/format_log_message.hpp b/grpc/src/ugrpc/server/impl/format_log_message.hpp index 17dea33bf59d..015b92dde67e 100644 --- a/grpc/src/ugrpc/server/impl/format_log_message.hpp +++ b/grpc/src/ugrpc/server/impl/format_log_message.hpp @@ -4,18 +4,22 @@ #include #include +#include +#include +#include #include USERVER_NAMESPACE_BEGIN namespace ugrpc::server::impl { -std::string FormatLogMessage( +logging::impl::LogExtraTskvFormatter FormatLogMessage( const std::multimap& metadata, std::string_view peer, std::chrono::system_clock::time_point start_time, std::string_view call_name, - grpc::StatusCode code + grpc::StatusCode code, + const logging::LogExtra* log_extra ); } // namespace ugrpc::server::impl diff --git a/grpc/src/ugrpc/server/impl/generic_service_worker.cpp b/grpc/src/ugrpc/server/impl/generic_service_worker.cpp index 47287e0a865f..1c5a34dcb870 100644 --- a/grpc/src/ugrpc/server/impl/generic_service_worker.cpp +++ b/grpc/src/ugrpc/server/impl/generic_service_worker.cpp @@ -12,11 +12,14 @@ namespace ugrpc::server::impl { namespace { -constexpr std::string_view kGenericMethodFullNamesFake[] = { - "Generic/Generic", -}; -constexpr std::string_view kGenericServiceNameFake = "Generic"; -constexpr ugrpc::impl::StaticServiceMetadata kGenericMetadataFake{kGenericServiceNameFake, kGenericMethodFullNamesFake}; +constexpr std::string_view kGenericServiceFullNameFake = "Generic"; + +constexpr std::array kGenericMethodsFake = {ugrpc::impl::MethodDescriptor{ + /*method_full_name*/ "Generic/Generic", + /*method_type*/ ugrpc::impl::RpcType::kBidiStreaming, +}}; + +constexpr ugrpc::impl::StaticServiceMetadata kGenericMetadataFake{kGenericServiceFullNameFake, kGenericMethodsFake}; } // namespace diff --git a/grpc/src/ugrpc/server/impl/parse_config.cpp b/grpc/src/ugrpc/server/impl/parse_config.cpp index 8ca976fc293d..b442905d2df8 100644 --- a/grpc/src/ugrpc/server/impl/parse_config.cpp +++ b/grpc/src/ugrpc/server/impl/parse_config.cpp @@ -1,7 +1,9 @@ #include +#include #include +#include #include #include #include @@ -11,6 +13,7 @@ #include #include +#include USERVER_NAMESPACE_BEGIN @@ -19,6 +22,7 @@ namespace ugrpc::server::impl { namespace { constexpr std::string_view kTaskProcessorKey = "task-processor"; +constexpr std::string_view kStatusCodesLogLevelKey = "status-codes-log-level"; template auto ParseOptional( @@ -71,6 +75,8 @@ server::ServiceConfig ParseServiceConfig( value[kTaskProcessorKey], defaults.task_processor, context, ParseTaskProcessor ), /*middlewares=*/{}, + /*status_codes_log_level=*/ + value[kStatusCodesLogLevelKey].As>({}), }; } diff --git a/grpc/src/ugrpc/server/middlewares/access_log/log_extra.cpp b/grpc/src/ugrpc/server/middlewares/access_log/log_extra.cpp new file mode 100644 index 000000000000..5c13942d55dc --- /dev/null +++ b/grpc/src/ugrpc/server/middlewares/access_log/log_extra.cpp @@ -0,0 +1,21 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server::middlewares::access_log { + +void SetAdditionalLogKeys(MiddlewareCallContext& context, logging::LogExtra&& log_extra) { + auto* extra = context.GetStorageContext().GetOptional(kLogExtraTag); + + if (extra) { + extra->Extend(std::move(log_extra)); + } else { + context.GetStorageContext().Emplace(kLogExtraTag, std::move(log_extra)); + } +} + +} // namespace ugrpc::server::middlewares::access_log + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/middlewares/access_log/middleware.cpp b/grpc/src/ugrpc/server/middlewares/access_log/middleware.cpp index b339ddcb3805..a202cfa608ed 100644 --- a/grpc/src/ugrpc/server/middlewares/access_log/middleware.cpp +++ b/grpc/src/ugrpc/server/middlewares/access_log/middleware.cpp @@ -21,14 +21,20 @@ void WriteAccessLog( constexpr auto kLevel = logging::Level::kInfo; if (access_tskv_logger.ShouldLog(kLevel)) { - logging::impl::TextLogItem log_item{impl::FormatLogMessage( - server_context.client_metadata(), - server_context.peer(), - context.GetSpan().GetStartSystemTime(), - context.GetCallName(), - status.error_code() - )}; - access_tskv_logger.Log(kLevel, log_item); + const auto* log_extra = context.GetStorageContext().GetOptional(kLogExtraTag); + + access_tskv_logger.Log( + kLevel, + impl::FormatLogMessage( + server_context.client_metadata(), + server_context.peer(), + context.GetSpan().GetStartSystemTime(), + context.GetCallName(), + status.error_code(), + log_extra + ) + .ExtractTextLogItem() + ); } } catch (const std::exception& ex) { LOG_ERROR() << "Error in WriteAccessLog: " << ex; diff --git a/grpc/src/ugrpc/server/middlewares/access_log/middleware.hpp b/grpc/src/ugrpc/server/middlewares/access_log/middleware.hpp index 1647e694faa7..760be090158b 100644 --- a/grpc/src/ugrpc/server/middlewares/access_log/middleware.hpp +++ b/grpc/src/ugrpc/server/middlewares/access_log/middleware.hpp @@ -2,6 +2,7 @@ #include +#include #include USERVER_NAMESPACE_BEGIN diff --git a/grpc/src/ugrpc/server/middlewares/log/component.cpp b/grpc/src/ugrpc/server/middlewares/log/component.cpp index 2dfc9cc63c21..baf2851a0204 100644 --- a/grpc/src/ugrpc/server/middlewares/log/component.cpp +++ b/grpc/src/ugrpc/server/middlewares/log/component.cpp @@ -1,12 +1,14 @@ #include #include +#include #include #include #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -18,6 +20,8 @@ Settings Parse(const yaml_config::YamlConfig& config, formats::parse::To(settings.msg_log_level); settings.max_msg_size = config["msg-size-log-limit"].As(settings.max_msg_size); settings.local_log_level = config["local-log-level"].As(settings.local_log_level); + settings.status_codes_log_level = + config["status-codes-log-level"].As>({}); return settings; } @@ -57,6 +61,21 @@ additionalProperties: false local-log-level: type: string description: local log level of the span for user-provided handler + status-codes-log-level: + type: object + properties: {} + additionalProperties: + type: string + description: log level + description: | + gRPC status code string -> log level map. + Allows overriding the log level for the response for individual statuses. + See grpcpp StatusCode documentation for the list of possible values: + https://grpc.github.io/grpc/cpp/namespacegrpc.html#aff1730578c90160528f6a8d67ef5c43b + + Example: + - FAILED_PRECONDITION: info + )"); } diff --git a/grpc/src/ugrpc/server/middlewares/log/middleware.cpp b/grpc/src/ugrpc/server/middlewares/log/middleware.cpp index ae510ce1156c..95c2c967bbed 100644 --- a/grpc/src/ugrpc/server/middlewares/log/middleware.cpp +++ b/grpc/src/ugrpc/server/middlewares/log/middleware.cpp @@ -2,8 +2,8 @@ #include #include - -#include +#include +#include #include @@ -17,7 +17,7 @@ std::string GetMessageForLogging(const google::protobuf::Message& message, const if (!logging::ShouldLog(settings.msg_log_level)) { return ""; } - return ugrpc::impl::GetMessageForLogging(message, settings.max_msg_size); + return ugrpc::ToLimitedDebugString(message, settings.max_msg_size); } class Logger { @@ -93,15 +93,17 @@ void Middleware::OnCallFinish(MiddlewareCallContext& context, const grpc::Status ); } } else { - auto error_details = ugrpc::impl::GetErrorDetailsForLogging(status); + auto error_details = ugrpc::ToLimitedDebugString(status, settings_.max_msg_size); logging::LogExtra extra{ {"type", "response"}, {ugrpc::impl::kCodeTag, ugrpc::ToString(status.error_code())}, {ugrpc::impl::kTypeTag, "error_status"}, {ugrpc::impl::kBodyTag, std::move(error_details)}, }; - const auto error_log_level = + const auto default_error_log_level = IsServerError(status.error_code()) ? logging::Level::kError : logging::Level::kWarning; + const auto error_log_level = + utils::FindOrDefault(settings_.status_codes_log_level, status.error_code(), default_error_log_level); logger.Log(error_log_level, "gRPC error", std::move(extra)); } } diff --git a/grpc/src/ugrpc/server/middlewares/log/middleware.hpp b/grpc/src/ugrpc/server/middlewares/log/middleware.hpp index 7d8ad7013cc2..a9284a2660ef 100644 --- a/grpc/src/ugrpc/server/middlewares/log/middleware.hpp +++ b/grpc/src/ugrpc/server/middlewares/log/middleware.hpp @@ -2,9 +2,12 @@ #include +#include + #include #include +#include USERVER_NAMESPACE_BEGIN @@ -35,6 +38,10 @@ struct Settings final { /// It applies to logs in user-provided handler /// @ref tracing::Span::SetLocalLogLevel logging::Level local_log_level{logging::Level::kDebug}; + + /// map of "status_code": log_level items to override span log level for specific status codes + /// see @ref ugrpc::kStatusCodesMap for available statuses + boost::container::flat_map status_codes_log_level; }; class Middleware final : public MiddlewareBase { diff --git a/grpc/src/ugrpc/server/server.cpp b/grpc/src/ugrpc/server/server.cpp index c580bad28444..508202c22a40 100644 --- a/grpc/src/ugrpc/server/server.cpp +++ b/grpc/src/ugrpc/server/server.cpp @@ -198,6 +198,7 @@ impl::ServiceInternals Server::Impl::MakeServiceInternals(ServiceConfig&& config statistics_storage_, std::move(config.middlewares), config_source_, + std::move(config.status_codes_log_level), }; } diff --git a/grpc/src/ugrpc/server/service_component_base.cpp b/grpc/src/ugrpc/server/service_component_base.cpp index 8cece85ba3ba..8a59d9284dac 100644 --- a/grpc/src/ugrpc/server/service_component_base.cpp +++ b/grpc/src/ugrpc/server/service_component_base.cpp @@ -52,6 +52,20 @@ additionalProperties: false type: string description: the task processor to use for responses defaultDescription: uses grpc-server.service-defaults.task-processor + status-codes-log-level: + type: object + properties: {} + additionalProperties: + type: string + description: log level + description: | + gRPC status code string -> log level map. + Allows overriding the log level for the span for individual statuses. + See grpcpp StatusCode documentation for the list of possible values: + https://grpc.github.io/grpc/cpp/namespacegrpc.html#aff1730578c90160528f6a8d67ef5c43b + + Example: + - FAILED_PRECONDITION: info )"); } diff --git a/grpc/src/ugrpc/status_codes.cpp b/grpc/src/ugrpc/status_codes.cpp index b7caf9deeefd..021e77874251 100644 --- a/grpc/src/ugrpc/status_codes.cpp +++ b/grpc/src/ugrpc/status_codes.cpp @@ -2,9 +2,12 @@ #include +#include +#include #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -72,4 +75,30 @@ bool IsServerError(grpc::StatusCode status) noexcept { } // namespace ugrpc +namespace formats::parse { + +grpc::StatusCode Parse(const yaml_config::YamlConfig& value, To) { + return utils::ParseFromValueString(value, ugrpc::kStatusCodesMap); +} + +grpc::StatusCode Parse(std::string_view value, To) { + const auto result = ugrpc::kStatusCodesMap.TryFind(value); + if (!result) { + throw std::runtime_error(fmt::format( + "Invalid value of grpc::StatusCode: '{}' is not one of {}", value, ugrpc::kStatusCodesMap.DescribeSecond() + )); + } + return *result; +} + +} // namespace formats::parse + +namespace formats::serialize { + +formats::json::Value Serialize(const grpc::StatusCode& value, formats::serialize::To) { + return formats::json::ValueBuilder(ugrpc::ToString(value)).ExtractValue(); +} + +} // namespace formats::serialize + USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/status_utils.cpp b/grpc/src/ugrpc/status_utils.cpp index 2d05370ccd0e..3eef83b2786e 100644 --- a/grpc/src/ugrpc/status_utils.cpp +++ b/grpc/src/ugrpc/status_utils.cpp @@ -1,6 +1,6 @@ #include -#include +#include USERVER_NAMESPACE_BEGIN @@ -27,7 +27,7 @@ std::optional ToGoogleRpcStatus(const grpc::Status& status) std::string GetGStatusLimitedMessage(const google::rpc::Status& status) { constexpr std::size_t kLengthLimit = 1024; - return impl::ToLimitedDebugString(status, kLengthLimit); + return ToLimitedDebugString(status, kLengthLimit); } } // namespace ugrpc diff --git a/grpc/src/ugrpc/tests/service.cpp b/grpc/src/ugrpc/tests/service.cpp index c63975d07b0c..5475b5b660a2 100644 --- a/grpc/src/ugrpc/tests/service.cpp +++ b/grpc/src/ugrpc/tests/service.cpp @@ -30,10 +30,7 @@ void ServiceBase::RegisterService(server::GenericServiceBase& service) { server::ServiceConfig ServiceBase::MakeServiceConfig() { middlewares_change_allowed_ = false; - return server::ServiceConfig{ - engine::current_task::GetTaskProcessor(), - server_middlewares_, - }; + return server::ServiceConfig{engine::current_task::GetTaskProcessor(), server_middlewares_, {}}; } void ServiceBase::StartServer(client::ClientFactorySettings&& client_factory_settings) { diff --git a/grpc/tests/channel_arguments_builder_test.cpp b/grpc/tests/channel_arguments_builder_test.cpp deleted file mode 100644 index d270d4bedcca..000000000000 --- a/grpc/tests/channel_arguments_builder_test.cpp +++ /dev/null @@ -1,462 +0,0 @@ -#include - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -USERVER_NAMESPACE_BEGIN - -namespace { - -formats::json::Value BuildSimpleRetryPolicyConfig() { - formats::json::ValueBuilder retry_policy{formats::common::Type::kObject}; - retry_policy["maxAttempts"] = 5; - retry_policy["initialBackoff"] = "0.010s"; - retry_policy["maxBackoff"] = "0.300s"; - retry_policy["backoffMultiplier"] = 2; - retry_policy["retryableStatusCodes"] = formats::json::MakeArray("UNAVAILABLE"); - return retry_policy.ExtractValue(); -} - -formats::json::Value BuildMethodConfig( - const std::vector>& name, - std::optional timeout, - const formats::json::Value& retry_policy -) { - formats::json::ValueBuilder method_config{formats::common::Type::kObject}; - - for (auto [service_name, method_name] : name) { - method_config["name"].PushBack(formats::json::MakeObject("service", service_name, "method", method_name)); - } - - if (timeout.has_value()) { - method_config["timeout"] = ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString(*timeout)); - } - - method_config["retryPolicy"] = retry_policy; - - return method_config.ExtractValue(); -} - -formats::json::Value BuildMethodConfig( - std::string_view service_name, - std::string_view method_name, - std::optional timeout, - const formats::json::Value& retry_policy -) { - return BuildMethodConfig({{service_name, method_name}}, timeout, retry_policy); -} - -formats::json::Value -BuildDefaultMethodConfig(std::optional timeout, const formats::json::Value& retry_policy) { - // If the 'service' field is empty, the 'method' field must be empty, and - // this MethodConfig specifies the default for all methods (it's the default - // config). - return BuildMethodConfig("", "", timeout, retry_policy); -} - -void VerifyMethodConfig( - const formats::json::Value& method_config, - std::optional service_name, - std::optional method_name, - std::optional timeout, - std::optional attempts -) { - ASSERT_TRUE(method_config["name"].IsArray()); - ASSERT_EQ(1, method_config["name"].GetSize()); - const auto& name = method_config["name"][0]; - ASSERT_EQ(service_name, name["service"].As>()); - ASSERT_EQ(method_name, name["method"].As>()); - - ASSERT_EQ(timeout.has_value(), method_config.HasMember("timeout")); - if (timeout.has_value()) { - ASSERT_EQ(google::protobuf::util::TimeUtil::ToString(*timeout), method_config["timeout"].As()); - } - - ASSERT_EQ(attempts.has_value(), method_config.HasMember("retryPolicy")); - if (attempts.has_value()) { - const auto& retry_policy = method_config["retryPolicy"]; - ASSERT_TRUE(retry_policy.IsObject()); - ASSERT_TRUE(retry_policy.HasMember("maxAttempts")); - ASSERT_EQ(attempts, retry_policy["maxAttempts"].As()); - } -} - -} // namespace - -UTEST(ServiceConfigBuilderTest, BuildEmpty) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - { - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{metadata, std::nullopt}; - const auto service_config = service_config_builder.Build(ugrpc::client::ClientQos{}); - ASSERT_TRUE(service_config.IsNull()); - } - - { - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{metadata, "{}"}; - const auto service_config = service_config_builder.Build(ugrpc::client::ClientQos{}); - ASSERT_TRUE(service_config.IsObject() && service_config.IsEmpty()); - } -} - -UTEST(ServiceConfigBuilderTest, BuildEmptyRetryPolicy) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - const auto default_timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); - const auto retry_policy_json = BuildSimpleRetryPolicyConfig(); - const auto method_config_json = BuildDefaultMethodConfig(default_timeout, retry_policy_json); - const auto static_service_config = - formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); - LOG_DEBUG() << "static_service_config: " << static_service_config; - - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{ - metadata, formats::json::ToString(static_service_config)}; - - const ugrpc::client::Qos qos_default{/*attempts*/ 1, /**/ std::nullopt}; - ugrpc::client::ClientQos client_qos; - client_qos.methods.SetDefault(qos_default); - - const auto service_config = service_config_builder.Build(client_qos); - LOG_DEBUG() << "service_config: " << service_config; - - ASSERT_TRUE(service_config.HasMember("methodConfig")); - const auto& method_config = service_config["methodConfig"]; - ASSERT_TRUE(method_config.IsArray()); - ASSERT_EQ(1, method_config.GetSize()); - - VerifyMethodConfig( - method_config[0], - /*service_name=*/{}, - /*method_name=*/{}, - /*timeout=*/default_timeout, - /*attempts=*/{} - ); -} - -UTEST(ServiceConfigBuilderTest, StaticAndQosSameMethod) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - const auto service_name = metadata.service_full_name; - - const auto timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); - const auto retry_policy_json = BuildSimpleRetryPolicyConfig(); - const auto method_config_json = - BuildMethodConfig(service_name, GetMethodName(metadata, 0), timeout, retry_policy_json); - const auto static_service_config = - formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); - LOG_DEBUG() << "static_service_config: " << static_service_config; - - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{ - metadata, formats::json::ToString(static_service_config)}; - - const ugrpc::client::Qos qos0{/*attempts*/ 3, /**/ std::nullopt}; - ugrpc::client::ClientQos client_qos; - client_qos.methods.Set(GetMethodFullName(metadata, 0), qos0); - - const auto service_config = service_config_builder.Build(client_qos); - LOG_DEBUG() << "service_config: " << service_config; - - ASSERT_TRUE(service_config.HasMember("methodConfig")); - const auto& method_config = service_config["methodConfig"]; - ASSERT_TRUE(method_config.IsArray()); - ASSERT_EQ(1, method_config.GetSize()); - - VerifyMethodConfig( - method_config[0], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 0), - /*timeout=*/timeout, - /*attempts=*/qos0.attempts - ); -} - -UTEST(ServiceConfigBuilderTest, StaticAndQosDifferentMethods) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - const auto service_name = metadata.service_full_name; - - const auto timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); - const auto retry_policy_json = BuildSimpleRetryPolicyConfig(); - const auto method_config_json = - BuildMethodConfig(service_name, GetMethodName(metadata, 0), timeout, retry_policy_json); - const auto static_service_config = - formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); - LOG_DEBUG() << "static_service_config: " << static_service_config; - - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{ - metadata, formats::json::ToString(static_service_config)}; - - const ugrpc::client::Qos qos1{/*attempts*/ 3, /**/ std::nullopt}; - ugrpc::client::ClientQos client_qos; - client_qos.methods.Set(GetMethodFullName(metadata, 1), qos1); - - const auto service_config = service_config_builder.Build(client_qos); - LOG_DEBUG() << "service_config: " << service_config; - - ASSERT_TRUE(service_config.HasMember("methodConfig")); - const auto& method_config = service_config["methodConfig"]; - ASSERT_TRUE(method_config.IsArray()); - ASSERT_EQ(2, method_config.GetSize()); - - VerifyMethodConfig( - method_config[0], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 0), - /*timeout=*/timeout, - /*attempts=*/retry_policy_json["maxAttempts"].As() - ); - - VerifyMethodConfig( - method_config[1], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 1), - /*timeout=*/{}, - /*attempts=*/qos1.attempts - ); -} - -UTEST(ServiceConfigBuilderTest, StaticAndDefaultQos) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - const auto service_name = metadata.service_full_name; - - const auto timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); - const auto retry_policy_json = BuildSimpleRetryPolicyConfig(); - const auto method_config_json = - BuildMethodConfig(service_name, GetMethodName(metadata, 0), timeout, retry_policy_json); - const auto static_service_config = - formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); - LOG_DEBUG() << "static_service_config: " << static_service_config; - - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{ - metadata, formats::json::ToString(static_service_config)}; - - const ugrpc::client::Qos qos_default{/*attempts*/ 3, /**/ std::nullopt}; - ugrpc::client::ClientQos client_qos; - client_qos.methods.SetDefault(qos_default); - - const auto service_config = service_config_builder.Build(client_qos); - LOG_DEBUG() << "service_config: " << service_config; - - ASSERT_TRUE(service_config.HasMember("methodConfig")); - const auto& method_config = service_config["methodConfig"]; - ASSERT_TRUE(method_config.IsArray()); - ASSERT_EQ(2, method_config.GetSize()); - - VerifyMethodConfig( - method_config[0], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 0), - /*timeout=*/timeout, - /*attempts=*/qos_default.attempts - ); - - VerifyMethodConfig( - method_config[1], - /*service_name=*/{}, - /*method_name=*/{}, - /*timeout=*/{}, - /*attempts=*/qos_default.attempts - ); -} - -UTEST(ServiceConfigBuilderTest, DefaultStaticAndQos) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - const auto service_name = metadata.service_full_name; - - const auto default_timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); - const auto retry_policy_json = BuildSimpleRetryPolicyConfig(); - const auto method_config_json = BuildDefaultMethodConfig(default_timeout, retry_policy_json); - const auto static_service_config = - formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); - LOG_DEBUG() << "static_service_config: " << static_service_config; - - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{ - metadata, formats::json::ToString(static_service_config)}; - - const ugrpc::client::Qos qos0{/*attempts*/ 3, /**/ std::nullopt}; - ugrpc::client::ClientQos client_qos; - client_qos.methods.Set(GetMethodFullName(metadata, 0), qos0); - - const auto service_config = service_config_builder.Build(client_qos); - LOG_DEBUG() << "service_config: " << service_config; - - ASSERT_TRUE(service_config.HasMember("methodConfig")); - const auto& method_config = service_config["methodConfig"]; - ASSERT_TRUE(method_config.IsArray()); - ASSERT_EQ(2, method_config.GetSize()); - - VerifyMethodConfig( - method_config[0], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 0), - /*timeout=*/default_timeout, - /*attempts=*/qos0.attempts - ); - - VerifyMethodConfig( - method_config[1], - /*service_name=*/{}, - /*method_name=*/{}, - /*timeout=*/default_timeout, - /*attempts=*/retry_policy_json["maxAttempts"].As() - ); -} - -UTEST(ServiceConfigBuilderTest, ComplexName) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - const auto service_name = metadata.service_full_name; - - const auto timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); - const auto retry_policy_json = BuildSimpleRetryPolicyConfig(); - const auto name = std::vector>{ - {service_name, GetMethodName(metadata, 0)}, {service_name, GetMethodName(metadata, 1)}, {"", ""}}; - const auto method_config_json = BuildMethodConfig(name, timeout, retry_policy_json); - - const auto static_service_config = - formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); - LOG_DEBUG() << "static_service_config: " << static_service_config; - - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{ - metadata, formats::json::ToString(static_service_config)}; - - const ugrpc::client::Qos qos0{/*attempts*/ 3, /**/ std::nullopt}; - ugrpc::client::ClientQos client_qos; - client_qos.methods.Set(GetMethodFullName(metadata, 0), qos0); - - const auto service_config = service_config_builder.Build(client_qos); - LOG_DEBUG() << "service_config: " << service_config; - - ASSERT_TRUE(service_config.HasMember("methodConfig")); - const auto& method_config = service_config["methodConfig"]; - ASSERT_TRUE(method_config.IsArray()); - ASSERT_EQ(3, method_config.GetSize()); - - VerifyMethodConfig( - method_config[0], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 0), - /*timeout=*/timeout, - /*attempts=*/qos0.attempts - ); - - VerifyMethodConfig( - method_config[1], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 1), - /*timeout=*/timeout, - /*attempts=*/retry_policy_json["maxAttempts"].As() - ); - - VerifyMethodConfig( - method_config[2], - /*service_name=*/{}, - /*method_name=*/{}, - /*timeout=*/timeout, - /*attempts=*/retry_policy_json["maxAttempts"].As() - ); -} - -UTEST(ServiceConfigBuilderTest, Complex) { - const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); - - const auto service_name = metadata.service_full_name; - - const auto timeout0 = google::protobuf::util::TimeUtil::MillisecondsToDuration(100); - const auto timeout2 = google::protobuf::util::TimeUtil::MillisecondsToDuration(500); - const auto default_timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1500); - - const auto retry_policy_json = BuildSimpleRetryPolicyConfig(); - - const auto method0_config_json = - BuildMethodConfig(service_name, GetMethodName(metadata, 0), timeout0, retry_policy_json); - const auto method2_config_json = - BuildMethodConfig(service_name, GetMethodName(metadata, 2), timeout2, retry_policy_json); - const auto method3_config_json = - BuildMethodConfig(service_name, GetMethodName(metadata, 3), std::nullopt, retry_policy_json); - const auto default_method_config_json = BuildDefaultMethodConfig(default_timeout, retry_policy_json); - - const auto static_service_config = formats::json::MakeObject( - "methodConfig", - formats::json::MakeArray( - method0_config_json, method2_config_json, method3_config_json, default_method_config_json - ) - ); - LOG_DEBUG() << "static_service_config: " << static_service_config; - - const ugrpc::client::impl::ServiceConfigBuilder service_config_builder{ - metadata, formats::json::ToString(static_service_config)}; - - const ugrpc::client::Qos qos0{/*attempts*/ 2, /**/ std::nullopt}; - const ugrpc::client::Qos qos1{/*attempts*/ 3, /**/ std::nullopt}; - const ugrpc::client::Qos qos3{/*attempts*/ std::nullopt, /**/ std::nullopt}; - const ugrpc::client::Qos qos_default{/*attempts*/ 4, /**/ std::nullopt}; - ugrpc::client::ClientQos client_qos; - client_qos.methods.Set(GetMethodFullName(metadata, 0), qos0); - client_qos.methods.Set(GetMethodFullName(metadata, 1), qos1); - client_qos.methods.Set(GetMethodFullName(metadata, 3), qos3); - client_qos.methods.SetDefault(qos_default); - - const auto service_config = service_config_builder.Build(client_qos); - LOG_DEBUG() << "service_config: " << service_config; - - ASSERT_TRUE(service_config.HasMember("methodConfig")); - const auto& method_config = service_config["methodConfig"]; - ASSERT_TRUE(method_config.IsArray()); - ASSERT_EQ(5, method_config.GetSize()); - - VerifyMethodConfig( - method_config[0], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 0), - /*timeout=*/timeout0, - /*attempts=*/qos0.attempts - ); - - VerifyMethodConfig( - method_config[1], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 1), - /*timeout=*/default_timeout, - /*attempts=*/qos1.attempts - ); - - VerifyMethodConfig( - method_config[2], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 2), - /*timeout=*/timeout2, - /*attempts=*/qos_default.attempts - ); - - VerifyMethodConfig( - method_config[3], - /*service_name=*/service_name, - /*method_name=*/GetMethodName(metadata, 3), - /*timeout=*/std::nullopt, - /*attempts=*/qos_default.attempts - ); - - VerifyMethodConfig( - method_config[4], - /*service_name=*/{}, - /*method_name=*/{}, - /*timeout=*/default_timeout, - /*attempts=*/qos_default.attempts - ); -} - -USERVER_NAMESPACE_END diff --git a/grpc/tests/client_factory_test.cpp b/grpc/tests/client_factory_test.cpp index 00510a607015..bbbada0f523a 100644 --- a/grpc/tests/client_factory_test.cpp +++ b/grpc/tests/client_factory_test.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -13,7 +14,7 @@ using GrpcClientMultichannel = tests::ServiceFixtureMultichannel(); - const auto& data = ugrpc::client::impl::GetClientData(client); + const auto& data = ugrpc::client::impl::ClientDataAccessor::GetClientData(client); const auto stub_state = data.GetStubState(); ASSERT_EQ(stub_state->stubs.Size(), GetParam()); } diff --git a/grpc/tests/client_middleware_hooks_test.cpp b/grpc/tests/client_middleware_hooks_test.cpp index 838d7942dcf7..d6e50d63b065 100644 --- a/grpc/tests/client_middleware_hooks_test.cpp +++ b/grpc/tests/client_middleware_hooks_test.cpp @@ -218,25 +218,32 @@ UTEST_F(ClientMiddlewaresHooksTest, HappyPathBidirectionalStreaming) { } UTEST_F(ClientMiddlewaresHooksTest, HappyPathDetailedUnary) { - SetHappyPathUnary(); - ::testing::MockFunction checkpoint; + + EXPECT_CALL(Service(), SayHello).WillOnce([&](CallContext&, Request&& request) { + checkpoint.Call("Handler"); + + Response response; + response.set_name("Hello " + request.name()); + return response; + }); + { const ::testing::InSequence s; - // Pre* called after request created + // Pre* called before Handler EXPECT_CALL(Middleware(0), PreStartCall).Times(1); EXPECT_CALL(Middleware(0), PreSendMessage).Times(1); EXPECT_CALL(Middleware(0), PostRecvMessage).Times(0); EXPECT_CALL(Middleware(0), PostFinish).Times(0); - EXPECT_CALL(checkpoint, Call("AfterCallStart")); - // Post* called after Finish + EXPECT_CALL(checkpoint, Call("Handler")); + + // Post* called after Handler EXPECT_CALL(Middleware(0), PreStartCall).Times(0); EXPECT_CALL(Middleware(0), PreSendMessage).Times(0); EXPECT_CALL(Middleware(0), PostRecvMessage).Times(1); EXPECT_CALL(Middleware(0), PostFinish).Times(1); - EXPECT_CALL(checkpoint, Call("AfterCallDone")); } Request request; @@ -244,14 +251,12 @@ UTEST_F(ClientMiddlewaresHooksTest, HappyPathDetailedUnary) { // Pre* called after request created auto future = Client().AsyncSayHello(request); - checkpoint.Call("AfterCallStart"); const auto status = future.WaitUntil(engine::Deadline::FromDuration(utest::kMaxTestWaitTime)); EXPECT_EQ(status, engine::FutureStatus::kReady); const auto response = future.Get(); EXPECT_EQ(response.name(), "Hello userver"); - checkpoint.Call("AfterCallDone"); } UTEST_F(ClientMiddlewaresHooksTest, HappyPathDetailedClientStreaming) { @@ -408,7 +413,7 @@ UTEST_F(ClientMiddlewaresHooksTest, MiddlewareExceptionUnaryPreStart) { Request request; request.set_name("userver"); - UEXPECT_THROW(auto future = Client().AsyncSayHello(request), std::runtime_error); + UEXPECT_THROW(auto future = Client().SayHello(request), std::runtime_error); } UTEST_F(ClientMiddlewaresHooksTest, MiddlewareExceptionUnaryPreSend) { @@ -426,7 +431,7 @@ UTEST_F(ClientMiddlewaresHooksTest, MiddlewareExceptionUnaryPreSend) { Request request; request.set_name("userver"); - UEXPECT_THROW(auto future = Client().AsyncSayHello(request), std::runtime_error); + UEXPECT_THROW(auto future = Client().SayHello(request), std::runtime_error); } UTEST_F(ClientMiddlewaresHooksTest, MiddlewareExceptionUnaryPostRecv) { @@ -519,22 +524,24 @@ UTEST_F(ClientMiddlewaresHooksTest, MiddlewareExceptionBidirectionalStreaming) { } UTEST_F(ClientMiddlewaresHooksTest, ExceptionWhenCancelledUnary) { + engine::SingleUseEvent event; + EXPECT_CALL(Service(), SayHello).WillOnce([&](CallContext&, Request&&) { + event.Send(); + engine::InterruptibleSleepFor(utest::kMaxTestWaitTime); + return Response{}; + }); + EXPECT_CALL(Middleware(0), PreStartCall).Times(1); EXPECT_CALL(Middleware(0), PreSendMessage).Times(1); EXPECT_CALL(Middleware(0), PostRecvMessage).Times(0); // skipped, because no response message. EXPECT_CALL(Middleware(0), PostFinish).Times(0); - SetUnary([](CallContext&, Request&&) -> UnaryResult { - engine::InterruptibleSleepFor(utest::kMaxTestWaitTime); - - return Response{}; - }); - { Request request; request.set_name("userver"); const auto future = Client().AsyncSayHello(request); + event.Wait(); engine::current_task::GetCancellationToken().RequestCancel(); // The destructor of `future` will cancel the RPC and await grpcpp cleanup (and don't run middlewares). @@ -580,16 +587,45 @@ UTEST_F(ClientMiddlewaresHooksTest, BadStatusClientStreaming) { UEXPECT_THROW(auto response = stream.Finish(), ugrpc::client::InvalidArgumentError); } -UTEST_F(ClientMiddlewaresHooksTest, Abandoned) { +UTEST_F(ClientMiddlewaresHooksTest, AbandonedUnary) { + engine::SingleUseEvent event; + EXPECT_CALL(Service(), SayHello).WillOnce([&](CallContext&, Request&&) { + event.Send(); + engine::InterruptibleSleepFor(utest::kMaxTestWaitTime); + return Response{}; + }); + + EXPECT_CALL(Middleware(0), PreStartCall).Times(1); + EXPECT_CALL(Middleware(0), PreSendMessage).Times(1); + EXPECT_CALL(Middleware(0), PostRecvMessage).Times(0); + EXPECT_CALL(Middleware(0), PostFinish).Times(0); + + EXPECT_FALSE(GetStatistics("grpc.client.total").SingleMetricOptional("abandoned-error")); + EXPECT_FALSE(GetStatistics("grpc.client.total", {{"grpc_code", "CANCELLED"}}).SingleMetricOptional("status")); + + { + Request request; + auto response_future = Client().AsyncSayHello(request); + event.Wait(); + } + + EXPECT_EQ(GetMetric("abandoned-error"), 1); + EXPECT_FALSE(GetStatistics("grpc.client.total", {{"grpc_code", "CANCELLED"}}).SingleMetricOptional("status")); + + EXPECT_EQ(GetMetric("cancelled"), 0); + EXPECT_EQ(GetMetric("status", {{"grpc_code", "OK"}}), 0); + EXPECT_EQ(GetMetric("status", {{"grpc_code", "UNKNOWN"}}), 0); +} + +UTEST_F(ClientMiddlewaresHooksTest, AbandonedStreaming) { SetHappyPathBidirectionalStreaming(); SetHappyPathClientStreaming(); SetHappyPathServerStreaming(); - SetHappyPathUnary(); - // Five streams were created. - EXPECT_CALL(Middleware(0), PreStartCall).Times(5); - // WriteAndCheck + ReadMany + AsyncSayHello - EXPECT_CALL(Middleware(0), PreSendMessage).Times(3); + // Four streams were created. + EXPECT_CALL(Middleware(0), PreStartCall).Times(4); + // WriteAndCheck + ReadMany + EXPECT_CALL(Middleware(0), PreSendMessage).Times(2); // Skipped, because no response messages. EXPECT_CALL(Middleware(0), PostRecvMessage).Times(0); // We don't run middlewares in a destructors of RPC. @@ -625,17 +661,12 @@ UTEST_F(ClientMiddlewaresHooksTest, Abandoned) { UEXPECT_NO_THROW(const auto stream = Client().ReadMany(request)); } check_metrics(3); - { - Request request; - UEXPECT_NO_THROW(auto future = Client().AsyncSayHello(request)); - } - check_metrics(4); { auto stream = Client().Chat(); StreamResponse response; UEXPECT_NO_THROW(const auto future = stream.ReadAsync(response)); } - check_metrics(5); + check_metrics(4); } UTEST_F(ClientMiddlewaresHooksTest, BadStatusServerStreaming) { diff --git a/grpc/tests/compat/channel_arguments_builder_test.cpp b/grpc/tests/compat/channel_arguments_builder_test.cpp new file mode 100644 index 000000000000..b11b0a788854 --- /dev/null +++ b/grpc/tests/compat/channel_arguments_builder_test.cpp @@ -0,0 +1,321 @@ +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +formats::json::Value BuildSimpleRetryPolicyConfig(std::uint32_t max_attempts) { + formats::json::ValueBuilder retry_policy{formats::common::Type::kObject}; + retry_policy["maxAttempts"] = max_attempts; + retry_policy["initialBackoff"] = "0.010s"; + retry_policy["maxBackoff"] = "0.300s"; + retry_policy["backoffMultiplier"] = 2; + retry_policy["retryableStatusCodes"] = formats::json::MakeArray("UNAVAILABLE"); + return retry_policy.ExtractValue(); +} + +formats::json::Value BuildMethodConfig( + const std::vector, std::optional>>& name, + std::optional timeout, + const formats::json::Value& retry_policy +) { + formats::json::ValueBuilder method_config{formats::common::Type::kObject}; + + for (const auto& [service, method] : name) { + formats::json::ValueBuilder name_entry{formats::common::Type::kObject}; + if (service.has_value()) { + name_entry["service"] = service; + } + if (method.has_value()) { + name_entry["method"] = method; + } + method_config["name"].PushBack(name_entry.ExtractValue()); + } + + if (timeout.has_value()) { + method_config["timeout"] = ugrpc::impl::ToString(google::protobuf::util::TimeUtil::ToString(*timeout)); + } + + method_config["retryPolicy"] = retry_policy; + + return method_config.ExtractValue(); +} + +formats::json::Value +BuildDefaultMethodConfig(std::optional timeout, const formats::json::Value& retry_policy) { + // If the 'service' field is empty, the 'method' field must be empty, and + // this MethodConfig specifies the default for all methods (it's the default + // config). + return BuildMethodConfig({{std::nullopt, std::nullopt}}, timeout, retry_policy); +} + +void VerifyMethodConfig( + const formats::json::Value& method_config, + std::vector, std::optional>> name, + std::optional timeout, + std::optional attempts +) { + ASSERT_TRUE(method_config["name"].IsArray()); + ASSERT_EQ(name.size(), method_config["name"].GetSize()); + size_t i = 0; + for (const auto& [service, method] : name) { + ASSERT_EQ(service, method_config["name"][i]["service"].As>()); + ASSERT_EQ(method, method_config["name"][i]["method"].As>()); + ++i; + } + + ASSERT_EQ(timeout.has_value(), method_config.HasMember("timeout")); + if (timeout.has_value()) { + ASSERT_EQ(google::protobuf::util::TimeUtil::ToString(*timeout), method_config["timeout"].As()); + } + + ASSERT_EQ(attempts.has_value(), method_config.HasMember("retryPolicy")); + if (attempts.has_value()) { + const auto& retry_policy = method_config["retryPolicy"]; + ASSERT_TRUE(retry_policy.IsObject()); + ASSERT_TRUE(retry_policy.HasMember("maxAttempts")); + ASSERT_EQ(attempts, retry_policy["maxAttempts"].As()); + } +} + +} // namespace + +UTEST(ServiceConfigBuilderTest, BuildEmpty) { + const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); + + const ugrpc::client::RetryConfig retry_config; + + { + const ugrpc::client::impl::compat::ServiceConfigBuilder service_config_builder{ + metadata, retry_config, /*static_service_config=*/std::nullopt}; + const auto service_config = service_config_builder.Build(ugrpc::client::ClientQos{}); + ASSERT_TRUE(service_config.IsNull()); + } + + { + const ugrpc::client::impl::compat::ServiceConfigBuilder service_config_builder{ + metadata, retry_config, /*static_service_config=*/"{}"}; + const auto service_config = service_config_builder.Build(ugrpc::client::ClientQos{}); + ASSERT_TRUE(service_config.IsObject() && service_config.IsEmpty()); + } +} + +UTEST(ServiceConfigBuilderTest, Static) { + const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); + + const ugrpc::client::RetryConfig retry_config{/*attempts=*/2}; + + const ugrpc::client::impl::compat::ServiceConfigBuilder service_config_builder{ + metadata, retry_config, /*static_service_config=*/std::nullopt}; + const auto service_config = service_config_builder.Build(ugrpc::client::ClientQos{}); + LOG_DEBUG() << "service_config: " << service_config; + + ASSERT_TRUE(service_config.HasMember("methodConfig")); + const auto& method_config = service_config["methodConfig"]; + ASSERT_TRUE(method_config.IsArray()); + ASSERT_EQ(1, method_config.GetSize()); + + VerifyMethodConfig( + method_config[0], + {{"sample.ugrpc.UnitTestService", "ReadMany"}, + {"sample.ugrpc.UnitTestService", "WriteMany"}, + {"sample.ugrpc.UnitTestService", "Chat"}}, + /*timeout=*/std::nullopt, + /*attempts=*/retry_config.attempts + ); +} + +UTEST(ServiceConfigBuilderTest, Qos) { + const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); + + const ugrpc::client::RetryConfig retry_config; + + const auto default_timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); + + std::uint32_t max_attempts = 5; + const auto retry_policy_json = BuildSimpleRetryPolicyConfig(max_attempts); + const auto method_config_json = BuildDefaultMethodConfig(default_timeout, retry_policy_json); + const auto static_service_config = + formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); + LOG_DEBUG() << "static_service_config: " << static_service_config; + + const ugrpc::client::impl::compat::ServiceConfigBuilder service_config_builder{ + metadata, retry_config, formats::json::ToString(static_service_config)}; + + const ugrpc::client::Qos qos_default{/*attempts=*/2, /*timeout=*/std::nullopt}; + ugrpc::client::ClientQos client_qos; + client_qos.methods.SetDefault(qos_default); + + const auto service_config = service_config_builder.Build(client_qos); + LOG_DEBUG() << "service_config: " << service_config; + + ASSERT_TRUE(service_config.HasMember("methodConfig")); + const auto& method_config = service_config["methodConfig"]; + ASSERT_TRUE(method_config.IsArray()); + ASSERT_EQ(2, method_config.GetSize()); + + VerifyMethodConfig( + method_config[0], + {{"sample.ugrpc.UnitTestService", "ReadMany"}, + {"sample.ugrpc.UnitTestService", "WriteMany"}, + {"sample.ugrpc.UnitTestService", "Chat"}}, + /*timeout=*/default_timeout, + /*attempts=*/*qos_default.attempts + ); + + VerifyMethodConfig( + method_config[1], + {{/*service_name=*/std::nullopt, /*method_name=*/std::nullopt}}, + /*timeout=*/default_timeout, + /*attempts=*/max_attempts + ); +} + +UTEST(ServiceConfigBuilderTest, QosNoRetry) { + const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); + + const ugrpc::client::RetryConfig retry_config; + + const auto default_timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1000); + + std::uint32_t max_attempts = 5; + const auto retry_policy_json = BuildSimpleRetryPolicyConfig(max_attempts); + const auto method_config_json = BuildDefaultMethodConfig(default_timeout, retry_policy_json); + const auto static_service_config = + formats::json::MakeObject("methodConfig", formats::json::MakeArray(method_config_json)); + LOG_DEBUG() << "static_service_config: " << static_service_config; + + const ugrpc::client::impl::compat::ServiceConfigBuilder service_config_builder{ + metadata, retry_config, formats::json::ToString(static_service_config)}; + + const ugrpc::client::Qos qos_default{/*attempts=*/1, /*timeout=*/std::nullopt}; + ugrpc::client::ClientQos client_qos; + client_qos.methods.SetDefault(qos_default); + + const auto service_config = service_config_builder.Build(client_qos); + LOG_DEBUG() << "service_config: " << service_config; + + ASSERT_TRUE(service_config.HasMember("methodConfig")); + const auto& method_config = service_config["methodConfig"]; + ASSERT_TRUE(method_config.IsArray()); + ASSERT_EQ(2, method_config.GetSize()); + + VerifyMethodConfig( + method_config[0], + {{"sample.ugrpc.UnitTestService", "ReadMany"}, + {"sample.ugrpc.UnitTestService", "WriteMany"}, + {"sample.ugrpc.UnitTestService", "Chat"}}, + /*timeout=*/default_timeout, + /*attempts=*/{} + ); + + VerifyMethodConfig( + method_config[1], + {{/*service_name=*/std::nullopt, /*method_name=*/std::nullopt}}, + /*timeout=*/default_timeout, + /*attempts=*/max_attempts + ); +} + +UTEST(ServiceConfigBuilderTest, Complex) { + const auto metadata = sample::ugrpc::UnitTestServiceClient::GetMetadata(); + + const ugrpc::client::RetryConfig retry_config; + + const auto service_name = metadata.service_full_name; + + const auto timeout0 = google::protobuf::util::TimeUtil::MillisecondsToDuration(100); + const auto timeout2 = google::protobuf::util::TimeUtil::MillisecondsToDuration(500); + const auto default_timeout = google::protobuf::util::TimeUtil::MillisecondsToDuration(1500); + + std::uint32_t max_attempts = 5; + const auto retry_policy_json = BuildSimpleRetryPolicyConfig(max_attempts); + + const auto method0_config_json = + BuildMethodConfig({{service_name, GetMethodName(metadata, 0)}}, timeout0, retry_policy_json); + const auto method2_config_json = + BuildMethodConfig({{service_name, GetMethodName(metadata, 2)}}, timeout2, retry_policy_json); + const auto method3_config_json = + BuildMethodConfig({{service_name, GetMethodName(metadata, 3)}}, std::nullopt, retry_policy_json); + const auto default_method_config_json = BuildDefaultMethodConfig(default_timeout, retry_policy_json); + + const auto static_service_config = formats::json::MakeObject( + "methodConfig", + formats::json::MakeArray( + method0_config_json, method2_config_json, method3_config_json, default_method_config_json + ) + ); + LOG_DEBUG() << "static_service_config: " << static_service_config; + + const ugrpc::client::impl::compat::ServiceConfigBuilder service_config_builder{ + metadata, retry_config, formats::json::ToString(static_service_config)}; + + const ugrpc::client::Qos qos0{/*attempts=*/2, /*timeout=*/std::nullopt}; + const ugrpc::client::Qos qos1{/*attempts=*/3, /*timeout=*/std::nullopt}; + const ugrpc::client::Qos qos3{/*attempts=*/std::nullopt, /*timeout=*/std::nullopt}; + const ugrpc::client::Qos qos_default{/*attempts=*/4, /*timeout=*/std::nullopt}; + ugrpc::client::ClientQos client_qos; + client_qos.methods.Set(GetMethodFullName(metadata, 0), qos0); + client_qos.methods.Set(GetMethodFullName(metadata, 1), qos1); + client_qos.methods.Set(GetMethodFullName(metadata, 3), qos3); + client_qos.methods.SetDefault(qos_default); + + const auto service_config = service_config_builder.Build(client_qos); + LOG_DEBUG() << "service_config: " << service_config; + + ASSERT_TRUE(service_config.HasMember("methodConfig")); + const auto& method_config = service_config["methodConfig"]; + ASSERT_TRUE(method_config.IsArray()); + ASSERT_EQ(5, method_config.GetSize()); + + VerifyMethodConfig( + method_config[0], + {{service_name, GetMethodName(metadata, 0)}}, + /*timeout=*/timeout0, + /*attempts=*/max_attempts + ); + + VerifyMethodConfig( + method_config[1], + {{service_name, GetMethodName(metadata, 1)}}, + /*timeout=*/default_timeout, + /*attempts=*/qos1.attempts + ); + + VerifyMethodConfig( + method_config[2], + {{service_name, GetMethodName(metadata, 2)}}, + /*timeout=*/timeout2, + /*attempts=*/qos_default.attempts + ); + + VerifyMethodConfig( + method_config[3], + {{service_name, GetMethodName(metadata, 3)}}, + /*timeout=*/std::nullopt, + /*attempts=*/qos_default.attempts + ); + + VerifyMethodConfig( + method_config[4], + {{/*service_name=*/std::nullopt, /*method_name=*/std::nullopt}}, + /*timeout=*/default_timeout, + /*attempts=*/max_attempts + ); +} + +USERVER_NAMESPACE_END diff --git a/grpc/tests/deadline_metrics_test.cpp b/grpc/tests/deadline_metrics_test.cpp index 45d4ed5e17ec..ca4e1a8c5450 100644 --- a/grpc/tests/deadline_metrics_test.cpp +++ b/grpc/tests/deadline_metrics_test.cpp @@ -149,6 +149,9 @@ UTEST_F(DeadlineStatsTests, ClientDeadlineUpdated) { expected_value += 3; EXPECT_EQ(GetClientStatistic(kDeadlinePropagated), expected_value); + // reset TaskInheritedDeadline, set once is too short for many requests + tests::InitTaskInheritedDeadline(); + // Requests without deadline // TaskInheritedData will be set as deadline EXPECT_TRUE(PerformRequest(false)); diff --git a/grpc/tests/logging_test.cpp b/grpc/tests/logging_test.cpp index 6fa0a5c4d79e..f6940b0e93a8 100644 --- a/grpc/tests/logging_test.cpp +++ b/grpc/tests/logging_test.cpp @@ -2,11 +2,11 @@ #include +#include +#include #include #include -#include - #include #include #include @@ -15,6 +15,28 @@ USERVER_NAMESPACE_BEGIN namespace { +constexpr std::string_view kBaseTskvPattern = R"(tskv\t)" + R"(timestamp=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\t)" + R"(timezone=[-+]\d{2}\d{2}\t)" + R"(user_agent=.+\t)" + R"(ip=[.0-9a-f:\[\]]+\:[0-9]+\t)" + R"(x_real_ip=[.0-9a-f:\[\]]+\:[0-9]+\t)" + R"(request=[a-zA-Z./0-9]+\t)" + R"(request_time=\d+\.\d+\t)" + R"(upstream_response_time=\d+\.\d+\t)" + R"(grpc_status=\d+\t)" + R"(grpc_status_code=[A-Z_]+)"; + +const utils::regex kBaseTskvRegex{std::string{kBaseTskvPattern} + R"(\n)"}; + +// Constant to check result of LogExtra middleware +const utils::regex kTskvWithTagsRegex{ + std::string{kBaseTskvPattern} + R"(\tuser_id=12345)" + R"(\tsession_id=abc-def-ghi)" + R"(\trequest_id=req-98765)" + R"(\tcustom_tag=test-value)" + R"(\tnumeric_tag=42\n)"}; + class UnitTestService final : public sample::ugrpc::UnitTestServiceBase { public: SayHelloResult SayHello(CallContext& /*context*/, sample::ugrpc::GreetingRequest&& request) override { @@ -48,9 +70,54 @@ class ServiceWithAccessLogFixture : public ugrpc::tests::ServiceFixtureBase { GrpcService service_{}; }; +// Test middleware that adds custom tags to LogExtra +class TagsMiddleware final : public ugrpc::server::MiddlewareBase { +public: + /// [grpc log extra tag] + void OnCallStart(ugrpc::server::MiddlewareCallContext& context) const override { + ugrpc::server::middlewares::access_log::SetAdditionalLogKeys( + context, + logging::LogExtra{ + std::pair("user_id", "12345"), + std::pair("session_id", "abc-def-ghi"), + std::pair("request_id", "req-98765"), + std::pair("custom_tag", "test-value"), + std::pair("numeric_tag", "42"), + } + ); + } + /// [grpc log extra tag] +}; + +template +class ServiceWithAccessLogTagsFixture : public ugrpc::tests::ServiceFixtureBase { +public: + ServiceWithAccessLogTagsFixture() { + ugrpc::server::middlewares::access_log::Settings access_log_settings; + access_log_settings.access_tskv_logger = access_logger_.GetLogger(); + + SetServerMiddlewares({ + std::make_shared(std::move(access_log_settings)), + std::make_shared(), + }); + + RegisterService(service_); + StartServer(); + } + + ~ServiceWithAccessLogTagsFixture() override { StopServer(); } + + utest::LogCaptureLogger& AccessLog() { return access_logger_; } + +private: + utest::LogCaptureLogger access_logger_{logging::Format::kRaw}; + GrpcService service_{}; +}; + } // namespace using GrpcAccessLog = ServiceWithAccessLogFixture; +using GrpcAccessLogWithTags = ServiceWithAccessLogTagsFixture; UTEST_F(GrpcAccessLog, Test) { auto client = MakeClient(); @@ -60,20 +127,20 @@ UTEST_F(GrpcAccessLog, Test) { GetServer().StopServing(); - constexpr std::string_view kExpectedPattern = R"(tskv\t)" - R"(timestamp=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\t)" - R"(timezone=[-+]\d{2}\d{2}\t)" - R"(user_agent=.+\t)" - R"(ip=[.0-9a-f:\[\]]+\:[0-9]+\t)" - R"(x_real_ip=[.0-9a-f:\[\]]+\:[0-9]+\t)" - R"(request=[a-zA-Z./0-9]+\t)" - R"(request_time=\d+\.\d+\t)" - R"(upstream_response_time=\d+\.\d+\t)" - R"(grpc_status=\d+\t)" - R"(grpc_status_code=[A-Z_]+\n)"; + const auto logs = GetSingleLog(AccessLog().GetAll()); + EXPECT_TRUE(utils::regex_match(logs.GetLogRaw(), kBaseTskvRegex)) << logs; +} + +UTEST_F(GrpcAccessLogWithTags, TestWithCustomTags) { + auto client = MakeClient(); + sample::ugrpc::GreetingRequest out; + out.set_name("userver"); + auto response = client.SayHello(out); + + GetServer().StopServing(); const auto logs = GetSingleLog(AccessLog().GetAll()); - EXPECT_TRUE(utils::regex_match(logs.GetLogRaw(), utils::regex(kExpectedPattern))) << logs; + EXPECT_TRUE(utils::regex_match(logs.GetLogRaw(), kTskvWithTagsRegex)) << logs; } USERVER_NAMESPACE_END diff --git a/grpc/tests/metadata_test.cpp b/grpc/tests/metadata_test.cpp new file mode 100644 index 000000000000..c949fb11ef58 --- /dev/null +++ b/grpc/tests/metadata_test.cpp @@ -0,0 +1,161 @@ +#include + +#include +#include + +#include + +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +class SimpleMetadataService final : public sample::ugrpc::UnitTestServiceBase { +public: + SayHelloResult SayHello(CallContext& context, sample::ugrpc::GreetingRequest&& request) override { + /// [server_read_client_metadata] + const std::multimap& client_metadata = + context.GetServerContext().client_metadata(); + const std::optional custom_header = utils::FindOptional(client_metadata, "custom-header"); + /// [server_read_client_metadata] + + /// [server_write_initial_metadata] + context.GetServerContext().AddInitialMetadata("response-header", "response-value"); + /// [server_write_initial_metadata] + + /// [server_write_trailing_metadata] + context.GetServerContext().AddTrailingMetadata("request-id", "req-123"); + /// [server_write_trailing_metadata] + + const std::optional user_id = utils::FindOptional(client_metadata, "user-id"); + context.GetServerContext().AddInitialMetadata("server-version", "1.0.0"); + context.GetServerContext().AddTrailingMetadata("processing-time", "42ms"); + + std::string greeting = "Hello " + request.name(); + if (custom_header) { + greeting += " (header: " + std::string(custom_header->data(), custom_header->size()) + ")"; + } + if (user_id) { + greeting += " (user: " + std::string(user_id->data(), user_id->size()) + ")"; + } + + sample::ugrpc::GreetingResponse response; + response.set_name(greeting); + return response; + } + + ReadManyResult + ReadMany(CallContext& context, sample::ugrpc::StreamGreetingRequest&& request, ReadManyWriter& writer) override { + context.GetServerContext().AddInitialMetadata("stream-started", "true"); + context.GetServerContext().AddInitialMetadata("total-items", std::to_string(request.number())); + + sample::ugrpc::StreamGreetingResponse response; + response.set_name("Stream response for " + request.name()); + + for (int i = 0; i < request.number(); ++i) { + response.set_number(i); + writer.Write(response); + } + + context.GetServerContext().AddTrailingMetadata("stream-completed", "true"); + + return grpc::Status::OK; + } +}; + +} // namespace + +using MetadataTest = ugrpc::tests::ServiceFixture; + +UTEST_F(MetadataTest, ClientSendMetadata) { + auto client = MakeClient(); + + sample::ugrpc::GreetingRequest request; + request.set_name("test"); + + /// [client_write_metadata] + ugrpc::client::CallOptions call_options; + call_options.AddMetadata("custom-header", "custom-value"); + /// [client_write_metadata] + call_options.AddMetadata("user-id", "12345"); + + auto response = client.SayHello(request, std::move(call_options)); + EXPECT_EQ(response.name(), "Hello test (header: custom-value) (user: 12345)"); +} + +UTEST_F(MetadataTest, ClientReadInitialMetadata) { + auto client = MakeClient(); + + sample::ugrpc::GreetingRequest request; + request.set_name("test"); + + auto future = client.AsyncSayHello(request, {}); + auto response = future.Get(); + + /// [client_read_initial_metadata] + const std::multimap& initial_metadata = + future.GetContext().GetClientContext().GetServerInitialMetadata(); + /// [client_read_initial_metadata] + + EXPECT_THAT(initial_metadata, testing::Contains(testing::Pair("response-header", "response-value"))); + EXPECT_THAT(initial_metadata, testing::Contains(testing::Pair("server-version", "1.0.0"))); +} + +UTEST_F(MetadataTest, ClientReadTrailingMetadata) { + auto client = MakeClient(); + + sample::ugrpc::GreetingRequest request; + request.set_name("test"); + + auto future = client.AsyncSayHello(request, {}); + auto response = future.Get(); + + /// [client_read_trailing_metadata] + const std::multimap& trailing_metadata = + future.GetContext().GetClientContext().GetServerTrailingMetadata(); + /// [client_read_trailing_metadata] + + EXPECT_THAT(trailing_metadata, testing::Contains(testing::Pair("request-id", "req-123"))); + EXPECT_THAT(trailing_metadata, testing::Contains(testing::Pair("processing-time", "42ms"))); +} + +UTEST_F(MetadataTest, StreamingRequestResponseMetadata) { + auto client = MakeClient(); + + sample::ugrpc::StreamGreetingRequest request; + request.set_name("stream-test"); + request.set_number(3); + + ugrpc::client::CallOptions call_options; + call_options.AddMetadata("stream-id", "stream-123"); + call_options.AddMetadata("custom-request-header", "request-value"); + + auto stream = client.ReadMany(request, std::move(call_options)); + + sample::ugrpc::StreamGreetingResponse response; + int count = 0; + while (stream.Read(response)) { + EXPECT_EQ(response.name(), "Stream response for stream-test"); + EXPECT_EQ(response.number(), count); + ++count; + } + EXPECT_EQ(count, 3); + + const grpc::ClientContext& client_context = stream.GetContext().GetClientContext(); + + const std::multimap& initial_metadata = + client_context.GetServerInitialMetadata(); + EXPECT_THAT(initial_metadata, testing::Contains(testing::Pair("stream-started", "true"))); + EXPECT_THAT(initial_metadata, testing::Contains(testing::Pair("total-items", "3"))); + + const std::multimap& trailing_metadata = + client_context.GetServerTrailingMetadata(); + EXPECT_THAT(trailing_metadata, testing::Contains(testing::Pair("stream-completed", "true"))); +} + +USERVER_NAMESPACE_END diff --git a/grpc/tests/protobuf_logging_test.cpp b/grpc/tests/protobuf_logging_test.cpp new file mode 100644 index 000000000000..5711b819040e --- /dev/null +++ b/grpc/tests/protobuf_logging_test.cpp @@ -0,0 +1,169 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +UTEST(ProtobufLogging, GetMessageWithData) { + google::protobuf::StringValue test_message; + test_message.set_value("test string with some content"); + + const auto result = ugrpc::ToLimitedDebugString(test_message, 1000); + + EXPECT_THAT(result, testing::HasSubstr("test string with some content")); + EXPECT_THAT(result, testing::HasSubstr("value:")); +} + +UTEST(ProtobufLogging, GetMessageEmptyMessage) { + google::protobuf::Empty empty_message; + const auto result = ugrpc::ToLimitedDebugString(empty_message, 100); + + EXPECT_EQ(result, ""); +} + +UTEST(ProtobufLogging, GetMessageSizeLimit) { + google::protobuf::StringValue test_message; + test_message.set_value("test string with some content"); + + const auto small_result = ugrpc::ToLimitedDebugString(test_message, 10); + const auto large_result = ugrpc::ToLimitedDebugString(test_message, 1000); + + EXPECT_LE(small_result.size(), 10u); + EXPECT_GT(large_result.size(), small_result.size()); + EXPECT_THAT(large_result, testing::HasSubstr("test string with some content")); +} + +UTEST(ProtobufLogging, OkStatus) { + grpc::Status ok_status = grpc::Status::OK; + const auto result = ugrpc::ToUnlimitedDebugString(ok_status); + + EXPECT_EQ(result, "OK"); +} + +UTEST(ProtobufLogging, SimpleError) { + grpc::Status simple_error(grpc::StatusCode::NOT_FOUND, "Resource not found"); + const auto result = ugrpc::ToUnlimitedDebugString(simple_error); + + EXPECT_THAT(result, testing::HasSubstr("NOT_FOUND")); + EXPECT_THAT(result, testing::HasSubstr("Resource not found")); + EXPECT_THAT(result, testing::HasSubstr("code:")); + EXPECT_THAT(result, testing::HasSubstr("error message:")); +} + +UTEST(ProtobufLogging, ComplexError) { + google::rpc::Status error_details; + error_details.set_code(static_cast(grpc::StatusCode::INVALID_ARGUMENT)); + error_details.set_message("Invalid parameter provided"); + + auto* detail = error_details.add_details(); + detail->set_type_url("type.googleapis.com/google.protobuf.StringValue"); + google::protobuf::StringValue detail_value; + detail_value.set_value("Additional error context"); + detail->set_value(detail_value.SerializeAsString()); + + grpc::Status complex_status( + grpc::StatusCode::INVALID_ARGUMENT, "Invalid parameter provided", error_details.SerializeAsString() + ); + + const auto result = ugrpc::ToUnlimitedDebugString(complex_status); + + EXPECT_THAT(result, testing::HasSubstr("INVALID_ARGUMENT")); + EXPECT_THAT(result, testing::HasSubstr("Invalid parameter provided")); + EXPECT_THAT(result, testing::HasSubstr("error details:")); + EXPECT_THAT(result, testing::HasSubstr("Additional error context")); + EXPECT_THAT(result, testing::HasSubstr("type.googleapis.com")); +} + +UTEST(ProtobufLogging, EdgeCasesZeroMaxSize) { + google::protobuf::StringValue test_message; + test_message.set_value("test string with some content"); + + const auto result = ugrpc::ToLimitedDebugString(test_message, 0); + EXPECT_TRUE(result.empty()); +} + +UTEST(ProtobufLogging, GetErrorDetailsSizeLimiting) { + std::string large_message(2000, 'A'); + google::rpc::Status error_details; + error_details.set_code(static_cast(grpc::StatusCode::NOT_FOUND)); + error_details.set_message("Some error"); + auto* detail = error_details.add_details(); + google::protobuf::StringValue detail_value; + detail_value.set_value(large_message); + detail->set_value(detail_value.SerializeAsString()); + grpc::Status large_error( + grpc::StatusCode::NOT_FOUND, "Invalid parameter provided", error_details.SerializeAsString() + ); + const auto small_result = ugrpc::ToLimitedDebugString(large_error, 100); + EXPECT_LE(small_result.size(), 200u); + + const auto medium_result = ugrpc::ToLimitedDebugString(large_error, 500); + EXPECT_LE(medium_result.size(), 600u); + EXPECT_GT(medium_result.size(), small_result.size()); + + const auto large_result = ugrpc::ToLimitedDebugString(large_error, 3000); + EXPECT_GT(large_result.size(), medium_result.size()); +} + +UTEST(ProtobufLogging, GetErrorDetailsUnlimited) { + std::string large_message(5000, 'B'); + google::rpc::Status error_details; + auto* detail = error_details.add_details(); + google::protobuf::StringValue detail_value; + detail_value.set_value(large_message); + detail->set_value(detail_value.SerializeAsString()); + grpc::Status large_error( + grpc::StatusCode::NOT_FOUND, "Invalid parameter provided", error_details.SerializeAsString() + ); + const auto unlimited_result = ugrpc::ToUnlimitedDebugString(large_error); + + EXPECT_THAT(unlimited_result, testing::HasSubstr(large_message)); + EXPECT_THAT(unlimited_result, testing::HasSubstr("Invalid parameter provided")); + EXPECT_GT(unlimited_result.size(), 5000); +} + +UTEST(ProtobufLogging, EdgeCasesEmptyDetails) { + grpc::Status status_with_empty_details(grpc::StatusCode::INTERNAL, "Internal error", ""); + + const auto result = ugrpc::ToLimitedDebugString(status_with_empty_details); + EXPECT_THAT(result, testing::HasSubstr("INTERNAL")); + EXPECT_THAT(result, testing::HasSubstr("Internal error")); + EXPECT_THAT(result, testing::Not(testing::HasSubstr("error details:"))); +} + +UTEST(ProtobufLogging, LogHelperOperator) { + google::protobuf::StringValue message; + message.set_value("stdout test message"); + LOG_INFO("Stdout test: {}", message); +} + +UTEST(ProtobufLogging, LogHelperOperatorEmptyMessage) { + google::protobuf::Empty empty_message; + LOG_INFO("Empty protobuf: {}", empty_message); +} + +UTEST(ProtobufLogging, FmtFormatterWithData) { + std::string message = "test string with some content"; + google::protobuf::StringValue test_message; + test_message.set_value(message); + + const auto result = fmt::format("Message: {}", test_message); + EXPECT_THAT(result, testing::HasSubstr(message)); +} + +UTEST(ProtobufLogging, FmtFormatterEmptyMessage) { + google::protobuf::Empty empty_message; + const auto result = fmt::format("Empty: {}", empty_message); + + EXPECT_EQ(result, "Empty: "); +} + +USERVER_NAMESPACE_END diff --git a/grpc/tests/protobuf_utils_test.cpp b/grpc/tests/protobuf_utils_test.cpp index 4547667a6bad..06c8ba0bf8a4 100644 --- a/grpc/tests/protobuf_utils_test.cpp +++ b/grpc/tests/protobuf_utils_test.cpp @@ -1,8 +1,7 @@ #include #include - -#include +#include #include #include @@ -45,11 +44,11 @@ UTEST(ToLimitedDebugString, Basic) { message.set_id("swag"); *message.add_names() = "test-name-1"; *message.add_names() = "test-name-2"; - auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + auto out = ugrpc::ToLimitedDebugString(message, kLimit); const auto expected = message.DebugString(); EXPECT_EQ(out, expected); - out = ugrpc::impl::ToLimitedDebugString(message, 8); + out = ugrpc::ToLimitedDebugString(message, 8); EXPECT_EQ(out, expected.substr(0, 8)); } @@ -57,7 +56,7 @@ UTEST(ToLimitedDebugString, Fit) { constexpr std::size_t kLimit = 20; sample::ugrpc::GreetingResponse message; message.set_name("1234567890"); - const auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + const auto out = ugrpc::ToLimitedDebugString(message, kLimit); EXPECT_EQ(out, "name: \"1234567890\"\n"); } @@ -65,7 +64,7 @@ UTEST(ToLimitedDebugString, Limited) { constexpr std::size_t kLimit = 10; sample::ugrpc::GreetingResponse message; message.set_name("1234567890"); - const auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + const auto out = ugrpc::ToLimitedDebugString(message, kLimit); EXPECT_EQ(out, "name: \"123"); } @@ -73,7 +72,7 @@ UTEST(ToLimitedDebugString, Complex) { constexpr std::size_t kLimit = 512; const auto message = ConstructComplexMessage(); const auto expected = ugrpc::impl::ToString(message.Utf8DebugString().substr(0, kLimit)); - ASSERT_EQ(expected, ugrpc::impl::ToLimitedDebugString(message, kLimit)); + ASSERT_EQ(expected, ugrpc::ToLimitedDebugString(message, kLimit)); } USERVER_NAMESPACE_END diff --git a/grpc/tests/retry_test.cpp b/grpc/tests/retry_test.cpp index 559b1ce49695..888b668511e5 100644 --- a/grpc/tests/retry_test.cpp +++ b/grpc/tests/retry_test.cpp @@ -32,7 +32,7 @@ using RetryTest = ugrpc::tests::ServiceFixture; } // namespace -UTEST_F(RetryTest, Attempts) { +UTEST_F(RetryTest, QosAttempts) { ugrpc::client::Qos qos; qos.attempts = 4; ugrpc::client::ClientQos client_qos; @@ -53,4 +53,18 @@ UTEST_F(RetryTest, Attempts) { UEXPECT_NO_THROW(response = client.SayHello(request)); } +UTEST_F(RetryTest, CallOptionsAttempts) { + auto client = MakeClient(); + + sample::ugrpc::GreetingRequest request; + request.set_name("testname"); + /// [SetAttempts] + ugrpc::client::CallOptions call_options; + call_options.SetAttempts(4); + + sample::ugrpc::GreetingResponse response; + UEXPECT_NO_THROW(response = client.SayHello(request, std::move(call_options))); + /// [SetAttempts] +} + USERVER_NAMESPACE_END diff --git a/grpc/tests/secret_fields_test.cpp b/grpc/tests/secret_fields_test.cpp index f9c0a65f63e2..1e1f0670e2ee 100644 --- a/grpc/tests/secret_fields_test.cpp +++ b/grpc/tests/secret_fields_test.cpp @@ -1,15 +1,15 @@ -#include - #include #if defined(ARCADIA_ROOT) || GOOGLE_PROTOBUF_VERSION >= 4022000 -#include -#include -#include +#include #include #include +#include #include +#include +#include + #include #include #include @@ -138,11 +138,11 @@ UTEST(ToLimitedDebugStringWithSecrets, Basic) { message.set_password("qwerty12345678"); message.set_name("test-name"); message.set_count(7); - auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + auto out = ugrpc::ToLimitedDebugString(message, kLimit); const std::string_view expected = "id: \"swag\"\nname: \"test-name\"\npassword: [REDACTED]\ncount: [REDACTED]\n"; EXPECT_EQ(out, expected); - out = ugrpc::impl::ToLimitedDebugString(message, 44); + out = ugrpc::ToLimitedDebugString(message, 44); EXPECT_EQ(out, expected.substr(0, 44)); } @@ -157,7 +157,7 @@ UTEST(ToLimitedDebugStringWithSecrets, InnerSecrets) { item->set_value("value"); const std::string_view expected = "id: \"swag\"\nitems {\n index: 7\n value: [REDACTED]\n}\n"; - const auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + const auto out = ugrpc::ToLimitedDebugString(message, kLimit); EXPECT_EQ(out, expected); } @@ -172,7 +172,7 @@ UTEST(ToLimitedDebugStringWithSecrets, SecretsAndFit) { item->set_value("value"); const std::string_view expected = "id: \"swag\"\n"; - const auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + const auto out = ugrpc::ToLimitedDebugString(message, kLimit); EXPECT_EQ(out, expected); } @@ -187,7 +187,7 @@ UTEST(ToLimitedDebugStringWithSecrets, TheSameName) { const auto expected = message.DebugString(); - const auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + const auto out = ugrpc::ToLimitedDebugString(message, kLimit); EXPECT_EQ(out, expected); } @@ -203,7 +203,7 @@ UTEST(ToLimitedDebugStringWithSecrets, StructSecret) { const auto expected = "name1: \"name1\"\ncurrent_state: [REDACTED]\nname2: \"name2\"\n"; - const auto out = ugrpc::impl::ToLimitedDebugString(message, kLimit); + const auto out = ugrpc::ToLimitedDebugString(message, kLimit); EXPECT_EQ(out, expected); } diff --git a/grpc/tests/service_config_test.cpp b/grpc/tests/service_config_test.cpp index 671a934badce..35d2a2ab434b 100644 --- a/grpc/tests/service_config_test.cpp +++ b/grpc/tests/service_config_test.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -55,7 +56,7 @@ class GrpcClientWithServiceConfig : public ugrpc::tests::ServiceFixtureBase { UTEST_F(GrpcClientWithServiceConfig, DefaultServiceConfig) { auto client = MakeClient(); - auto& data = ugrpc::client::impl::GetClientData(client); + const auto& data = ugrpc::client::impl::ClientDataAccessor::GetClientData(client); // This is a very important moment. THe default service_config is applied not // upon creation, but when the name resolution process returns no service diff --git a/grpc/tests/timeout_test.cpp b/grpc/tests/timeout_test.cpp index 16d592684da5..ee4b9e1ecfe1 100644 --- a/grpc/tests/timeout_test.cpp +++ b/grpc/tests/timeout_test.cpp @@ -1,7 +1,5 @@ #include -#include // for GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING - #include #include @@ -15,14 +13,6 @@ USERVER_NAMESPACE_BEGIN namespace { -#ifdef GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DISABLED_IN_OLD_GRPC_TEST_NAME(name) name -#else -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DISABLED_IN_OLD_GRPC_TEST_NAME(name) DISABLED_##name -#endif - class UnitTestService final : public sample::ugrpc::UnitTestServiceBase { public: SayHelloResult SayHello(CallContext& context, sample::ugrpc::GreetingRequest&& request) override { @@ -49,7 +39,7 @@ using TimeoutTest = ugrpc::tests::ServiceFixture; } // namespace -UTEST_F(TimeoutTest, DISABLED_IN_OLD_GRPC_TEST_NAME(PerAttemptTimeout)) { +UTEST_F(TimeoutTest, QosTimeout) { ugrpc::client::Qos qos; qos.attempts = 4; qos.timeout = tests::kLongTimeout; @@ -71,4 +61,18 @@ UTEST_F(TimeoutTest, DISABLED_IN_OLD_GRPC_TEST_NAME(PerAttemptTimeout)) { UEXPECT_NO_THROW(response = client.SayHello(request)); } +UTEST_F(TimeoutTest, CallOptionsTimeout) { + auto client = MakeClient(); + + sample::ugrpc::GreetingRequest request; + request.set_name("testname"); + + ugrpc::client::CallOptions call_options; + call_options.SetAttempts(4); + call_options.SetTimeout(tests::kLongTimeout); + + sample::ugrpc::GreetingResponse response; + UEXPECT_NO_THROW(response = client.SayHello(request, std::move(call_options))); +} + USERVER_NAMESPACE_END diff --git a/kafka/CMakeLists.txt b/kafka/CMakeLists.txt index dd7c202d58f5..4e2ffbe422d1 100644 --- a/kafka/CMakeLists.txt +++ b/kafka/CMakeLists.txt @@ -12,6 +12,7 @@ userver_module( LINK_LIBRARIES_PRIVATE RdKafka::rdkafka DBTEST_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/tests" DBTEST_LINK_LIBRARIES userver::kafka-utest + DEPENDS core ) target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-ignored-qualifiers") diff --git a/kafka/include/userver/kafka/exceptions.hpp b/kafka/include/userver/kafka/exceptions.hpp index 265e1a79f3b6..634ca7b5fa8f 100644 --- a/kafka/include/userver/kafka/exceptions.hpp +++ b/kafka/include/userver/kafka/exceptions.hpp @@ -117,18 +117,18 @@ class ParseHeadersException final : std::runtime_error { }; /// @brief Exception thrown when Seek* process failed. -/// @ref ConsumeScope::Seek -/// @ref ConsumeScope::SeekToBeginning -/// @ref ConsumeScope::SeekToEnd +/// @ref ConsumerScope::Seek +/// @ref ConsumerScope::SeekToBeginning +/// @ref ConsumerScope::SeekToEnd class SeekException final : public std::runtime_error { public: using std::runtime_error::runtime_error; }; /// @brief Exception thrown when Seek* arguments are invalid. -/// @ref ConsumeScope::Seek -/// @ref ConsumeScope::SeekToBeginning -/// @ref ConsumeScope::SeekToEnd +/// @ref ConsumerScope::Seek +/// @ref ConsumerScope::SeekToBeginning +/// @ref ConsumerScope::SeekToEnd class SeekInvalidArgumentException final : public std::invalid_argument { public: using std::invalid_argument::invalid_argument; diff --git a/kafka/include/userver/kafka/impl/broker_secrets.hpp b/kafka/include/userver/kafka/impl/broker_secrets.hpp index 5991cce553f9..1e4fe5187586 100644 --- a/kafka/include/userver/kafka/impl/broker_secrets.hpp +++ b/kafka/include/userver/kafka/impl/broker_secrets.hpp @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include @@ -12,9 +14,32 @@ namespace kafka::impl { struct Secret final { using SecretType = utils::NonLoggable; + struct SaslCredentials final { + /// @brief SASL username for use with the PLAIN and SASL-SCRAM-.. mechanisms. + SecretType username; + /// @brief SASL password for use with the PLAIN and SASL-SCRAM-.. mechanisms. + SecretType password; + }; + + struct SslCredentials final { + /// @brief Path to client's public key (PEM) used for authentication. + SecretType ssl_certificate_location; + /// @brief Path to client's private key (PEM) used for authentication. + SecretType ssl_key_location; + /// @brief (Optional) Private key passphrase. + std::optional ssl_key_password{}; + }; + + /// @brief Brokers URI comma-separated list. + /// @note Is is allowed to pass only one broker URI. + /// Client discovers over brokers automatically. std::string brokers; - SecretType username; - SecretType password; + + /// @brief Security protocol corresponding credentials. + /// PLAINTEXT -> none + /// SASL_PLAINTEXT/SASL_SSL -> SaslCredentials + /// SSL -> SslCredentials + std::variant credentials{}; }; class BrokerSecrets final { diff --git a/kafka/include/userver/kafka/impl/configuration.hpp b/kafka/include/userver/kafka/impl/configuration.hpp index 60ab628a2f2f..2ca75b8ff4c0 100644 --- a/kafka/include/userver/kafka/impl/configuration.hpp +++ b/kafka/include/userver/kafka/impl/configuration.hpp @@ -33,8 +33,11 @@ struct SecurityConfiguration final { std::string security_mechanism; std::string ssl_ca_location; }; + struct Ssl final { + std::string ssl_ca_location; + }; - using SecurityProtocol = std::variant; + using SecurityProtocol = std::variant; SecurityProtocol security_protocol{}; }; diff --git a/kafka/include/userver/kafka/impl/holders.hpp b/kafka/include/userver/kafka/impl/holders.hpp index 405d26485de1..76e1abf5a5d8 100644 --- a/kafka/include/userver/kafka/impl/holders.hpp +++ b/kafka/include/userver/kafka/impl/holders.hpp @@ -43,7 +43,7 @@ class HolderBase final { friend class ConsumerImpl; friend class HeadersHolder; - [[nodiscard]] T* release() noexcept; + [[nodiscard]] T* Release() noexcept; std::unique_ptr> ptr_; }; @@ -75,7 +75,7 @@ enum class ClientType : std::uint8_t { kConsumer, kProducer }; /// @brief Kafka client wrapper (consumer or producer). /// Holds the client and its events queue. -template +template class KafkaClientHolder final { public: explicit KafkaClientHolder(ConfHolder conf); diff --git a/kafka/include/userver/kafka/rebalance_types.hpp b/kafka/include/userver/kafka/rebalance_types.hpp index 14f63c3c62b1..b08c1e36878d 100644 --- a/kafka/include/userver/kafka/rebalance_types.hpp +++ b/kafka/include/userver/kafka/rebalance_types.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -30,7 +31,7 @@ struct TopicPartitionView final { using TopicPartitionBatchView = utils::span; /// @brief Callback invoked when a rebalance event occurs. -/// @warning The rebalance callback must be set before calling ConsumeScope::Start or after calling ConsumeScope::Stop . +/// @warning The rebalance callback should be set before calling ConsumerScope::Start or after ConsumerScope::Stop. /// The callback must not throw exceptions; any thrown exceptions will be caught and logged by the consumer /// implementation. /// @note The callback is invoked after the assign or revoke event has been successfully processed. diff --git a/kafka/src/kafka/consumer_component.cpp b/kafka/src/kafka/consumer_component.cpp index b22e00f66ac2..5040b44619b5 100644 --- a/kafka/src/kafka/consumer_component.cpp +++ b/kafka/src/kafka/consumer_component.cpp @@ -74,7 +74,10 @@ additionalProperties: false description: | Client identifier. May be an arbitrary string. - Optional, but you should set this property on each instance because it enables you to more easily correlate requests on the broker with the client instance which made it, which can be helpful in debugging and troubleshooting scenarios. + Optional, but you should set this property on each instance + because it enables you to more easily correlate requests on the broker + with the client instance which made it, + which can be helpful in debugging and troubleshooting scenarios. defaultDescription: userver topics: type: array @@ -146,6 +149,7 @@ additionalProperties: false - PLAINTEXT - SASL_SSL - SASL_PLAINTEXT + - SSL sasl_mechanisms: type: string description: | @@ -159,7 +163,7 @@ additionalProperties: false type: string description: | file or directory path to CA certificate(s) for verifying the broker's key. - Must be set if `security_protocol` equals `SASL_SSL`. + Must be set if `security_protocol` equals `SASL_SSL` or `SSL`. If set to `probe`, CA certificates are probed from the default certificates paths defaultDescription: none topic_metadata_refresh_interval: diff --git a/kafka/src/kafka/impl/broker_secrets.cpp b/kafka/src/kafka/impl/broker_secrets.cpp index 8a4ecff16eb0..5024375d35e5 100644 --- a/kafka/src/kafka/impl/broker_secrets.cpp +++ b/kafka/src/kafka/impl/broker_secrets.cpp @@ -8,12 +8,25 @@ USERVER_NAMESPACE_BEGIN namespace kafka::impl { Secret Parse(const formats::json::Value& doc, formats::parse::To) { - Secret secret{ - doc["brokers"].As(), - doc["username"].As(), - doc["password"].As()}; - - return secret; + auto brokers = doc["brokers"].As(); + if (doc.HasMember("username")) { + return Secret{ + std::move(brokers), + Secret::SaslCredentials{ + doc["username"].As(), + doc["password"].As(), + }}; + } else if (doc.HasMember("ssl_certificate_location")) { + return Secret{ + std::move(brokers), + Secret::SslCredentials{ + doc["ssl_certificate_location"].As(), + doc["ssl_key_location"].As(), + doc["ssl_key_password"].As>(), + }}; + } else { + return Secret{std::move(brokers)}; + } } BrokerSecrets::BrokerSecrets(const formats::json::Value& doc) { @@ -29,7 +42,7 @@ const Secret& BrokerSecrets::GetSecretByComponentName(const std::string& compone return secret_it->second; } - throw std::runtime_error{fmt::format("No secrets for {} component", component_name)}; + throw std::runtime_error{fmt::format("No secrets for '{}' component", component_name)}; } } // namespace kafka::impl diff --git a/kafka/src/kafka/impl/configuration.cpp b/kafka/src/kafka/impl/configuration.cpp index 145bd4cdb73f..b33cf7c6191c 100644 --- a/kafka/src/kafka/impl/configuration.cpp +++ b/kafka/src/kafka/impl/configuration.cpp @@ -28,7 +28,7 @@ namespace { // https://docs.confluent.io/platform/current/clients/librdkafka/html/rdkafka_8h.html#a06ade2ca41f32eb82c6f7e3d4acbe19f void KafkaLogCallback(const rd_kafka_t*, int level, const char* facility, const char* message) noexcept { try { - LOG(impl::convertRdKafkaLogLevelToLoggingLevel(level)) + LOG(impl::ConvertRdKafkaLogLevelToLoggingLevel(level)) << logging::LogExtra{{{"facility", facility}}} << message; } catch (const std::exception& e) { UASSERT_MSG(false, e.what()); @@ -107,10 +107,12 @@ SecurityConfiguration Parse(const yaml_config::YamlConfig& config, formats::pars static constexpr std::string_view kPlainTextProtocol{"PLAINTEXT"}; static constexpr std::string_view kSaslPlainTextProtocol{"SASL_PLAINTEXT"}; static constexpr std::string_view kSaslSSLProtocol{"SASL_SSL"}; + static constexpr std::string_view kSSLProtocol{"SSL"}; static constexpr std::array kSupportedSecurityProtocols{ kPlainTextProtocol, kSaslSSLProtocol, kSaslPlainTextProtocol, + kSSLProtocol, }; static constexpr std::array kSupportedSaslSecurityMechanisms{"PLAIN", "SCRAM-SHA-512"}; @@ -125,6 +127,13 @@ SecurityConfiguration Parse(const yaml_config::YamlConfig& config, formats::pars return security; } + if (protocol == kSSLProtocol) { + security.security_protocol.emplace(SecurityConfiguration::Ssl{ + /*ssl_ca_location=*/config["ssl_ca_location"].As(), + }); + return security; + } + const auto mechanism = config["sasl_mechanisms"].As(); if (!IsSupportedOption(kSupportedSaslSecurityMechanisms, mechanism)) { ThrowUnsupportedOption("SASL security mechanism", mechanism, kSupportedSaslSecurityMechanisms); @@ -244,22 +253,54 @@ void Configuration::SetSecurity(const SecurityConfiguration& security, const Sec utils::Visit( security.security_protocol, [](const SecurityConfiguration::Plaintext&) { LOG_INFO("Using PLAINTEXT security protocol"); }, - [this, &secrets](const SecurityConfiguration::SaslPlaintext& sasl_ssl) { + [this, &secrets](const SecurityConfiguration::SaslPlaintext& sasl_plaintext) { LOG_INFO("Using SASL_PLAINTEXT security protocol"); + UINVARIANT( + std::holds_alternative(secrets.credentials), + "For 'SASL_PLAINTEXT' security protocol, 'username' and 'password' are required in secdist " + "'kafka_settings'" + ); + const auto& credentials = std::get(secrets.credentials); + SetOption("security.protocol", "SASL_PLAINTEXT"); - SetOption("sasl.mechanism", sasl_ssl.security_mechanism); - SetOption("sasl.username", secrets.username); - SetOption("sasl.password", secrets.password); + SetOption("sasl.mechanism", sasl_plaintext.security_mechanism); + SetOption("sasl.username", credentials.username); + SetOption("sasl.password", credentials.password); }, [this, &secrets](const SecurityConfiguration::SaslSsl& sasl_ssl) { LOG_INFO("Using SASL_SSL security protocol"); + UINVARIANT( + std::holds_alternative(secrets.credentials), + "For 'SASL_SSL' security protocol, 'username' and 'password' are required in secdist " + "'kafka_settings'" + ); + const auto& credentials = std::get(secrets.credentials); + SetOption("security.protocol", "SASL_SSL"); - SetOption("sasl.mechanism", sasl_ssl.security_mechanism); - SetOption("sasl.username", secrets.username); - SetOption("sasl.password", secrets.password); SetOption("ssl.ca.location", sasl_ssl.ssl_ca_location); + SetOption("sasl.mechanism", sasl_ssl.security_mechanism); + SetOption("sasl.username", credentials.username); + SetOption("sasl.password", credentials.password); + }, + [this, &secrets](const SecurityConfiguration::Ssl& ssl) { + LOG_INFO("Using SSL security protocol"); + + UINVARIANT( + std::holds_alternative(secrets.credentials), + "For 'SSL' security protocol, 'ssl_certificate_location', 'ssl_key_location' and optionally " + "'ssl_key_password' are required in secdist 'kafka_settings'" + ); + + const auto& credentials = std::get(secrets.credentials); + SetOption("security.protocol", "SSL"); + SetOption("ssl.ca.location", ssl.ssl_ca_location); + SetOption("ssl.certificate.location", credentials.ssl_certificate_location); + SetOption("ssl.key.location", credentials.ssl_key_location); + if (credentials.ssl_key_password.has_value()) { + SetOption("ssl.key.password", *credentials.ssl_key_password); + } } ); } diff --git a/kafka/src/kafka/impl/consumer_impl.cpp b/kafka/src/kafka/impl/consumer_impl.cpp index f02cae940a30..ea06f13ffc39 100644 --- a/kafka/src/kafka/impl/consumer_impl.cpp +++ b/kafka/src/kafka/impl/consumer_impl.cpp @@ -296,7 +296,7 @@ void ConsumerImpl::ErrorCallback(rd_kafka_resp_err_t error, const char* reason, } void ConsumerImpl::LogCallback(const char* facility, const char* message, int log_level) { - LOG(convertRdKafkaLogLevelToLoggingLevel(log_level)) + LOG(ConvertRdKafkaLogLevelToLoggingLevel(log_level)) << logging::LogExtra{{{"kafka_callback", "log_callback"}, {"facility", facility}}} << message; } @@ -583,7 +583,7 @@ std::optional ConsumerImpl::TakeEventMessage(EventHolder&& event_holder UASSERT(IsMessageEvent(event_holder)); UASSERT(rd_kafka_event_message_count(event_holder.GetHandle()) == 1); - MessageHolder message{event_holder.release()}; + MessageHolder message{event_holder.Release()}; if (message->err != RD_KAFKA_RESP_ERR_NO_ERROR) { LOG_WARNING("Polled messages contains an error: {}", rd_kafka_err2str(message->err)); diff --git a/kafka/src/kafka/impl/holders.cpp b/kafka/src/kafka/impl/holders.cpp index 08dd30ee9cb6..86db0fde9a50 100644 --- a/kafka/src/kafka/impl/holders.cpp +++ b/kafka/src/kafka/impl/holders.cpp @@ -38,7 +38,7 @@ void HolderBase::reset() noexcept { } template Deleter> -T* HolderBase::release() noexcept { +T* HolderBase::Release() noexcept { return ptr_.release(); } @@ -70,14 +70,14 @@ rd_kafka_conf_t* ConfHolder::GetHandle() const noexcept { return impl_->conf.Get void ConfHolder::ForgetUnderlyingConf() noexcept { [[maybe_unused]] auto _ = impl_->conf.ptr_.release(); } -template -struct KafkaClientHolder::Impl { +template +struct KafkaClientHolder::Impl { explicit Impl(ConfHolder conf) : handle([conf = std::move(conf)]() mutable { ErrorBuffer err_buf; const auto rd_kafka_client_type = - client_type == ClientType::kConsumer ? RD_KAFKA_CONSUMER : RD_KAFKA_PRODUCER; + TClientType == ClientType::kConsumer ? RD_KAFKA_CONSUMER : RD_KAFKA_PRODUCER; #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" @@ -92,7 +92,7 @@ struct KafkaClientHolder::Impl { /// `rd_kafka_new` succeeds const auto type_str = [] { - switch (client_type) { + switch (TClientType) { case ClientType::kConsumer: return "consumer"; case ClientType::kProducer: @@ -106,7 +106,7 @@ struct KafkaClientHolder::Impl { conf.ForgetUnderlyingConf(); - if (client_type == ClientType::kConsumer) { + if (TClientType == ClientType::kConsumer) { /// Redirects main queue to consumer's queue. rd_kafka_poll_set_consumer(holder.GetHandle()); } @@ -114,7 +114,7 @@ struct KafkaClientHolder::Impl { return holder; }()), queue([this] { - switch (client_type) { + switch (TClientType) { case ClientType::kConsumer: return rd_kafka_queue_get_consumer(handle.GetHandle()); case ClientType::kProducer: @@ -128,24 +128,24 @@ struct KafkaClientHolder::Impl { HolderBase queue; }; -template -KafkaClientHolder::KafkaClientHolder(ConfHolder conf) : impl_(std::move(conf)) {} +template +KafkaClientHolder::KafkaClientHolder(ConfHolder conf) : impl_(std::move(conf)) {} -template -KafkaClientHolder::~KafkaClientHolder() = default; +template +KafkaClientHolder::~KafkaClientHolder() = default; -template -rd_kafka_t* KafkaClientHolder::GetHandle() const noexcept { +template +rd_kafka_t* KafkaClientHolder::GetHandle() const noexcept { return impl_->handle.GetHandle(); } -template -rd_kafka_queue_t* KafkaClientHolder::GetQueue() const noexcept { +template +rd_kafka_queue_t* KafkaClientHolder::GetQueue() const noexcept { return impl_->queue.GetHandle(); } -template -void KafkaClientHolder::reset() noexcept { +template +void KafkaClientHolder::reset() noexcept { impl_->queue.reset(); impl_->handle.reset(); } @@ -190,7 +190,7 @@ HeadersHolder::HeadersHolder(HeaderViews headers) : impl_{headers} {} rd_kafka_headers_t* HeadersHolder::GetHandle() const noexcept { return impl_->holder.GetHandle(); } -rd_kafka_headers_t* HeadersHolder::release() noexcept { return impl_->holder.release(); } +rd_kafka_headers_t* HeadersHolder::release() noexcept { return impl_->holder.Release(); } HeadersHolder::HeadersHolder(HeadersHolder&& other) noexcept = default; diff --git a/kafka/src/kafka/impl/log_level.cpp b/kafka/src/kafka/impl/log_level.cpp index 1136b6d0a9a2..c4af010b5632 100644 --- a/kafka/src/kafka/impl/log_level.cpp +++ b/kafka/src/kafka/impl/log_level.cpp @@ -22,7 +22,7 @@ enum class RdKafkaLogLevel { } // namespace -logging::Level convertRdKafkaLogLevelToLoggingLevel(int log_level) noexcept { +logging::Level ConvertRdKafkaLogLevelToLoggingLevel(int log_level) noexcept { const auto rd_kafka_log_level = static_cast(log_level); switch (rd_kafka_log_level) { diff --git a/kafka/src/kafka/impl/log_level.hpp b/kafka/src/kafka/impl/log_level.hpp index 48185a9a49d4..18cb613b351c 100644 --- a/kafka/src/kafka/impl/log_level.hpp +++ b/kafka/src/kafka/impl/log_level.hpp @@ -6,7 +6,7 @@ USERVER_NAMESPACE_BEGIN namespace kafka::impl { -logging::Level convertRdKafkaLogLevelToLoggingLevel(int log_level) noexcept; +logging::Level ConvertRdKafkaLogLevelToLoggingLevel(int log_level) noexcept; } diff --git a/kafka/src/kafka/impl/producer_impl.cpp b/kafka/src/kafka/impl/producer_impl.cpp index bda8fe940c73..a857226e3020 100644 --- a/kafka/src/kafka/impl/producer_impl.cpp +++ b/kafka/src/kafka/impl/producer_impl.cpp @@ -47,7 +47,7 @@ void ProducerImpl::ErrorCallback(rd_kafka_resp_err_t error, const char* reason, } void ProducerImpl::LogCallback(const char* facility, const char* message, int log_level) const { - LOG(convertRdKafkaLogLevelToLoggingLevel(log_level)) + LOG(ConvertRdKafkaLogLevelToLoggingLevel(log_level)) << logging::LogExtra{{{"kafka_callback", "log_callback"}, {"facility", facility}}} << message; } diff --git a/kafka/src/kafka/producer_component.cpp b/kafka/src/kafka/producer_component.cpp index dfd092b790d9..f8db7089d45e 100644 --- a/kafka/src/kafka/producer_component.cpp +++ b/kafka/src/kafka/producer_component.cpp @@ -48,7 +48,10 @@ additionalProperties: false description: | Client identifier. May be an arbitrary string. - Optional, but you should set this property on each instance because it enables you to more easily correlate requests on the broker with the client instance which made it, which can be helpful in debugging and troubleshooting scenarios. + Optional, but you should set this property on each instance + because it enables you to more easily correlate requests on the broker + with the client instance which made it, + which can be helpful in debugging and troubleshooting scenarios. defaultDescription: userver delivery_timeout: type: string @@ -108,6 +111,7 @@ additionalProperties: false - PLAINTEXT - SASL_SSL - SASL_PLAINTEXT + - SSL sasl_mechanisms: type: string description: | @@ -121,9 +125,10 @@ additionalProperties: false type: string description: | file or directory path to CA certificate(s) for verifying the broker's key. - Must be set if `security_protocol` equals `SASL_SSL`. + Must be set if `security_protocol` equals `SASL_SSL` or `SSL`. If set to `probe`, CA certificates are probed from the default certificates paths defaultDescription: none + rd_kafka_custom_options: type: object description: | diff --git a/kafka/tests/configuration_test.cpp b/kafka/tests/configuration_test.cpp index c18b1df723ec..ba7d0d228866 100644 --- a/kafka/tests/configuration_test.cpp +++ b/kafka/tests/configuration_test.cpp @@ -12,6 +12,8 @@ using namespace std::chrono_literals; class ConfigurationTest : public kafka::utest::KafkaCluster {}; +using ConfigurationDeathTest = ConfigurationTest; + } // namespace UTEST_F(ConfigurationTest, Producer) { @@ -129,7 +131,7 @@ UTEST_F(ConfigurationTest, ConsumerNonDefault) { EXPECT_EQ(configuration->GetOption("socket.keepalive.enable"), "true"); } -UTEST_F(ConfigurationTest, ProducerSecure) { +UTEST_F(ConfigurationTest, ProducerSaslSsl) { kafka::impl::ProducerConfiguration producer_configuration{}; producer_configuration.security.security_protocol = kafka::impl::SecurityConfiguration::SaslSsl{ /*security_mechanism=*/"SCRAM-SHA-512", @@ -137,8 +139,10 @@ UTEST_F(ConfigurationTest, ProducerSecure) { }; kafka::impl::Secret secrets; - secrets.username = kafka::impl::Secret::SecretType{"username"}; - secrets.password = kafka::impl::Secret::SecretType{"password"}; + secrets.credentials = kafka::impl::Secret::SaslCredentials{ + kafka::impl::Secret::SecretType{"username"}, + kafka::impl::Secret::SecretType{"password"}, + }; std::optional configuration; UEXPECT_NO_THROW(configuration.emplace(MakeProducerConfiguration("kafka-producer", producer_configuration, secrets)) @@ -151,15 +155,17 @@ UTEST_F(ConfigurationTest, ProducerSecure) { EXPECT_EQ(configuration->GetOption("ssl.ca.location"), "probe"); } -UTEST_F(ConfigurationTest, ProducerSecurePlaintext) { +UTEST_F(ConfigurationTest, ProducerSaslPlaintext) { kafka::impl::ProducerConfiguration producer_configuration{}; producer_configuration.security.security_protocol = kafka::impl::SecurityConfiguration::SaslPlaintext{ /*security_mechanism=*/"SCRAM-SHA-512", }; kafka::impl::Secret secrets; - secrets.username = kafka::impl::Secret::SecretType{"username"}; - secrets.password = kafka::impl::Secret::SecretType{"password"}; + secrets.credentials = kafka::impl::Secret::SaslCredentials{ + kafka::impl::Secret::SecretType{"username"}, + kafka::impl::Secret::SecretType{"password"}, + }; std::optional configuration; UEXPECT_NO_THROW(configuration.emplace(MakeProducerConfiguration("kafka-producer", producer_configuration, secrets)) @@ -171,7 +177,7 @@ UTEST_F(ConfigurationTest, ProducerSecurePlaintext) { EXPECT_EQ(configuration->GetOption("sasl.password"), "password"); } -UTEST_F(ConfigurationTest, ConsumerSecure) { +UTEST_F(ConfigurationTest, ConsumerSaslSsl) { kafka::impl::ConsumerConfiguration consumer_configuration{}; consumer_configuration.security.security_protocol = kafka::impl::SecurityConfiguration::SaslSsl{ /*security_mechanism=*/"SCRAM-SHA-512", @@ -179,8 +185,10 @@ UTEST_F(ConfigurationTest, ConsumerSecure) { }; kafka::impl::Secret secrets; - secrets.username = kafka::impl::Secret::SecretType{"username"}; - secrets.password = kafka::impl::Secret::SecretType{"password"}; + secrets.credentials = kafka::impl::Secret::SaslCredentials{ + kafka::impl::Secret::SecretType{"username"}, + kafka::impl::Secret::SecretType{"password"}, + }; std::optional configuration; UEXPECT_NO_THROW(configuration.emplace(MakeConsumerConfiguration("kafka-consumer", consumer_configuration, secrets)) @@ -193,15 +201,17 @@ UTEST_F(ConfigurationTest, ConsumerSecure) { EXPECT_EQ(configuration->GetOption("ssl.ca.location"), "/etc/ssl/cert.ca"); } -UTEST_F(ConfigurationTest, ConsumerSecurePlaintext) { +UTEST_F(ConfigurationTest, ConsumerSaslPlaintext) { kafka::impl::ConsumerConfiguration consumer_configuration{}; consumer_configuration.security.security_protocol = kafka::impl::SecurityConfiguration::SaslPlaintext{ /*security_mechanism=*/"SCRAM-SHA-512", }; kafka::impl::Secret secrets; - secrets.username = kafka::impl::Secret::SecretType{"username"}; - secrets.password = kafka::impl::Secret::SecretType{"password"}; + secrets.credentials = kafka::impl::Secret::SaslCredentials{ + kafka::impl::Secret::SecretType{"username"}, + kafka::impl::Secret::SecretType{"password"}, + }; std::optional configuration; UEXPECT_NO_THROW(configuration.emplace(MakeConsumerConfiguration("kafka-consumer", consumer_configuration, secrets)) @@ -213,6 +223,52 @@ UTEST_F(ConfigurationTest, ConsumerSecurePlaintext) { EXPECT_EQ(configuration->GetOption("sasl.password"), "password"); } +UTEST_F(ConfigurationTest, ProducerSsl) { + kafka::impl::ProducerConfiguration producer_configuration{}; + producer_configuration.security.security_protocol = kafka::impl::SecurityConfiguration::Ssl{ + /*ssl_ca_location=*/"/etc/ssl/ca.crt", + }; + + kafka::impl::Secret secrets; + secrets.credentials = kafka::impl::Secret::SslCredentials{ + kafka::impl::Secret::SecretType{"/etc/ssl/client.crt"}, + kafka::impl::Secret::SecretType{"/etc/ssl/client.key"}, + kafka::impl::Secret::SecretType{"password123"}, + }; + + std::optional configuration; + UEXPECT_NO_THROW(configuration.emplace(MakeProducerConfiguration("kafka-producer", producer_configuration, secrets)) + ); + + EXPECT_EQ(configuration->GetOption("security.protocol"), "ssl"); + EXPECT_EQ(configuration->GetOption("ssl.ca.location"), "/etc/ssl/ca.crt"); + EXPECT_EQ(configuration->GetOption("ssl.certificate.location"), "/etc/ssl/client.crt"); + EXPECT_EQ(configuration->GetOption("ssl.key.location"), "/etc/ssl/client.key"); + EXPECT_EQ(configuration->GetOption("ssl.key.password"), "password123"); +} + +UTEST_F(ConfigurationTest, ConsumerSSL) { + kafka::impl::ConsumerConfiguration consumer_configuration{}; + consumer_configuration.security.security_protocol = kafka::impl::SecurityConfiguration::Ssl{ + /*ssl_ca_location=*/"/etc/ssl/ca.crt", + }; + + kafka::impl::Secret secrets; + secrets.credentials = kafka::impl::Secret::SslCredentials{ + kafka::impl::Secret::SecretType{"/etc/ssl/client.crt"}, + kafka::impl::Secret::SecretType{"/etc/ssl/client.key"}, + }; + + std::optional configuration; + UEXPECT_NO_THROW(configuration.emplace(MakeConsumerConfiguration("kafka-consumer", consumer_configuration, secrets)) + ); + + EXPECT_EQ(configuration->GetOption("security.protocol"), "ssl"); + EXPECT_EQ(configuration->GetOption("ssl.ca.location"), "/etc/ssl/ca.crt"); + EXPECT_EQ(configuration->GetOption("ssl.certificate.location"), "/etc/ssl/client.crt"); + EXPECT_EQ(configuration->GetOption("ssl.key.location"), "/etc/ssl/client.key"); +} + UTEST_F(ConfigurationTest, IncorrectComponentName) { UEXPECT_THROW(MakeProducerConfiguration("producer"), std::runtime_error); UEXPECT_THROW(MakeConsumerConfiguration("consumer"), std::runtime_error); @@ -234,4 +290,178 @@ UTEST_F(ConfigurationTest, ConsumerResolveGroupId) { EXPECT_EQ(configuration->GetOption("group.id"), "test-group-pod-example-com"); } +UTEST_F_DEATH(ConfigurationDeathTest, ContradictorySecurityConfiguration) { + kafka::impl::ProducerConfiguration sasl_ssl{}; + sasl_ssl.security.security_protocol = kafka::impl::SecurityConfiguration::SaslSsl{ + /*security_mechanism=*/"SCRAM-SHA-512", + /*ssl_ca_location=*/"probe", + }; + kafka::impl::ProducerConfiguration sasl_plaintext{}; + sasl_plaintext.security.security_protocol = kafka::impl::SecurityConfiguration::SaslPlaintext{ + /*security_mechanism=*/"SCRAM-SHA-512", + }; + kafka::impl::ConsumerConfiguration ssl{}; + ssl.security.security_protocol = kafka::impl::SecurityConfiguration::Ssl{ + /*ssl_ca_location=*/"/etc/ssl/ca.crt", + }; + + kafka::impl::Secret secrets_none; + kafka::impl::Secret secrets_sasl; + secrets_sasl.credentials = kafka::impl::Secret::SaslCredentials{ + kafka::impl::Secret::SecretType{"username"}, + kafka::impl::Secret::SecretType{"password"}, + }; + kafka::impl::Secret secrets_ssl; + secrets_ssl.credentials = kafka::impl::Secret::SslCredentials{ + kafka::impl::Secret::SecretType{"/etc/ssl/client.crt"}, + kafka::impl::Secret::SecretType{"/etc/ssl/client.key"}, + kafka::impl::Secret::SecretType{"password123"}, + }; + + std::optional configuration; +#ifdef NDEBUG + UEXPECT_THROW( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_ssl, secrets_none)), std::exception + ); + UEXPECT_THROW( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_ssl, secrets_ssl)), std::exception + ); + UEXPECT_THROW( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_plaintext, secrets_none)), std::exception + ); + UEXPECT_THROW( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_plaintext, secrets_ssl)), std::exception + ); + UEXPECT_THROW( + configuration.emplace(MakeConsumerConfiguration("kafka-consumer", ssl, secrets_none)), std::exception + ); + UEXPECT_THROW( + configuration.emplace(MakeConsumerConfiguration("kafka-consumer", ssl, secrets_sasl)), std::exception + ); +#else + UEXPECT_DEATH( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_ssl, secrets_none)), + "For 'SASL_SSL' security protocol, 'username' and 'password' are required in secdist 'kafka_settings'" + ); + UEXPECT_DEATH( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_ssl, secrets_ssl)), + "For 'SASL_SSL' security protocol, 'username' and 'password' are required in secdist 'kafka_settings'" + ); + UEXPECT_DEATH( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_plaintext, secrets_none)), + "For 'SASL_PLAINTEXT' security protocol, 'username' and 'password' are required in secdist 'kafka_settings'" + ); + UEXPECT_DEATH( + configuration.emplace(MakeProducerConfiguration("kafka-producer", sasl_plaintext, secrets_ssl)), + "For 'SASL_PLAINTEXT' security protocol, 'username' and 'password' are required in secdist 'kafka_settings'" + ); + UEXPECT_DEATH( + configuration.emplace(MakeConsumerConfiguration("kafka-consumer", ssl, secrets_none)), + "For 'SSL' security protocol, 'ssl_certificate_location', 'ssl_key_location' and optionally 'ssl_key_password' " + "are required in secdist 'kafka_settings'" + ); + UEXPECT_DEATH( + configuration.emplace(MakeConsumerConfiguration("kafka-consumer", ssl, secrets_sasl)), + "For 'SSL' security protocol, 'ssl_certificate_location', 'ssl_key_location' and optionally 'ssl_key_password' " + "are required in secdist 'kafka_settings'" + ); +#endif +} + +UTEST_F(ConfigurationTest, BrokerSecrets) { + const auto make_kafka_settings = [](formats::json::Value component_settings) { + return formats::json::MakeObject( + "kafka_settings", formats::json::MakeObject("kafka-client", std::move(component_settings)) + ); + }; + + const auto only_brokers = formats::json::MakeObject("brokers", "localhost:1111"); + const auto sasl = formats::json::MakeObject( + "brokers", + "localhost:1111", + // + "username", + "user", + // + "password", + "pass" + // + ); + const auto ssl = formats::json::MakeObject( + "brokers", + "localhost:1111", + // + "ssl_certificate_location", + "/etc/ssl/client.crt", + // + "ssl_key_location", + "/etc/ssl/client.key", + // + "ssl_key_password", + "pass" + // + ); + const auto ssl_no_password = formats::json::MakeObject( + "brokers", + "localhost:1111", + // + "ssl_certificate_location", + "/etc/ssl/client.crt", + // + "ssl_key_location", + "/etc/ssl/client.key" + // + ); + + std::optional broker_secrets; + std::optional secret; + + { + UEXPECT_NO_THROW(broker_secrets.emplace(make_kafka_settings(only_brokers))); + UEXPECT_NO_THROW(secret.emplace(broker_secrets->GetSecretByComponentName("kafka-client"))); + ASSERT_TRUE(std::holds_alternative(secret->credentials)); + EXPECT_EQ(secret->brokers, "localhost:1111"); + } + { + UEXPECT_NO_THROW(broker_secrets.emplace(make_kafka_settings(sasl))); + UEXPECT_NO_THROW(secret.emplace(broker_secrets->GetSecretByComponentName("kafka-client"))); + ASSERT_TRUE(std::holds_alternative(secret->credentials)); + EXPECT_EQ(secret->brokers, "localhost:1111"); + EXPECT_EQ(std::get(secret->credentials).username.GetUnderlying(), "user"); + EXPECT_EQ(std::get(secret->credentials).password.GetUnderlying(), "pass"); + } + { + UEXPECT_NO_THROW(broker_secrets.emplace(make_kafka_settings(ssl))); + UEXPECT_NO_THROW(secret.emplace(broker_secrets->GetSecretByComponentName("kafka-client"))); + ASSERT_TRUE(std::holds_alternative(secret->credentials)); + EXPECT_EQ(secret->brokers, "localhost:1111"); + EXPECT_EQ( + std::get(secret->credentials).ssl_certificate_location.GetUnderlying(), + "/etc/ssl/client.crt" + ); + EXPECT_EQ( + std::get(secret->credentials).ssl_key_location.GetUnderlying(), + "/etc/ssl/client.key" + ); + EXPECT_EQ( + std::get(secret->credentials).ssl_key_password->GetUnderlying(), "pass" + ); + } + { + UEXPECT_NO_THROW(broker_secrets.emplace(make_kafka_settings(ssl_no_password))); + UEXPECT_NO_THROW(secret.emplace(broker_secrets->GetSecretByComponentName("kafka-client"))); + ASSERT_TRUE(std::holds_alternative(secret->credentials)); + EXPECT_EQ(secret->brokers, "localhost:1111"); + EXPECT_EQ( + std::get(secret->credentials).ssl_certificate_location.GetUnderlying(), + "/etc/ssl/client.crt" + ); + EXPECT_EQ( + std::get(secret->credentials).ssl_key_location.GetUnderlying(), + "/etc/ssl/client.key" + ); + EXPECT_FALSE(std::get(secret->credentials).ssl_key_password.has_value()); + } +} + USERVER_NAMESPACE_END diff --git a/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp b/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp index 86d0262ae59d..8ad91f6f0e1d 100644 --- a/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp +++ b/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp @@ -88,7 +88,7 @@ class KafkaCluster : public ::testing::Test { std::deque MakeProducers( std::size_t count, - std::function nameGenerator, + std::function name_generator, impl::ProducerConfiguration configuration = {} ); diff --git a/kafka/utest/src/kafka/utest/kafka_fixture.cpp b/kafka/utest/src/kafka/utest/kafka_fixture.cpp index 21942e363d48..737fc2d8b2d3 100644 --- a/kafka/utest/src/kafka/utest/kafka_fixture.cpp +++ b/kafka/utest/src/kafka/utest/kafka_fixture.cpp @@ -129,12 +129,12 @@ Producer KafkaCluster::MakeProducer(const std::string& name, impl::ProducerConfi std::deque KafkaCluster::MakeProducers( std::size_t count, - std::function nameGenerator, + std::function name_generator, impl::ProducerConfiguration configuration ) { std::deque producers; for (std::size_t i{0}; i < count; ++i) { - producers.emplace_back(utils::LazyPrvalue([&] { return MakeProducer(nameGenerator(i), configuration); })); + producers.emplace_back(utils::LazyPrvalue([&] { return MakeProducer(name_generator(i), configuration); })); } return producers; diff --git a/libraries/easy/CMakeLists.txt b/libraries/easy/CMakeLists.txt index 33f6da209c43..7b5c38799531 100644 --- a/libraries/easy/CMakeLists.txt +++ b/libraries/easy/CMakeLists.txt @@ -5,6 +5,7 @@ userver_module( SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" LINK_LIBRARIES userver::postgresql UTEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*_test.cpp" + DEPENDS core ) if(USERVER_BUILD_SAMPLES) diff --git a/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/client/impl/codegen_declarations.hpp b/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/client/impl/codegen_declarations.hpp new file mode 100644 index 000000000000..a1d2e41cde7a --- /dev/null +++ b/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/client/impl/codegen_declarations.hpp @@ -0,0 +1,6 @@ +#pragma once + +// This header contains all static includes for code-generated +// *_client.structs.usrv.pb.hpp files. +// +// Do not include this header in your code, use non-impl includes instead! diff --git a/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/impl/codegen_declarations.hpp b/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/impl/codegen_declarations.hpp new file mode 100644 index 000000000000..debf33293563 --- /dev/null +++ b/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/impl/codegen_declarations.hpp @@ -0,0 +1,8 @@ +#pragma once + +// This header contains all static includes for code-generated +// *_service.structs.usrv.pb.hpp files. +// +// Do not include this header in your code, use non-impl includes instead! + +#include diff --git a/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/stream.hpp b/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/stream.hpp new file mode 100644 index 000000000000..0b1ad5920775 --- /dev/null +++ b/libraries/grpc-proto-structs/include/userver/grpc-proto-structs/server/stream.hpp @@ -0,0 +1,110 @@ +#pragma once + +/// @file userver/grpc-proto-structs/server/stream.hpp +/// @brief proto-struct based server streaming interfaces + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace grpc_proto_structs::server { + +/// @brief proto-struct based Reader adapter +/// +/// This class is not thread-safe. +/// +/// If any method throws, further methods must not be called on the same stream. +/// @see @ref ugrpc::server::Reader. +template +class Reader { +public: + using RequestMessage = proto_structs::traits::CompatibleMessageType; + using ProtobufMessageReader = ugrpc::server::Reader; + + explicit Reader(ProtobufMessageReader& reader) : reader_{reader} {} + + /// @brief Await and read the next incoming message. + /// + /// Read protobuf message corresponding to `Request` with @ref ugrpc::server::Reader::Read + /// and construct `Request` from it. + /// + /// @see @ref ugrpc::server::Reader::Read method for details. + bool Read(Request& request) { + RequestMessage message; + if (reader_.Read(message)) { + proto_structs::MessageToStruct(message, request); + return true; + } + return false; + } + +private: + ProtobufMessageReader& reader_; +}; + +/// @brief proto-struct based Writer adapter +/// +/// This class is not thread-safe. +/// +/// If any method throws, further methods must not be called on the same stream. +/// @see @ref ugrpc::server::Writer. +template +class Writer { +public: + using ResponseMessage = proto_structs::traits::CompatibleMessageType; + using ProtobufMessageWriter = ugrpc::server::Writer; + + explicit Writer(ProtobufMessageWriter& writer) : writer_{writer} {} + + /// @{ + /// @brief Write the next outgoing message. + /// + /// Convert response to corresponding protobuf message and pass it to @ref ugrpc::server::Writer::Write. + /// + /// @see @ref ugrpc::server::Writer::Write method for details. + void Write(Response& response) { writer_.Write(proto_structs::StructToMessage(response)); } + + void Write(Response& response, const grpc::WriteOptions& options) { + writer_.Write(proto_structs::StructToMessage(response), options); + } + + void Write(Response&& response) { writer_.Write(proto_structs::StructToMessage(std::move(response))); } + + void Write(Response&& response, const grpc::WriteOptions& options) { + writer_.Write(proto_structs::StructToMessage(std::move(response)), options); + } + /// @} + +private: + ProtobufMessageWriter& writer_; +}; + +/// @brief proto-struct based ReaderWriter adapter +/// +/// If any method throws, further methods must not be called on the same stream. +/// +/// This class allows the following concurrent calls: +/// +/// - `Read`; +/// - `Write` +/// +/// and there can only be one Read and one Write in flight at a time. +/// +/// If any method throws, further methods must not be called on the same stream. +/// @see @ref ugrpc::server::ReaderWriter. +template +class ReaderWriter : public Reader, public Writer { +public: + using ProtobufMessageReaderWriter = ugrpc::server:: + ReaderWriter::RequestMessage, typename Writer::ResponseMessage>; + + explicit ReaderWriter(ProtobufMessageReaderWriter& reader_writer) + : Reader{reader_writer}, Writer{reader_writer} {} +}; + +} // namespace grpc_proto_structs::server + +USERVER_NAMESPACE_END diff --git a/libraries/grpc-proto-structs/src/grpc-proto-structs.cpp b/libraries/grpc-proto-structs/src/grpc-proto-structs.cpp new file mode 100644 index 000000000000..467e96908389 --- /dev/null +++ b/libraries/grpc-proto-structs/src/grpc-proto-structs.cpp @@ -0,0 +1 @@ +namespace grpc_proto_structs {} diff --git a/libraries/grpc-reflection/CMakeLists.txt b/libraries/grpc-reflection/CMakeLists.txt index dfac35d21cc6..f712c542e006 100644 --- a/libraries/grpc-reflection/CMakeLists.txt +++ b/libraries/grpc-reflection/CMakeLists.txt @@ -9,6 +9,7 @@ userver_module( SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" LINK_LIBRARIES ${PROJECT_NAME}-proto userver::grpc UTEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*_test.cpp" + DEPENDS grpc ) if(USERVER_BUILD_TESTS) diff --git a/libraries/proto-structs/codegen-tests/proto/box/autobox/cycles.proto b/libraries/proto-structs/codegen-tests/proto/box/autobox/cycles.proto new file mode 100644 index 000000000000..0cab10c82b76 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/box/autobox/cycles.proto @@ -0,0 +1,115 @@ +syntax = "proto3"; + +package box.autobox; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox;boxautobox"; + +message Self { + Self self = 1; +} + +message First { + Second c = 1; +} + +message Second { + Third c = 1; +} + +message Third { + First c = 1; +} + +message CycleEnd { + CycleStart cycle = 1; +} + +message CycleStart { + CycleEnd cycle = 1; + repeated CycleEnd not_boxed = 2; +} + +message Main1 { + message Inner { + ImBelowMain1 cycle = 1; + } + Main1.Inner inner = 1; +} + +message ImBelowMain1 { + Main1 cycle = 1; +} + +message IamAboveMain2 { + Main2 cycle = 1; +} + +message Main2 { + message Outer { + Main2.Inner inner = 1; + } + message Inner { + IamAboveMain2 cycle = 1; + } + Inner inner = 1; +} + +message NewCycle { + message Inner1 { + message InnerInner { + NewCycle.Inner2Below inner = 1; + } + InnerInner inner = 1; + } + message Inner2Below { + message InnerInner { + NewCycle.Inner1 inner = 1; + } + InnerInner i = 1; + } + Inner1 inner = 1; +} + +message NewCycle2 { + message Inner2Above { + message InnerInner { + NewCycle2.Inner1 inner = 1; + } + NewCycle2.Inner2Above.InnerInner i = 1; + } + message Inner1 { + message InnerInner { + NewCycle2.Inner2Above inner = 1; + } + NewCycle2.Inner1.InnerInner inner = 1; + } + NewCycle2.Inner1 inner = 1; +} + +message OptionalSelf { + optional OptionalSelf self = 1; +} + +message OptionalDouble { + message First { + optional Second c = 1; + } + + message Second { + optional First c = 1; + } +} + +message OptionalTriple { + message First { + optional Second c = 1; + } + + message Second { + optional Third c = 1; + } + + message Third { + optional First c = 1; + } +} diff --git a/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_nested.proto b/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_nested.proto new file mode 100644 index 000000000000..00adf5c1c53d --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_nested.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package box.autobox; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox;boxautobox"; + +// There is no dependency cycle, just need to define `Bar` before `Foo`. +message DependencyOnNested { + message Foo { + Bar.Nested field = 1; + } + + message Bar { + message Nested {} + } +} + +// Same as above, but with double nesting. +message DependencyOnNestedNested { + message Foo { + Bar.Nested.Nested2 field = 1; + } + + message Bar { + message Nested { + message Nested2 {} + } + } +} + +// There is no dependency cycle, but still need to define `Bar` before `Foo`, despite the field being a `std::vector`. +// This is because we need the definition of `Bar` to mention `Bar::Nested`. +message RepeatedDependencyOnNested { + message Foo { + repeated Bar.Nested field = 1; + } + + message Bar { + message Nested {} + } +} + +// There is a dependency cycle `Foo <-> Bar`, and the dependency `Foo.field -> Bar.Nested` is unbreakable using fwd, +// because `Bar` needs to be defined to mention `Bar::Nested`. +message CyclicDependencyOnNested { + message Foo { + // Should NOT be boxed, it is meaningless. Instead, definition of `Foo` should be moved after `Bar`. + Bar.Nested field = 1; + } + + message Bar { + message Nested {} + + // Should be boxed to break the cycle. + Foo field = 1; + } +} + +// Same as above. Boxing `Foo.field` is meaningless, because `Bar` needs to be defined to mention `Bar::Nested`. +message RepeatedCyclicDependencyOnNested { + message Foo { + // Should NOT be boxed, it is meaningless. Instead, definition of `Foo` should be moved after `Bar`. + repeated Bar.Nested field = 1; + } + + message Bar { + message Nested {} + + // Should be boxed to break the cycle. + Foo field = 1; + } +} + +// Sanity check: for WEAK dependencies on direct neighbours (not their nested types), a forward declaration is enough. +message VectorsDoNotCreateCycles { + message Foo { + repeated Bar field = 1; + } + + message Bar { + repeated Foo field = 1; + } +} diff --git a/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_self.proto b/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_self.proto new file mode 100644 index 000000000000..061daad3c479 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/box/autobox/dependency_on_self.proto @@ -0,0 +1,54 @@ +syntax = "proto3"; + +package box.autobox; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox;boxautobox"; + +message DependencyOnSelfDirect { + // Should be boxed. + DependencyOnSelfDirect field = 1; +} + +message DependencyOnSelfNested { + message Nested { + // Should be boxed. + DependencyOnSelfNested field = 1; + } +} + +message DependencyOnSelfOptional { + // Should be boxed. + optional DependencyOnSelfOptional field = 1; +} + +message DependencyOnSelfRepeated { + // Should not be boxed: the outer struct is already declared at this point. + repeated DependencyOnSelfRepeated field = 1; +} + +message DependencyWithinSelf { + message A {} + + message B { + // Should not be boxed. + // + // When accounting edges at the namespace level, we should say that there is no edge from DependencyWithinSelf + // to itself, even though there is a dependency from within DependencyWithinSelf to within DependencyWithinSelf. + A field = 1; + } +} + +message DependencyOnSelfCollateral { + message A {} + + message B { + // Should not be boxed. + // + // Although there is an edge from within DependencyOnSelfCollateral to DependencyOnSelfCollateral, + // this dependency is strictly within DependencyOnSelfCollateral and is irrelevant when sorting dependencies + // at namespace scope. + A field = 1; + } + + optional DependencyOnSelfCollateral field = 1; +} diff --git a/libraries/proto-structs/codegen-tests/proto/box/autobox/unbreakable_cycle.proto b/libraries/proto-structs/codegen-tests/proto/box/autobox/unbreakable_cycle.proto new file mode 100644 index 000000000000..6810a2b81ef5 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/box/autobox/unbreakable_cycle.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package box.autobox; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/autobox;boxautobox"; + +message UnbreakableCycle { + message A { + message Inner {} + + B.Inner b = 1; + } + + message B { + message Inner {} + + optional A.Inner a = 1; + } +} diff --git a/libraries/proto-structs/codegen-tests/proto/box/options/cycles.proto b/libraries/proto-structs/codegen-tests/proto/box/options/cycles.proto new file mode 100644 index 000000000000..a02a21899603 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/box/options/cycles.proto @@ -0,0 +1,121 @@ +syntax = "proto3"; + +package box.options; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/options;boxoptions"; + +import "userver/structs/annotations.proto"; + +message MyMap { + map self = 1 [(userver.structs.field).indirect = true]; +} + +message Self { + Self self = 1 [(userver.structs.field).indirect = true]; +} + +message First { + Second c = 1; +} + +message Second { + Third c = 1; +} + +message Third { + First c = 1 [(userver.structs.field).indirect = true]; +} + +message CycleEnd { + CycleStart cycle = 1; +} + +message CycleStart { + CycleEnd cycle = 1 [(userver.structs.field).indirect = true]; + repeated CycleEnd not_boxed = 2; +} + +message Main1 { + message Inner { + ImBelowMain1 cycle = 1 [(userver.structs.field).indirect = true]; + } + Main1.Inner inner = 1; +} + +message ImBelowMain1 { + Main1 cycle = 1; +} + +message IamAboveMain2 { + Main2 cycle = 1; +} + +message Main2 { + message Outer { + Main2.Inner inner = 1; + } + message Inner { + IamAboveMain2 cycle = 1 [(userver.structs.field).indirect = true]; + } + Inner inner = 1; +} + +message NewCycle { + message Inner1 { + message InnerInner { + NewCycle.Inner2Below inner = 1; + } + InnerInner inner = 1; + } + message Inner2Below { + message InnerInner { + NewCycle.Inner1 inner = 1 [(userver.structs.field).indirect = true]; + } + InnerInner i = 1; + } + Inner1 inner = 1; +} + +message NewCycle2 { + message Inner2Above { + message InnerInner { + NewCycle2.Inner1 inner = 1 [(userver.structs.field).indirect = true]; + } + NewCycle2.Inner2Above.InnerInner i = 1; + } + message Inner1 { + message InnerInner { + NewCycle2.Inner2Above inner = 1; + } + NewCycle2.Inner1.InnerInner inner = 1; + } + NewCycle2.Inner1 inner = 1; +} + +message OptionalSelf { + optional OptionalSelf self = 1 [(userver.structs.field).indirect = true]; +} + +message OptionalDouble { + message First { + optional Second c = 1 [(userver.structs.field).indirect = true]; + } + + message Second { + optional First c = 1; + } +} + +message OptionalTriple { + message First { + optional Second c = 1; + } + + message Second { + optional Third c = 1; + } + + message Third { + optional First c = 1 [(userver.structs.field).indirect = true]; + } +} diff --git a/libraries/proto-structs/codegen-tests/proto/box/options/initialization.proto b/libraries/proto-structs/codegen-tests/proto/box/options/initialization.proto new file mode 100644 index 000000000000..5b9cd0970c3a --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/box/options/initialization.proto @@ -0,0 +1,53 @@ +// Fields containing a message that is defined in the same file must be left uninitialized. +// Otherwise, the compiler suddenly requires definitions of all mentioned forward-declared types: +// +// ```cpp +// struct Foo; +// +// struct Bar { +// std::vector foo; // ok +// }; +// +// struct Baz { +// Bar bar{}; // error: std::vector needs Foo?! +// }; +// ``` +// +// Compare: +// * https://godbolt.org/z/1796cr4vP +// * https://godbolt.org/z/fbxc8cz8v +// +// Reason for this behavior is that if an initialization is given, then the compiler must immediately check +// that it is valid, and the constructor of `std::vector` requires that the item type is complete. + +syntax = "proto3"; + +package box.options; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/box/options;boxoptions"; + +import "userver/structs/annotations.proto"; + +message DependsOnNotDefinedYet { + NotDefinedYet plain_field = 1 [(userver.structs.field).indirect = true]; + optional NotDefinedYet optional_field = 2 [(userver.structs.field).indirect = true]; + repeated NotDefinedYet vector_field = 3; + map map_field = 4 [(userver.structs.field).indirect = true]; + + oneof oneof { + NotDefinedYet oneof_field = 5 [(userver.structs.field).indirect = true]; + } +} + +message IndirectlyDependsOnNotDefinedYet { + DependsOnNotDefinedYet plain_field = 1 [(userver.structs.field).indirect = true]; + optional DependsOnNotDefinedYet optional_field = 2 [(userver.structs.field).indirect = true]; + repeated DependsOnNotDefinedYet vector_field = 3; + map map_field = 4 [(userver.structs.field).indirect = true]; + + oneof oneof { + DependsOnNotDefinedYet oneof_field = 5 [(userver.structs.field).indirect = true]; + } +} + +message NotDefinedYet {} diff --git a/libraries/proto-structs/codegen-tests/proto/enums/names.proto b/libraries/proto-structs/codegen-tests/proto/enums/names.proto new file mode 100644 index 000000000000..3ee691358bf3 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/enums/names.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/enums;enums"; + +package enums; + +// Should be left as-is. Note: unscoped, unprefixed enum values are not recommended. +enum Unprefixed { + UNSPECIFIED = 0; + FOO_VAR = 1; + BAR_VAR = 2; +} + +// Prefix should be cut when converted to a C++ enum class. +enum AllowedCuts { + ALLOWED_CUTS_UNSPECIFIED = 0; + ALLOWED_CUTS_FOO_VAR = 1; + ALLOWED_CUTS_DIGITS1 = 2; +} + +enum DisallowedCuts { + // Camel-case prefix is not cut away. The proper name should be `DISALLOWED_CUTS_UNSPECIFIED`. + DisallowedCuts_UNKNOWN = 0; + // Same value name as the enum name. + DISALLOWED_CUTS = 1; + // `1` is not a valid identifier. + DISALLOWED_CUTS1 = 2; + // `1` is not a valid identifier. + DISALLOWED_CUTS_2 = 3; + // Camel-case prefix is not cut away. The naming conventions recommend prefixing with an uppercase enum name. + DisallowedCutsCamel = 4; +} + +// This is a common trick that allows to achieve desired scoping for various languages. +// The message contains nothing except the nested enum. Outer message name should be `Enum`. +// When converting to a C++ enum class, remove the nested enum. +message NestedTrickEnum { + enum NestedTrick { + UNSPECIFIED = 0; + FOO_VAR = 1; + BAR_VAR = 2; + } +} + +enum WithoutPrefix { + BarVar = 0; + Bar_Foo = 1; + Bar_foo1 = 2; + Bar_FooQux = 3; + Bar_FooQuX1 = 4; +} diff --git a/libraries/proto-structs/codegen-tests/proto/maps/basic.proto b/libraries/proto-structs/codegen-tests/proto/maps/basic.proto new file mode 100644 index 000000000000..3bf22012d508 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/maps/basic.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/maps;maps"; + +package maps; + +message Basic { + map string_int = 1; + map string_string = 2; + map int_int = 3; + map int_string = 4; +} diff --git a/libraries/proto-structs/codegen-tests/proto/not_recommended_field_names/basic.proto b/libraries/proto-structs/codegen-tests/proto/not_recommended_field_names/basic.proto new file mode 100644 index 000000000000..40dc5fd58073 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/not_recommended_field_names/basic.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package not_recommended_field_names; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/not_recommended_field_names;not_recommended_field_names"; + +message Basic { + int32 protected = 1; + int32 final = 2; + int32 void__ = 3; + int32 float = 4; + int32 typeid = 5; + int32 new = 6; + int32 double = 7; + // Not generated by a plugin. + int32 major = 8; + int32 minor = 9; +} + +message Upper { + int32 XYandexUid = 1; + int32 Float = 2; + int32 New = 3; +} diff --git a/libraries/proto-structs/codegen-tests/proto/oneof/basic.proto b/libraries/proto-structs/codegen-tests/proto/oneof/basic.proto new file mode 100644 index 000000000000..3267d7d289ca --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/oneof/basic.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; + +package oneof; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof;oneof"; + +message Message1 { + string field = 1; +} + +enum Enum1 { + ENUM1_UNSPECIFIED = 0; + ENUM1_FOO = 1; +} + +message Parent { + optional string field_before = 1; + + message Message2 { + string field = 1; + } + + enum Enum2 { + ENUM2_UNSPECIFIED = 0; + ENUM2_FOO = 1; + } + + oneof lowercase { + int64 integer = 3; + string string = 2; + Message1 message1 = 5; + Message2 message2 = 6; + Enum1 enum1 = 7; + Enum2 enum2 = 8; + } + + oneof Uppercase { + string foo = 9; + int64 bar = 10; + } + + oneof single_field_oneof { + string single = 11; + } +} diff --git a/libraries/proto-structs/codegen-tests/proto/oneof/custom_oneof_type_name.proto b/libraries/proto-structs/codegen-tests/proto/oneof/custom_oneof_type_name.proto new file mode 100644 index 000000000000..e416241922fd --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/oneof/custom_oneof_type_name.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package oneof; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof;oneof"; + +import "userver/structs/annotations.proto"; + +// Demonstrates how to resolve a proto structs naming conflict by manually overriding the generated oneof type name. +message NameConflict1 { + message Nested { + // By default, this will generate a type named `FooId` and a field `foo_id`. + // Then the type of `simple_id` will be resolved to C++ type of oneof, not to the intended message below. + oneof foo_id { + // This is a manual work-around to the (hopefully) rare naming conflict. + option (userver.structs.oneof).generated_type_name = "FooIdCustom"; + + FooId simple_id = 1; + } + } + + message FooId {} +} + +// Similar to the above except for another naming scheme for the generated type. +message NameConflict2 { + message Nested { + // By default, this will generate a type named `TFooId` and a field `FooId`. + // Then the type of `simple_id` will be resolved to C++ type of oneof, not to the intended message below. + oneof FooId { + // This is a manual work-around to the (hopefully) rare naming conflict. + option (userver.structs.oneof).generated_type_name = "TFooIdCustom"; + + TFooId simple_id = 1; + } + } + + message TFooId {} +} diff --git a/libraries/proto-structs/codegen-tests/proto/oneof/proto2.proto b/libraries/proto-structs/codegen-tests/proto/oneof/proto2.proto new file mode 100644 index 000000000000..e07b4a4f8152 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/oneof/proto2.proto @@ -0,0 +1,53 @@ +syntax = "proto2"; + +package oneof; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof;oneof"; + +message Proto2 { + optional string field_before = 1; + + message Message { + required string field = 1; + } + + enum Enum { + ENUM_UNSPECIFIED = 0; + ENUM_FOO = 1; + } + + oneof oneof { + int64 integer = 3; + string string = 2; + Message message = 5; + Enum enum = 7; + + // https://protobuf.dev/programming-guides/proto2/#groups + group Group = 8 { + message MessageInGroup { + required string foo = 1; + } + + enum EnumInGroup { + ENUM_IN_GROUP_UNSPECIFIED = 0; + ENUM_IN_GROUP_FOO = 1; + } + + required int64 x = 1; + optional string y = 2; + + oneof group_oneof { + Message z = 3; + Enum w = 4; + } + } + } + + oneof single_field_oneof { + string single = 9; + } + + optional Group.MessageInGroup message_from_group = 10; + repeated Group.EnumInGroup enum_from_group = 11; + optional bool opt_bool = 12; +} diff --git a/libraries/proto-structs/codegen-tests/proto/simple/base.proto b/libraries/proto-structs/codegen-tests/proto/simple/base.proto new file mode 100644 index 000000000000..89cb1282169a --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/simple/base.proto @@ -0,0 +1,59 @@ +syntax = "proto3"; + +package simple; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple;simple"; + +import "simple/subdirectory/subdirectory.proto"; +import "simple/subdirectory/separate_enum.proto"; + +message SimpleStruct { + int32 some_integer = 1; + optional string some_text = 2; + bool is_condition = 3; + + oneof something { + int64 foo = 4; + string bar = 5; + }; + + repeated bytes some_bytes = 6; + + message NestedStruct { + message NestedStruct2 { + + enum InnerEnum1 { + FOO_VAL = 0; + BAR_VAL = 1; + } + + string swag2 = 1; + InnerEnum1 swag3 = 2; + } + + string swag = 1; + } + + enum InnerEnum2 { + FOO_VAL = 0; + BAR_VAL = 1; + } + + InnerEnum2 inner_enum = 7; + string digit1u = 8; + NestedStruct nested = 9; + optional NestedStruct optional_nested = 10; +} + +message SecondStruct { + string str = 1; + + simple.subdirectory.SubdirStruct subdir_struct = 2; + simple.subdirectory.MyEnum subdir_enum = 3; + simple.subdirectory.SeparateEnum separate_enum = 4; +} + +enum GlobalEnum { + FOO_VAL = 0; + BAR_VAL = 1; +} diff --git a/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/separate_enum.proto b/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/separate_enum.proto new file mode 100644 index 000000000000..dab0e85e3617 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/separate_enum.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package simple.subdirectory; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/subdirectory;simplesubdirectory"; + +enum SeparateEnum { + ValZero = 0; + ValOne = 1; +} diff --git a/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/subdirectory.proto b/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/subdirectory.proto new file mode 100644 index 000000000000..40fff1c5c667 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/subdirectory.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package simple.subdirectory; + +option go_package = "a.yandex-team.ru/taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/subdirectory;simplesubdirectory"; + +message SubdirStruct { + string name = 1; +} + +enum MyEnum { + Zero = 0; + One = 1; +} diff --git a/libraries/proto-structs/codegen-tests/src/box/autobox/cycles_test.cpp b/libraries/proto-structs/codegen-tests/src/box/autobox/cycles_test.cpp new file mode 100644 index 000000000000..80c4fbff257c --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/box/autobox/cycles_test.cpp @@ -0,0 +1,112 @@ +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +TEST(AutoboxCycles, Self) { + namespace ss = box::autobox::structs; + AssertFieldType>(); + AssertFieldCount(); +} + +TEST(AutoboxCycles, CycleLenIsThree) { + namespace ss = box::autobox::structs; + AssertFieldType>(); + AssertFieldType>(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(AutoboxCycles, box) { + namespace ss = box::autobox::structs; + AssertFieldType>(); + AssertFieldType>(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(AutoboxCycles, Main) { + namespace ss = box::autobox::structs; + AssertFieldType(); + AssertFieldType>(); + AssertFieldType>(); + AssertFieldType(); + AssertFieldType>(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(AutoboxCycles, NewCycle) { + namespace ss = box::autobox::structs; + AssertFieldType(); + AssertFieldCount(); + AssertFieldType(); + AssertFieldType>(); + AssertFieldType(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(AutoboxCycles, NewCycle2) { + namespace ss = box::autobox::structs; + AssertFieldType(); + AssertFieldCount(); + AssertFieldType(); + AssertFieldType>(); + AssertFieldType(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(AutoboxCycles, OptionalSelf) { + namespace ss = box::autobox::structs; + { + // Boost.Pfr refuses to process a struct in which first field is constructible from the same struct. + // So we can't use Boost.Pfr to check that `OptionalSelf` has exactly 1 field. + const auto [_] = ss::OptionalSelf{}; + } + AssertFieldType>>(); +} + +TEST(AutoboxCycles, OptionalDouble) { + namespace ss = box::autobox::structs; + AssertFieldCount(); + AssertFieldType>>(); + AssertFieldCount(); + AssertFieldType>>(); +} + +TEST(AutoboxCycles, OptionalTriple) { + namespace ss = box::autobox::structs; + AssertFieldCount(); + AssertFieldType>>(); + AssertFieldCount(); + AssertFieldType>>(); + AssertFieldCount(); + AssertFieldType>>(); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_nested_test.cpp b/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_nested_test.cpp new file mode 100644 index 000000000000..ac09a90c3224 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_nested_test.cpp @@ -0,0 +1,58 @@ +#include + +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +TEST(BoxDependencyOnNested, NonCyclic) { + using Scope = box::autobox::structs::DependencyOnNested; + AssertFieldCount(); + AssertFieldType(); +} + +TEST(BoxDependencyOnNested, NonCyclicDoublyNested) { + using Scope = box::autobox::structs::DependencyOnNestedNested; + AssertFieldCount(); + AssertFieldType(); +} + +TEST(BoxDependencyOnNested, RepeatedNonCyclic) { + using Scope = box::autobox::structs::RepeatedDependencyOnNested; + AssertFieldCount(); + AssertFieldType>(); +} + +TEST(BoxDependencyOnNested, Cyclic) { + using Scope = box::autobox::structs::CyclicDependencyOnNested; + + AssertFieldCount(); + AssertFieldType(); + + AssertFieldCount(); + AssertFieldType>(); +} + +TEST(BoxDependencyOnNested, RepeatedCyclic) { + using Scope = box::autobox::structs::RepeatedCyclicDependencyOnNested; + + AssertFieldCount(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldType>(); +} + +TEST(BoxDependencyOnNested, VectorsDoNotCreateCycles) { + using Scope = box::autobox::structs::VectorsDoNotCreateCycles; + + AssertFieldCount(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldType>(); +} diff --git a/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_self_test.cpp b/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_self_test.cpp new file mode 100644 index 000000000000..764718d4a06b --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/box/autobox/dependency_on_self_test.cpp @@ -0,0 +1,60 @@ +#include + +#include +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +TEST(BoxDependencyOnSelf, Direct) { + using Scope = box::autobox::structs::DependencyOnSelfDirect; + // Boost.Pfr refuses to process a struct in which first field is constructible from the same struct. + // So we can't use Boost.Pfr to check that `Scope` has exactly 1 field. + AssertFieldType>(); +} + +TEST(BoxDependencyOnSelf, Nested) { + using Scope = box::autobox::structs::DependencyOnSelfNested; + AssertFieldCount(); + AssertFieldType>(); +} + +TEST(BoxDependencyOnSelf, Optional) { + using Scope = box::autobox::structs::DependencyOnSelfOptional; + { + // Boost.Pfr refuses to process a struct in which first field is constructible from the same struct. + // So we can't use Boost.Pfr to check that `Scope` has exactly 1 field. + const auto [_] = Scope{}; + } + AssertFieldType>>(); +} + +TEST(BoxDependencyOnSelf, Repeated) { + using Scope = box::autobox::structs::DependencyOnSelfRepeated; + AssertFieldCount(); + AssertFieldType>(); +} + +TEST(BoxDependencyOnSelf, DependencyWithin) { + using Scope = box::autobox::structs::DependencyWithinSelf; + AssertFieldCount(); + AssertFieldType(); +} + +TEST(BoxDependencyOnSelf, DependencyOnSelfCollateral) { + using Scope = box::autobox::structs::DependencyOnSelfCollateral; + + { + // Boost.Pfr refuses to process a struct in which first field is constructible from the same struct. + // So we can't use Boost.Pfr to check that `Scope` has exactly 1 field. + const auto [_] = Scope{}; + } + AssertFieldType>>(); + + AssertFieldCount(); + AssertFieldType(); +} diff --git a/libraries/proto-structs/codegen-tests/src/box/autobox/unbreakable_cycle_test.cpp b/libraries/proto-structs/codegen-tests/src/box/autobox/unbreakable_cycle_test.cpp new file mode 100644 index 000000000000..5e1e4c0cf3bb --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/box/autobox/unbreakable_cycle_test.cpp @@ -0,0 +1,24 @@ +#include + +#include + +#include +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +TEST(AutoboxUnbreakableCycle, SimpleCycle) { + namespace ss = box::autobox::structs; + + static_assert(boost::pfr::tuple_size() == 1); + static_assert(std::same_as); + + static_assert(boost::pfr::tuple_size() == 1); + static_assert(std::same_as); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/box/options/cycles_test.cpp b/libraries/proto-structs/codegen-tests/src/box/options/cycles_test.cpp new file mode 100644 index 000000000000..11609c03e9de --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/box/options/cycles_test.cpp @@ -0,0 +1,119 @@ +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +TEST(OptionsCycles, Self) { + namespace ss = box::options::structs; + AssertFieldType>(); + AssertFieldCount(); +} + +TEST(OptionsCycles, MyMap) { + namespace ss = box::options::structs; + + AssertFieldType>>(); + + AssertFieldCount(); +} + +TEST(OptionsCycles, CycleLenIsThree) { + namespace ss = box::options::structs; + AssertFieldType(); + AssertFieldType(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(OptionsCycles, box) { + namespace ss = box::options::structs; + AssertFieldType>(); + AssertFieldType>(); + AssertFieldType(); + + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(OptionsCycles, Main) { + namespace ss = box::options::structs; + AssertFieldType(); + AssertFieldType>(); + AssertFieldType(); + AssertFieldType(); + AssertFieldType>(); + AssertFieldType(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(OptionsCycles, NewCycle) { + namespace ss = box::options::structs; + AssertFieldType(); + AssertFieldCount(); + AssertFieldType(); + AssertFieldType(); + AssertFieldType(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(OptionsCycles, NewCycle2) { + namespace ss = box::options::structs; + AssertFieldType(); + AssertFieldCount(); + AssertFieldType(); + AssertFieldType(); + AssertFieldType(); + AssertFieldType>(); + + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); + AssertFieldCount(); +} + +TEST(OptionsCycles, OptionalSelf) { + namespace ss = box::options::structs; + // Boost.Pfr actually refuses to process a struct, in which first field is constructible from the same struct. +#if 0 + AssertFieldCount(); +#endif + AssertFieldType>>(); +} + +TEST(OptionsCycles, OptionalDouble) { + namespace ss = box::options::structs; + AssertFieldCount(); + AssertFieldType>>(); + AssertFieldCount(); + AssertFieldType>(); +} + +TEST(OptionsCycles, OptionalTriple) { + namespace ss = box::options::structs; + AssertFieldCount(); + AssertFieldType>(); + AssertFieldCount(); + AssertFieldType>(); + AssertFieldCount(); + AssertFieldType>>(); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/box/options/initialization_test.cpp b/libraries/proto-structs/codegen-tests/src/box/options/initialization_test.cpp new file mode 100644 index 000000000000..74d78301face --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/box/options/initialization_test.cpp @@ -0,0 +1,64 @@ +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace structs = box::options::structs; + +USERVER_NAMESPACE_BEGIN + +TEST(BoxInitialization, NotDefinedYet) { AssertFieldCount(); } + +namespace { + +template +void CheckDependsOnNotDefinedYet() { + AssertFieldCount(); + AssertFieldType>(); + AssertFieldType>>(); + AssertFieldType>(); + AssertFieldType>>(); + AssertFieldType(); + + static_assert(std::same_as< + proto_structs::OneofAlternativeType<0, typename MessageType::Oneof>, + utils::Box>); + + MessageType test_struct{ + .plain_field = FieldType{}, + .optional_field = FieldType{}, + .vector_field = {FieldType{}}, + .map_field = {{{"key", FieldType{}}}}, + .oneof = MessageType::Oneof::make_oneof_field(FieldType{}), + }; + + auto test_message = proto_structs::StructToMessage(test_struct); + auto test_struct_again = proto_structs::MessageToStruct(test_message); + + // TODO re-enable test once operator== is available + // ASSERT_EQ(test_struct_again, test_struct); +} + +} // namespace + +TEST(BoxInitialization, DependsOnNotDefinedYet) { + CheckDependsOnNotDefinedYet(); +} + +TEST(BoxInitialization, IndirectlyDependsOnNotDefinedYet) { + CheckDependsOnNotDefinedYet(); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/enums/names_test.cpp b/libraries/proto-structs/codegen-tests/src/enums/names_test.cpp new file mode 100644 index 000000000000..44bf651dc794 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/enums/names_test.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +TEST(EnumNames, Unprefixed) { + using Enum = enums::structs::Unprefixed; + static_assert(std::is_enum_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); +} + +TEST(EnumNames, AllowedCuts) { + using Enum = enums::structs::AllowedCuts; + static_assert(std::is_enum_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); +} + +TEST(EnumNames, DisallowedCuts) { + using Enum = enums::structs::DisallowedCuts; + static_assert(std::is_enum_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); +} + +TEST(EnumNames, NestedTrick) { + using Enum = enums::structs::NestedTrickEnum; + static_assert(std::is_enum_v); + static_assert(std::is_same_v); +} + +TEST(EnumNames, WithoutPrefix) { + using Enum = enums::structs::WithoutPrefix; + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/maps/basic_test.cpp b/libraries/proto-structs/codegen-tests/src/maps/basic_test.cpp new file mode 100644 index 000000000000..6e06d27d0746 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/maps/basic_test.cpp @@ -0,0 +1,53 @@ +#include + +#include + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +TEST(MapsBasic, FundamentalTypes) { + using Struct = maps::structs::Basic; + static_assert(std::is_same_v>); + static_assert(std::is_same_v>); + static_assert(std::is_same_v>); + static_assert(std::is_same_v>); + + Struct s{}; + s.int_int = {{1, 2}, {3, 4}}; + s.int_string = {{1, "2"}, {3, "4"}}; + s.string_int = {{"1", 2}, {"3", 4}}; + s.string_string = {{"1", "2"}, {"3", "4"}}; + + Struct::ProtobufMessage vanilla{}; + proto_structs::StructToMessage(std::move(s), vanilla); + { + auto map = vanilla.int_int(); + ASSERT_EQ(map.size(), 2u); + ASSERT_EQ(map.at(1), 2); + ASSERT_EQ(map.at(3), 4); + } + { + auto map = vanilla.int_string(); + ASSERT_EQ(map.size(), 2u); + ASSERT_EQ(map.at(1), "2"); + ASSERT_EQ(map.at(3), "4"); + } + { + auto map = vanilla.string_int(); + ASSERT_EQ(map.size(), 2u); + ASSERT_EQ(map.at("1"), 2); + ASSERT_EQ(map.at("3"), 4); + } + { + auto map = vanilla.string_string(); + ASSERT_EQ(map.size(), 2u); + ASSERT_EQ(map.at("1"), "2"); + ASSERT_EQ(map.at("3"), "4"); + } +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/not_recommended_field_names/basic_test.cpp b/libraries/proto-structs/codegen-tests/src/not_recommended_field_names/basic_test.cpp new file mode 100644 index 000000000000..8e2ec3445249 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/not_recommended_field_names/basic_test.cpp @@ -0,0 +1,51 @@ +#include + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +TEST(NotRecommendedFieldNameBasic, Basic) { + using Struct = not_recommended_field_names::structs::Basic; + Struct s{}; + + s.protected_ = 0; + s.final_ = 1; + s.void__ = 2; + s.float_ = 3; + s.typeid_ = 4; + s.new_ = 5; + s.double_ = 6; + + Struct::ProtobufMessage vanilla{}; + proto_structs::StructToMessage(std::move(s), vanilla); + + ASSERT_EQ(vanilla.protected_(), 0); + ASSERT_EQ(vanilla.final(), 1); + ASSERT_EQ(vanilla.void__(), 2); + ASSERT_EQ(vanilla.float_(), 3); + ASSERT_EQ(vanilla.typeid_(), 4); + ASSERT_EQ(vanilla.new_(), 5); + ASSERT_EQ(vanilla.double_(), 6); + // Not generated by a plagin. + ASSERT_EQ(vanilla.minor(), 0); + ASSERT_EQ(vanilla.major(), 0); +} + +TEST(NotRecommendedFieldNameBasic, Upper) { + using Struct = not_recommended_field_names::structs::Upper; + Struct s{}; + s.XYandexUid = 1; + s.Float = 2; + s.New = 3; + + Struct::ProtobufMessage vanilla{}; + proto_structs::StructToMessage(std::move(s), vanilla); + ASSERT_EQ(vanilla.xyandexuid(), 1); + ASSERT_EQ(vanilla.float_(), 2); + ASSERT_EQ(vanilla.new_(), 3); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/oneof/basic_test.cpp b/libraries/proto-structs/codegen-tests/src/oneof/basic_test.cpp new file mode 100644 index 000000000000..fbc81ff8bca2 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/oneof/basic_test.cpp @@ -0,0 +1,88 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +TEST(OneofBasic, LowercaseEmpty) { + oneof::structs::Parent message; + static_assert(std::is_same_v); + + EXPECT_FALSE(message.lowercase.has_integer()); + EXPECT_THROW([[maybe_unused]] const auto& not_found = message.lowercase.integer(), proto_structs::OneofAccessError); +} + +TEST(OneofBasic, LowercaseFundamentalTypes) { + oneof::structs::Parent message; + + message.lowercase.set_integer(10); + EXPECT_TRUE(message.lowercase.has_integer()); + EXPECT_EQ(message.lowercase.integer(), 10); + EXPECT_FALSE(message.lowercase.has_string()); + EXPECT_THROW([[maybe_unused]] const auto& not_found1 = message.lowercase.string(), proto_structs::OneofAccessError); + + message.lowercase.set_string("text"); + EXPECT_TRUE(message.lowercase.has_string()); + EXPECT_EQ(message.lowercase.string(), "text"); + EXPECT_FALSE(message.lowercase.has_integer()); + EXPECT_THROW( + [[maybe_unused]] const auto& not_found2 = message.lowercase.integer(), proto_structs::OneofAccessError + ); +} + +TEST(OneofBasic, LowercaseMessage) { + oneof::structs::Parent message; + + message.lowercase.set_message1(oneof::structs::Message1{.field = "text"}); + EXPECT_TRUE(message.lowercase.has_message1()); + EXPECT_EQ(message.lowercase.message1().field, "text"); + + message.lowercase.set_message2({.field = "text"}); + EXPECT_TRUE(message.lowercase.has_message2()); + EXPECT_EQ(message.lowercase.message2().field, "text"); +} + +TEST(OneofBasic, LowercaseEnum) { + oneof::structs::Parent message; + + message.lowercase.set_enum1(oneof::structs::Enum1::kFoo); + EXPECT_TRUE(message.lowercase.has_enum1()); + EXPECT_EQ(message.lowercase.enum1(), oneof::structs::Enum1::kFoo); + + message.lowercase.set_enum2(oneof::structs::Parent::Enum2::kFoo); + EXPECT_TRUE(message.lowercase.has_enum2()); + EXPECT_EQ(message.lowercase.enum2(), oneof::structs::Parent::Enum2::kFoo); +} + +TEST(OneofBasic, Uppercase) { + oneof::structs::Parent message; + static_assert(std::is_same_v); + + message.Uppercase.set_foo("text"); + EXPECT_TRUE(message.Uppercase.has_foo()); + EXPECT_EQ(message.Uppercase.foo(), "text"); + + message.Uppercase.set_bar(10); + EXPECT_TRUE(message.Uppercase.has_bar()); + EXPECT_EQ(message.Uppercase.bar(), 10); +} + +TEST(OneofBasic, SingleFieldOneof) { + oneof::structs::Parent message; + static_assert(std::is_same_v); + + EXPECT_FALSE(message.single_field_oneof.has_single()); + + message.single_field_oneof.set_single("text"); + EXPECT_TRUE(message.single_field_oneof.has_single()); + EXPECT_EQ(message.single_field_oneof.single(), "text"); +} + +TEST(OneofBasic, SyntheticOneofIsIgnored) { + [[maybe_unused]] oneof::structs::Parent message; + static_assert(std::is_same_v>); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/oneof/custom_oneof_type_name_test.cpp b/libraries/proto-structs/codegen-tests/src/oneof/custom_oneof_type_name_test.cpp new file mode 100644 index 000000000000..384372ab7810 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/oneof/custom_oneof_type_name_test.cpp @@ -0,0 +1,30 @@ +#include + +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +TEST(CustomOneofTypeName, Conflict1) { + using Scope = oneof::structs::NameConflict1; + AssertFieldCount(); + // `FooIdCustom` type name was set by `option (userver.structs.oneof).generated_type_name`. + // Without the option, it would be called `FooId`, and there would be a naming conflict. + AssertFieldType(); + static_assert(std::same_as, Scope::FooId>); +} + +TEST(CustomOneofTypeName, Conflict2) { + using Scope = oneof::structs::NameConflict2; + AssertFieldCount(); + // `TFooIdCustom` type name was set by `option (userver.structs.oneof).generated_type_name`. + // Without the option, it would be called `TFooId`, and there would be a naming conflict. + AssertFieldType(); + static_assert(std::same_as, Scope::TFooId>); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/oneof/proto2_test.cpp b/libraries/proto-structs/codegen-tests/src/oneof/proto2_test.cpp new file mode 100644 index 000000000000..a0f184540ebf --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/oneof/proto2_test.cpp @@ -0,0 +1,116 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +TEST(OneofProto2, OneofEmpty) { + oneof::structs::Proto2 message; + static_assert(std::is_same_v); + + EXPECT_FALSE(message.oneof.has_integer()); + EXPECT_THROW([[maybe_unused]] const auto& not_found = message.oneof.integer(), proto_structs::OneofAccessError); +} + +TEST(OneofProto2, OneofFundamentalTypes) { + oneof::structs::Proto2 message; + + message.oneof.set_integer(42); + EXPECT_TRUE(message.oneof.has_integer()); + EXPECT_EQ(message.oneof.integer(), 42); + EXPECT_FALSE(message.oneof.has_string()); + EXPECT_THROW([[maybe_unused]] const auto& not_found1 = message.oneof.string(), proto_structs::OneofAccessError); + + message.oneof.set_string("proto2_text"); + EXPECT_TRUE(message.oneof.has_string()); + EXPECT_EQ(message.oneof.string(), "proto2_text"); + EXPECT_FALSE(message.oneof.has_integer()); + EXPECT_THROW([[maybe_unused]] const auto& not_found2 = message.oneof.integer(), proto_structs::OneofAccessError); +} + +TEST(OneofProto2, OneofMessage) { + oneof::structs::Proto2 message; + + message.oneof.set_message({.field = "message_text"}); + EXPECT_TRUE(message.oneof.has_message()); + EXPECT_EQ(message.oneof.message().field, "message_text"); +} + +TEST(OneofProto2, OneofEnum) { + oneof::structs::Proto2 message; + + message.oneof.set_enum_(oneof::structs::Proto2::Enum::kFoo); + EXPECT_TRUE(message.oneof.has_enum_()); + EXPECT_EQ(message.oneof.enum_(), oneof::structs::Proto2::Enum::kFoo); +} + +TEST(OneofProto2, OneofGroup) { + oneof::structs::Proto2 message; + + oneof::structs::Proto2::Group group; + group.x = 100; + group.y = "group_text"; + message.oneof.set_group(group); + + EXPECT_TRUE(message.oneof.has_group()); + EXPECT_EQ(message.oneof.group().x, 100); + EXPECT_EQ(message.oneof.group().y, "group_text"); +} + +TEST(OneofProto2, GroupOneof) { + oneof::structs::Proto2 message; + + oneof::structs::Proto2::Group group; + group.x = 100; + group.group_oneof.set_z({.field = "nested_message"}); + message.oneof.set_group(group); + + EXPECT_TRUE(message.oneof.has_group()); + EXPECT_TRUE(message.oneof.group().group_oneof.has_z()); + EXPECT_EQ(message.oneof.group().group_oneof.z().field, "nested_message"); + + // Change the oneof inside the group + message.oneof.mutable_group().group_oneof.set_w(oneof::structs::Proto2::Enum::kFoo); + EXPECT_TRUE(message.oneof.mutable_group().group_oneof.has_w()); + EXPECT_EQ(message.oneof.mutable_group().group_oneof.w(), oneof::structs::Proto2::Enum::kFoo); + EXPECT_FALSE(message.oneof.mutable_group().group_oneof.has_z()); +} + +TEST(OneofProto2, NestedTypesOutsideGroup) { + oneof::structs::Proto2 message; + + // Using MessageInGroup outside the group context + message.message_from_group.foo = "foo_value"; + EXPECT_EQ(message.message_from_group.foo, "foo_value"); + + // Using EnumInGroup outside the group context + message.enum_from_group.push_back(oneof::structs::Proto2::Group::EnumInGroup::kFoo); + EXPECT_EQ(message.enum_from_group.size(), std::size_t{1}); + EXPECT_EQ(message.enum_from_group[0], oneof::structs::Proto2::Group::EnumInGroup::kFoo); +} + +TEST(OneofProto2, SingleFieldOneof) { + oneof::structs::Proto2 message; + static_assert(std::is_same_v); + + EXPECT_FALSE(message.single_field_oneof.has_single()); + + message.single_field_oneof.set_single("single_field_text"); + EXPECT_TRUE(message.single_field_oneof.has_single()); + EXPECT_EQ(message.single_field_oneof.single(), "single_field_text"); +} + +TEST(OneofProto2, OptionalFieldBefore) { + oneof::structs::Proto2 message; + static_assert(std::is_same_v>); + + EXPECT_FALSE(message.field_before.has_value()); + + message.field_before = "field_before_value"; + EXPECT_TRUE(message.field_before.has_value()); + EXPECT_EQ(*message.field_before, "field_before_value"); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/simple/simple_test.cpp b/libraries/proto-structs/codegen-tests/src/simple/simple_test.cpp new file mode 100644 index 000000000000..c429d45533f4 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/simple/simple_test.cpp @@ -0,0 +1,117 @@ +#include + +#include + +#include + +#include +#include + +namespace ss = simple::structs; + +USERVER_NAMESPACE_BEGIN + +TEST(SingleFile, SimpleStruct) { + static_assert(std::is_aggregate_v); + [[maybe_unused]] ss::SimpleStruct message; + message.some_integer = 5; + message.some_text = std::optional("foo"); + message.is_condition = true; + message.some_bytes = {"foo", "bar"}; + message.something.set_bar("bar_val"); + message.inner_enum = ss::SimpleStruct::InnerEnum2::kFooVal; + message.nested.swag = "foo"; + message.optional_nested = std::optional{{.swag = "foo"}}; + + ss::SimpleStruct::ProtobufMessage vanilla; + + ::proto_structs::StructToMessage(std::move(message), vanilla); + + EXPECT_EQ(vanilla.some_integer(), 5); + EXPECT_EQ(vanilla.some_text(), "foo"); + EXPECT_EQ(vanilla.is_condition(), true); + EXPECT_EQ(vanilla.some_bytes().Get(0), "foo"); + EXPECT_EQ(vanilla.some_bytes().Get(1), "bar"); + EXPECT_EQ(vanilla.bar(), "bar_val"); + EXPECT_EQ(vanilla.inner_enum(), ss::SimpleStruct::ProtobufMessage::FOO_VAL); + EXPECT_EQ(vanilla.nested().swag(), "foo"); + EXPECT_EQ(vanilla.optional_nested().swag(), "foo"); + + ss::SimpleStruct to; + ::proto_structs::MessageToStruct(vanilla, to); + + ASSERT_EQ(to.some_integer, 5); + ASSERT_EQ(to.some_text, std::optional("foo")); + ASSERT_TRUE(to.is_condition); + std::vector exp = {"foo", "bar"}; + ASSERT_EQ(to.some_bytes, exp); + ASSERT_EQ(to.something.bar(), "bar_val"); + ASSERT_THROW([[maybe_unused]] auto foo = to.something.foo(), proto_structs::OneofAccessError); +} + +TEST(SingleFile, NestedStruct) { + static_assert(std::is_aggregate_v); + [[maybe_unused]] ss::SimpleStruct::NestedStruct nested; + nested.swag = "foo"; + + static_assert(std::is_aggregate_v); + [[maybe_unused]] ss::SimpleStruct::NestedStruct::NestedStruct2 nested2; + nested2.swag2 = "bar"; +} + +TEST(SingleFile, InnerEnum1) { + static_assert(std::is_enum_v); + [[maybe_unused]] const auto inner_enum1 = ss::SimpleStruct::NestedStruct::NestedStruct2::InnerEnum1::kBarVal; +} + +TEST(SingleFile, InnerEnum2) { + static_assert(std::is_enum_v); + [[maybe_unused]] const auto inner_enum2 = ss::SimpleStruct::InnerEnum2::kFooVal; +} + +TEST(SingleFile, SecondStruct) { + static_assert(std::is_aggregate_v); + [[maybe_unused]] ss::SecondStruct message; +} + +TEST(SingleFile, GlobalEnum) { + static_assert(std::is_enum_v); + [[maybe_unused]] ss::GlobalEnum message{}; +} + +TEST(Oneof, Empty) { + const ss::SimpleStruct::Something none; + EXPECT_FALSE(none); + EXPECT_FALSE(none.has_foo()); + EXPECT_FALSE(none.has_bar()); + EXPECT_THROW([[maybe_unused]] const auto& not_found1 = none.foo(), proto_structs::OneofAccessError); + EXPECT_THROW([[maybe_unused]] const auto& not_found2 = none.bar(), proto_structs::OneofAccessError); +} + +TEST(Oneof, MakeFoo) { + ss::SimpleStruct::Something foo; + foo.set_foo(42); + EXPECT_TRUE(foo); + EXPECT_TRUE(foo.has_foo()); + EXPECT_NO_THROW(EXPECT_EQ(foo.foo(), 42)); + EXPECT_FALSE(foo.has_bar()); + EXPECT_THROW([[maybe_unused]] const auto& not_found = foo.bar(), proto_structs::OneofAccessError); +} + +TEST(Oneof, MakeBar) { + ss::SimpleStruct::Something bar; + bar.set_bar("bar"); + EXPECT_TRUE(bar); + EXPECT_FALSE(bar.has_foo()); + EXPECT_THROW([[maybe_unused]] const auto& not_found = bar.foo(), proto_structs::OneofAccessError); + EXPECT_TRUE(bar.has_bar()); + EXPECT_NO_THROW(EXPECT_EQ(bar.bar(), "bar")); +} + +TEST(Oneof, OneofInStruct) { + [[maybe_unused]] ss::SimpleStruct message; + message.something.set_bar("bar"); + EXPECT_EQ(message.something.bar(), "bar"); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/codegen-tests/src/test_utils/type_assertions.hpp b/libraries/proto-structs/codegen-tests/src/test_utils/type_assertions.hpp new file mode 100644 index 000000000000..8b3e85c680f9 --- /dev/null +++ b/libraries/proto-structs/codegen-tests/src/test_utils/type_assertions.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace impl { + +template +constexpr std::size_t GetFieldCount() { + return boost::pfr::tuple_size::value; +} + +} // namespace impl + +template +constexpr void AssertFieldType() { + static_assert(std::is_same_v, "Must be equal"); +} + +template +constexpr void AssertFieldCount() { + static_assert(impl::GetFieldCount() == Count); +} + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/include/userver/proto-structs/any.hpp b/libraries/proto-structs/include/userver/proto-structs/any.hpp new file mode 100644 index 000000000000..a26ad4a3ac8d --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/any.hpp @@ -0,0 +1,124 @@ +#pragma once + +/// @file userver/proto-structs/any.hpp +/// @brief Class to access `google.protobuf.Any` stored message as a struct + +#include +#include + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace proto_structs { + +/// @brief Wrapper for `google.protobuf.Any` which provides interface to access stored message as compatible struct +class Any final { +public: + using ProtobufMessage = ::google::protobuf::Any; + + /// @brief Creates empty `Any`. + Any() noexcept = default; + + /// @brief Creates wrapper initializing its underlying storage with @a proto_any + Any(google::protobuf::Any proto_any) : storage_(std::move(proto_any)) {} + + /// @brief Creates `Any` holding @a obj + /// @tparam TStruct proto struct type + /// @throws WriteError if conversion of @a obj to its compatible message has failed + /// @throws AnyPackError if compatible protobuf message can not be packed to `google.protobuf.Any` + template + requires(!std::is_same_v, ::google::protobuf::Any>) && + (!std::is_same_v, Any>) && traits::ProtoStruct> + Any(TStruct&& obj) { + Pack(std::forward(obj)); + } + + /// @brief Packs @a obj in `Any` + /// @tparam TStruct proto struct type + /// @throws WriteError if conversion of @a obj to its compatible message has failed + /// @throws AnyPackError if compatible protobuf message can not be packed to `google.protobuf.Any` + template + requires(!std::is_same_v, ::google::protobuf::Any>) && + (!std::is_same_v, Any>) && traits::ProtoStruct> + Any& operator=(TStruct&& obj) { + Pack(std::forward(obj)); + return *this; + } + + /// @brief Returns `true` if `Any` contains `TStruct` + /// @tparam TStruct proto struct type + template + bool Is() const noexcept { + using Message = traits::CompatibleMessageType; + return storage_.Is(); + } + + /// @brief Returns `true` if underlying `google.protobuf.Any` contains `TMessage` + /// @tparam TMessage protobuf message type + template + bool Is() const noexcept { + return storage_.Is>(); + } + + /// @brief Unpacks `Any` to `TStruct` struct + /// @tparam TStruct proto struct type + /// @throws AnyUnpackError if underlying `google.protobuf.Any` does not contain message compatible to `TStruct` + /// @throws ReadError if conversion of unpacked protobuf message to proto struct has failed + template + TStruct Unpack() { + using Message = traits::CompatibleMessageType; + return MessageToStruct(Unpack()); + } + + /// @brief Unpacks underlying `google.protobuf.Any` to `TMessage` message + /// @tparam TMessage protobuf message type + /// @throws AnyUnpackError if underlying `google.protobuf.Any` does not contain `TMessage` type message + template + TMessage Unpack() { + TMessage msg; + + if (!storage_.UnpackTo(&msg)) { + throw AnyUnpackError(TMessage::descriptor()->full_name()); + } + + return msg; + } + + /// @brief Packs @a obj to `Any` + /// @tparam TStruct proto struct type + /// @throws WriteError if conversion of @a obj to its compatible message has failed + /// @throws AnyPackError if packing of compatible protobuf message to `google.protobuf.Any` has failed + template + requires traits::ProtoStruct> + void Pack(TStruct&& obj) { + using Message = traits::CompatibleMessageType>; + Pack(StructToMessage(std::forward(obj))); + } + + /// @brief Packs @a message to underlying `google.protobuf.Any` + /// @tparam TMessage protobuf message type + /// @throws AnyPackError if packing of protobuf message to `google.protobuf.Any` has failed + template + void Pack(const TMessage& message) { + if (!storage_.PackFrom(message)) { + throw AnyUnpackError(TMessage::descriptor()->full_name()); + } + } + + /// @brief Returns underlying `google.protobuf.Any` + const ::google::protobuf::Any& GetProtobufAny() const& noexcept { return storage_; } + + /// @brief Returns underlying `google.protobuf.Any` + ::google::protobuf::Any&& GetProtobufAny() && noexcept { return std::move(storage_); } + +private: + ::google::protobuf::Any storage_; +}; + +} // namespace proto_structs + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/include/userver/proto-structs/convert.hpp b/libraries/proto-structs/include/userver/proto-structs/convert.hpp new file mode 100644 index 000000000000..711f69c73a05 --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/convert.hpp @@ -0,0 +1,98 @@ +#pragma once + +/// @file userver/proto-structs/convert.hpp +/// @brief Functions for protobuf message to/from struct conversion +/// +/// Conversion functions rely on user-defined functions with the following signatures: +/// 1. `StructType ReadProtoStruct(proto_structs::io::ReadContext&, +/// proto_structs::io::To, +/// const MessageType&)` +/// +/// 2. `void WriteProtoStruct(proto_structs::io::WriteContext&, +/// const StructType&, +/// MessageType&)` +/// +/// This functions should be foundable by ADL lookup (just place them in the namespace where `StructType` is defined). +/// +/// It is recommended to define `WriteProtoStruct` function as template with a `StructType` parameter and use universal +/// reference `StructType&&` as a second parameter to enable perfect forwarding for conversions. + +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace proto_structs { + +/// @brief Converts protobuf message @a msg to proto struct @a obj +/// @tparam TMessage protobuf message type +/// @tparam TStruct proto struct type +/// @throws ReadError if conversion has failed +/// @warning If function throws an exception, @a obj is left in a valid but unspecified state +template +void MessageToStruct(const TMessage& msg, TStruct& obj) { + io::ReadContext ctx(*msg.GetDescriptor()); + + try { + obj = ReadProtoStruct(ctx, io::To>{}, msg); + } catch (const ReadError&) { + // simply re-throw proper errors + throw; + } catch (const std::exception& e) { + // some user-defined `ReadProtoStruct` threw an exception instead of adding an error to context, + // adding it manually an re-throwing as a proper exception type + ctx.AddError(e.what()); + } +} + +/// @brief Converts protobuf message @a msg to specified proto struct type +/// @tparam TStruct proto struct type +/// @tparam TMessage protobuf message type +/// @throws ReadError if conversion has failed +template +[[nodiscard]] TStruct MessageToStruct(const TMessage& msg) { + TStruct obj; + MessageToStruct(msg, obj); + return obj; +} + +/// @brief Converts proto struct @a obj to protobuf message @a msg +/// @tparam TStruct proto struct type +/// @tparam TMessage protobuf message type +/// @throws WriteError if conversion has failed +/// @warning If function throws an exception, @a msg is left in a valid but unspecified state +template +requires traits::ProtoStruct> +void StructToMessage(TStruct&& obj, TMessage& msg) { + io::WriteContext ctx(*msg.GetDescriptor()); + + try { + WriteProtoStruct(ctx, std::forward(obj), msg); + } catch (const WriteError&) { + // simply re-throw proper errors + throw; + } catch (const std::exception& e) { + // some user-defined `WriteProtoStruct` threw an exception instead of adding an error to context, + // adding it manually an re-throwing as a proper exception type + ctx.AddError(e.what()); + } +} + +/// @brief Converts proto struct @a obj to it's compatible protobuf message type +/// @tparam TStruct proto struct type +/// @throws WriteError if conversion has failed +template +requires traits::ProtoStruct> +[[nodiscard]] traits::CompatibleMessageType> StructToMessage(TStruct&& obj) { + using Message = traits::CompatibleMessageType>; + Message msg; + StructToMessage(std::forward(obj), msg); + return msg; +} + +} // namespace proto_structs + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/include/userver/proto-structs/exceptions.hpp b/libraries/proto-structs/include/userver/proto-structs/exceptions.hpp new file mode 100644 index 000000000000..8a18c373a1f3 --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/exceptions.hpp @@ -0,0 +1,70 @@ +#pragma once + +/// @file userver/proto-structs/exceptions.hpp +/// @brief Exceptions thrown by the library + +#include +#include + +USERVER_NAMESPACE_BEGIN + +/// @brief Top namespace for the proto-structs library +namespace proto_structs { + +/// @brief Library basic exception type +/// All other proto-structs exceptions are derived from this type. +class Error : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +/// @brief Conversion error base class +class ConversionError : public Error { +protected: + using Error::Error; +}; + +/// @brief Error reading proto struct from protobuf message +class ReadError final : public ConversionError { +public: + /// @brief Creates error with information what protobuf message field was considered invalid + /// Parameter @a path contains dot-separated field names from the top-level message up to erroneous field, for + /// example, "msg.field.nested_field". + ReadError(std::string_view path, std::string_view reason); +}; + +/// @brief Error writing proto struct to protobuf message +class WriteError final : public ConversionError { +public: + /// @brief Creates error with information what protobuf message field was not correctly initialized + /// Parameter @a path contains dot-separated field names from the top-level message up to field that was not + /// initialized, for example, "msg.field.nested_field". + WriteError(std::string_view path, std::string_view reason); +}; + +/// @brief Invalid attempt to access unset @ref proto_structs::Oneof field +class OneofAccessError : public Error { +public: + /// @brief Creates error on invalid attempt to access @a field_idx field of `oneof` + explicit OneofAccessError(std::size_t field_idx); +}; + +/// @brief Error packing protobuf message to @ref proto_structs::Any underlying storage. +class AnyPackError : public Error { +public: + /// @brief Creates error with information what protobuf message was not packed + explicit AnyPackError(std::string_view message_name); +}; + +/// @brief Error unpacking protobuf message from @ref proto_structs::Any underlying storage. +/// The main reason of this exception is the attempt to unpack message type different from the one stored in the +/// @ref proto_structs::Any underlying storage +class AnyUnpackError : public Error { +public: + /// @brief Creates error with information what protobuf message was not unpacked + explicit AnyUnpackError(std::string_view message_name); +}; + +} // namespace proto_structs + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/include/userver/proto-structs/hash_map.hpp b/libraries/proto-structs/include/userver/proto-structs/hash_map.hpp new file mode 100644 index 000000000000..c5872d8cf08c --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/hash_map.hpp @@ -0,0 +1,30 @@ +#pragma once + +/// @file userver/proto-structs/hash_map.hpp +/// @brief @copybrief proto_structs::HashMap + +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace proto_structs { + +/// @brief The hash map container used in userver proto structs by default. +/// +/// Currently implemented as just `std::unordered_map`. Please don't assume it! For example: +/// +/// * Don't pass the field to functions as `std::unordered_map`, use `proto_structs::HashMap` instead; +/// * Don't use node and bucket APIs of `std::unordered_map` with these fields. +template +using HashMap = std::unordered_map< + Key, + Value, + std::conditional_t, utils::StrCaseHash, std::hash>>; + +} // namespace proto_structs + +USERVER_NAMESPACE_END diff --git a/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_cpp.hpp b/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_cpp.hpp new file mode 100644 index 000000000000..a7c0c25be62a --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_cpp.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +// For keyword types. +#include diff --git a/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_hpp.hpp b/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_hpp.hpp new file mode 100644 index 000000000000..3d75041f0446 --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/impl/bundles/structs_hpp.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include diff --git a/libraries/proto-structs/include/userver/proto-structs/impl/oneof_codegen.hpp b/libraries/proto-structs/include/userver/proto-structs/impl/oneof_codegen.hpp new file mode 100644 index 000000000000..47c6f856ff52 --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/impl/oneof_codegen.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include +#include + +#include + +#define UPROTO_ONEOF_HEADER(oneof_type) \ +private: \ + enum { kCounterStart = __COUNTER__ + 1 }; /* An inline constant would violate odr. */ \ +public: \ + using Base::Base; + +#define UPROTO_ONEOF_FIELD(oneof_type, field_type, field_name) \ +private: \ + static constexpr std::size_t field_name##_index = __COUNTER__ - kCounterStart; \ + \ +public: \ + [[nodiscard]] constexpr bool has_##field_name() const noexcept; \ + [[nodiscard]] constexpr const field_type& field_name() const&; \ + [[nodiscard]] constexpr field_type&& field_name()&&; \ + constexpr void set_##field_name(const field_type& value); \ + constexpr void set_##field_name(field_type&& value); \ + constexpr field_type& mutable_##field_name(); \ + [[nodiscard]] static oneof_type make_##field_name(const field_type& value); \ + [[nodiscard]] static oneof_type make_##field_name(field_type&& value); + +#define UPROTO_ONEOF_DEFINE_FIELD_FUNCTIONS(oneof_type, field_type, field_name) \ + [[nodiscard]] constexpr bool oneof_type::has_##field_name() const noexcept { \ + return Base::Contains(field_name##_index); \ + } \ + [[nodiscard]] constexpr const field_type& oneof_type::field_name() const& { \ + return Base::Get(); \ + } \ + [[nodiscard]] constexpr field_type&& oneof_type::field_name()&& { \ + return std::move(*this).Base::Get(); \ + } \ + constexpr void oneof_type::set_##field_name(const field_type& value) { Base::Set(value); } \ + constexpr void oneof_type::set_##field_name(field_type&& value) { \ + Base::Set(std::move(value)); \ + } \ + constexpr field_type& oneof_type::mutable_##field_name() { return Base::GetMutable(); } \ + [[nodiscard]] inline oneof_type oneof_type::make_##field_name(const field_type& value) { \ + return oneof_type(std::in_place_index, value); \ + } \ + [[nodiscard]] inline oneof_type oneof_type::make_##field_name(field_type&& value) { \ + return oneof_type(std::in_place_index, std::move(value)); \ + } diff --git a/libraries/proto-structs/include/userver/proto-structs/impl/traits_light.hpp b/libraries/proto-structs/include/userver/proto-structs/impl/traits_light.hpp new file mode 100644 index 000000000000..958177d4f6e8 --- /dev/null +++ b/libraries/proto-structs/include/userver/proto-structs/impl/traits_light.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +USERVER_NAMESPACE_BEGIN + +namespace proto_structs::impl::traits { + +template