feat kafka: move schemas to separate files and embed them

Tests: протестировано локально и в CI
commit_hash:575e46371b331535dfb93baa955633f14ce9d3f7

Author: apolukhin

.mapping.json

@@ -2604,6 +2604,7 @@
   "kafka/include/userver/kafka/rebalance_types.hpp":"taxi/uservices/userver/kafka/include/userver/kafka/rebalance_types.hpp",
   "kafka/library.yaml":"taxi/uservices/userver/kafka/library.yaml",
   "kafka/src/kafka/consumer_component.cpp":"taxi/uservices/userver/kafka/src/kafka/consumer_component.cpp",
+  "kafka/src/kafka/consumer_component.yaml":"taxi/uservices/userver/kafka/src/kafka/consumer_component.yaml",
   "kafka/src/kafka/consumer_scope.cpp":"taxi/uservices/userver/kafka/src/kafka/consumer_scope.cpp",
   "kafka/src/kafka/exceptions.cpp":"taxi/uservices/userver/kafka/src/kafka/exceptions.cpp",
   "kafka/src/kafka/headers.cpp":"taxi/uservices/userver/kafka/src/kafka/headers.cpp",
@@ -2628,6 +2629,7 @@
   "kafka/src/kafka/impl/stats.cpp":"taxi/uservices/userver/kafka/src/kafka/impl/stats.cpp",
   "kafka/src/kafka/producer.cpp":"taxi/uservices/userver/kafka/src/kafka/producer.cpp",
   "kafka/src/kafka/producer_component.cpp":"taxi/uservices/userver/kafka/src/kafka/producer_component.cpp",
+  "kafka/src/kafka/producer_component.yaml":"taxi/uservices/userver/kafka/src/kafka/producer_component.yaml",
   "kafka/tests/configuration_test.cpp":"taxi/uservices/userver/kafka/tests/configuration_test.cpp",
   "kafka/tests/consumer_kafkatest.cpp":"taxi/uservices/userver/kafka/tests/consumer_kafkatest.cpp",
   "kafka/tests/headers_test.cpp":"taxi/uservices/userver/kafka/tests/headers_test.cpp",

cmake/UserverEmbedFile.cmake

@@ -3,8 +3,10 @@ include_guard(GLOBAL)
 function(userver_embed_file TARGET)
     set(OPTIONS)
     set(ONE_VALUE_ARGS NAME FILEPATH HPP_FILENAME)
-    set(MULTI_VALUE_ARGS SQL_FILES)
     cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN})
+    if(NOT ARG_NAME)
+        set(ARG_NAME "${ARG_FILEPATH}")
+    endif()
     if(NOT ARG_HPP_FILENAME)
         get_filename_component(ARG_HPP_FILENAME "${ARG_FILEPATH}" NAME)
     endif()
@@ -21,11 +23,12 @@ function(userver_embed_file TARGET)
     _userver_codegen_register_files("${CMAKE_CURRENT_BINARY_DIR}/embedded/embedded.cpp")
 
     get_property(USERVER_CMAKE_DIR GLOBAL PROPERTY userver_cmake_dir)
+    string(MAKE_C_IDENTIFIER "${ARG_NAME}" NAME_C_ESCAPED)
     add_custom_command(
         OUTPUT ${CONFIG_HPP}
         DEPENDS ${USERVER_CMAKE_DIR}/embedded_config.cmake ${ARG_FILEPATH}
         COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DFILEPATH=${ARG_FILEPATH}
-                -DOUTPUT=${CONFIG_HPP} -DNAME=${ARG_NAME} -P ${USERVER_CMAKE_DIR}/embedded_config.cmake ${CODEGEN}
+                -DOUTPUT=${CONFIG_HPP} -DNAME_C_ESCAPED=${NAME_C_ESCAPED} -DNAME=${ARG_NAME} -P ${USERVER_CMAKE_DIR}/embedded_config.cmake ${CODEGEN}
     )
     _userver_codegen_register_files("${CONFIG_HPP}")
     add_library(${TARGET} STATIC ${CONFIG_HPP} ${CMAKE_CURRENT_BINARY_DIR}/embedded/embedded.cpp)

cmake/UserverModule.cmake

@@ -24,6 +24,7 @@ function(userver_module MODULE)
         UBENCH_DATABASES
         UBENCH_ENV
         DEPENDS
+        EMBED_FILES
     )
     cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN})
     if(ARG_UNPARSED_ARGUMENTS)
@@ -149,4 +150,10 @@ function(userver_module MODULE)
             DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/.."
         )
     endif()
+
+    foreach(FILE ${ARG_EMBED_FILES})
+        string(MAKE_C_IDENTIFIER "userver-${MODULE}-embed_${FILE}" EMBED_TARGET_NAME)
+        userver_embed_file(${EMBED_TARGET_NAME} FILEPATH "${FILE}" HPP_FILENAME "${FILE}")
+        target_link_libraries(userver-${MODULE} PRIVATE "$<BUILD_INTERFACE:${EMBED_TARGET_NAME}>")
+    endforeach()
 endfunction()

cmake/embedded_config.cmake

@@ -19,32 +19,32 @@ set(TEMPLATE
 __asm__(
 #if defined(__APPLE__)
 \".const_data\\n\"
-\".global _@NAME@_start\\n\"
-\".global _@NAME@_end\\n\"
-\".global _@NAME@_size\\n\"
+\".global _@NAME_C_ESCAPED@_start\\n\"
+\".global _@NAME_C_ESCAPED@_end\\n\"
+\".global _@NAME_C_ESCAPED@_size\\n\"
 #else
 \".section .rodata\\n\"
 #endif
 \".balign 16\\n\"
-APPLE_PREFIX \"@NAME@_begin:\\n\"
+APPLE_PREFIX \"@NAME_C_ESCAPED@_begin:\\n\"
 R\"(
 .incbin \"${FILEPATH}\"
 )\"
 \".balign 1\\n\"
-APPLE_PREFIX \"@NAME@_end:\\n\"
+APPLE_PREFIX \"@NAME_C_ESCAPED@_end:\\n\"
 \".byte 0\\n\"
-APPLE_PREFIX \"@NAME@_size:\\n\"
-\".int \" APPLE_PREFIX \"@NAME@_end - \" APPLE_PREFIX \"@NAME@_begin\\n\"
+APPLE_PREFIX \"@NAME_C_ESCAPED@_size:\\n\"
+\".int \" APPLE_PREFIX \"@NAME_C_ESCAPED@_end - \" APPLE_PREFIX \"@NAME_C_ESCAPED@_begin\\n\"
 \".previous\\n\"
 );
 
-extern \"C\" const char @NAME@_begin[];
-extern \"C\" const char @NAME@_end;
-extern \"C\" const int @NAME@_size;
+extern \"C\" const char @NAME_C_ESCAPED@_begin[];
+extern \"C\" const char @NAME_C_ESCAPED@_end;
+extern \"C\" const int @NAME_C_ESCAPED@_size;
 
 
-__attribute__((constructor)) void @NAME@_call() {
-  ${NAMESPACE}::utils::RegisterResource(\"@NAME@\", std::string_view{@NAME@_begin, static_cast<size_t>(@NAME@_size)});
+inline __attribute__((constructor)) void @NAME_C_ESCAPED@_call() {
+  ${NAMESPACE}::utils::RegisterResource(\"@NAME@\", std::string_view{@NAME_C_ESCAPED@_begin, static_cast<size_t>(@NAME_C_ESCAPED@_size)});
 }
 "
 )

kafka/CMakeLists.txt

@@ -13,6 +13,9 @@ userver_module(
     DBTEST_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/tests"
     DBTEST_LINK_LIBRARIES userver::kafka-utest
     DEPENDS core
+    EMBED_FILES
+        src/kafka/consumer_component.yaml
+        src/kafka/producer_component.yaml
 )
 
 target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-ignored-qualifiers")

kafka/src/kafka/consumer_component.cpp

@@ -10,6 +10,10 @@
 #include <userver/storages/secdist/component.hpp>
 #include <userver/yaml_config/merge_schemas.hpp>
 
+#ifndef ARCADIA_ROOT
+#include "generated/src/kafka/consumer_component.yaml.hpp"  // Y_IGNORE
+#endif
+
 USERVER_NAMESPACE_BEGIN
 
 namespace kafka {
@@ -61,150 +65,7 @@ ConsumerComponent::~ConsumerComponent() { statistics_holder_.Unregister(); }
 ConsumerScope ConsumerComponent::GetConsumer() { return consumer_->MakeConsumerScope(); }
 
 yaml_config::Schema ConsumerComponent::GetStaticConfigSchema() {
-    return yaml_config::MergeSchemas<components::ComponentBase>(R"(
-type: object
-description: Kafka consumer component
-additionalProperties: false
-properties:
-    group_id:
-        type: string
-        description: |
-            consumer group id.
-            Topic partition evenly distributed
-            between consumers with the same `group_id`
-    client_id:
-        type: string
-        description: |
-            Client identifier.
-            May be an arbitrary string.
-            Optional, but you should set this property on each instance
-            because it enables you to more easily correlate requests on the broker
-            with the client instance which made it,
-            which can be helpful in debugging and troubleshooting scenarios.
-        defaultDescription: userver
-    topics:
-        type: array
-        description: list of topics consumer subscribes
-        items:
-            type: string
-            description: topic name
-    max_batch_size:
-        type: integer
-        description: maximum number of messages consumer waits for new messages before calling a callback
-        defaultDescription: 1
-    poll_timeout:
-        type: string
-        description: maximum amount of time consumer waits for new messages before calling a callback
-        defaultDescription: 1s
-    message_key_log_format:
-        type: string
-        enum:
-          - plaintext
-          - hex
-        description: |
-            Specifies the logging format for the message key.
-            - 'plaintext': logs the message key as-is.
-            - 'hex': logs the message key in hexadecimal format.
-        defaultDescription: plaintext
-    max_callback_duration:
-        type: string
-        description: |
-            duration user callback must fit not to be kicked from the consumer group.
-            The duration must fit in [1ms; 86400000ms]
-        defaultDescription: 5m
-    restart_after_failure_delay:
-        type: string
-        description: backoff consumer waits until restart after user-callback exception.
-        defaultDescription: 10s
-    auto_offset_reset:
-        type: string
-        description: |
-            action to take when there is no
-            initial offset in offset store
-            or the desired offset is out of range:
-            `smallest`, `earliest`, `beginning` - automatically reset
-            the offset to the smallest offset
-            `largest`, `latest`, `end` - automatically reset
-            the offset to the largest offset,
-            `error` - trigger an error (ERR__AUTO_OFFSET_RESET).
-            Note: the policy applies iff there are not committed
-            offsets in topic
-        enum:
-          - smallest
-          - earliest
-          - beginning
-          - largest
-          - latest
-          - end
-          - error
-    env_pod_name:
-        type: string
-        description: |
-            if defined and `group_id` value contains
-            `{pod_name}` substring, the substring
-            is replaced with the value of the environment
-            variable `env_pod_name`
-        defaultDescription: none
-    security_protocol:
-        type: string
-        description: protocol used to communicate with brokers
-        enum:
-          - PLAINTEXT
-          - SASL_SSL
-          - SASL_PLAINTEXT
-          - SSL
-    sasl_mechanisms:
-        type: string
-        description: |
-            SASL mechanism to use for authentication.
-            Must be set if `security_protocol` equals `SASL_SSL`
-        defaultDescription: none
-        enum:
-          - PLAIN
-          - SCRAM-SHA-512
-    ssl_ca_location:
-        type: string
-        description: |
-            file or directory path to CA certificate(s) for verifying the broker's key.
-            Must be set if `security_protocol` equals `SASL_SSL` or `SSL`.
-            If set to `probe`, CA certificates are probed from the default certificates paths
-        defaultDescription: none
-    topic_metadata_refresh_interval:
-        type: string
-        description: |
-            period of time at which
-            topic and broker metadata is refreshed
-            in order to discover any new brokers,
-            topics, partitions or partition leader changes
-        defaultDescription: 5m
-    metadata_max_age:
-        type: string
-        description: |
-            metadata cache max age.
-            Recommended value is 3 times `topic_metadata_refresh_interval`
-        defaultDescription: 15m
-    rd_kafka_custom_options:
-        type: object
-        description: |
-            a map of arbitrary `librdkafka` library configuration options.
-            Full list of options is available by link: https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
-            Note: This options is not guaranteed to be supported in userver-kafka, use it at your own risk
-        properties: {}
-        additionalProperties:
-            type: string
-            description: librdkafka option value
-        defaultDescription: '{}'
-    debug_info_log_level:
-        type: string
-        description: |
-            log level for everything debug information
-        defaultDescription: debug
-    operation_log_level:
-        type: string
-        description: |
-            log level for infos about ordinary actions
-        defaultDescription: debug
-)");
+    return yaml_config::MergeSchemasFromResource<components::ComponentBase>("src/kafka/consumer_component.yaml");
 }
 
 }  // namespace kafka