diff --git a/.github/import_generation.txt b/.github/import_generation.txt index 7273c0fa8c5..6f4247a6255 100644 --- a/.github/import_generation.txt +++ b/.github/import_generation.txt @@ -1 +1 @@ -25 +26 diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 4a5d128d765..93f70a8963f 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -b39a359f04c57875ea162524646ae879f06c7a33 +234f6d55e1e71a242c2767a22b990d68ff7fce72 diff --git a/cmake/common.cmake b/cmake/common.cmake index 546ce4e81f0..bd017425190 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -199,7 +199,7 @@ function(_ydb_sdk_add_library Tgt) $ ) target_compile_definitions(${Tgt} ${includeMode} - YDB_SDK_USE_STD_STRING + YDB_SDK_OSS ) endfunction() diff --git a/examples/auth/ssa_delegation/main.cpp b/examples/auth/ssa_delegation/main.cpp index b3a678ee2b8..9b18d4f9034 100644 --- a/examples/auth/ssa_delegation/main.cpp +++ b/examples/auth/ssa_delegation/main.cpp @@ -34,9 +34,9 @@ int main(int argc, char** argv) { opts.AddLongOption("microservice-id", "Microservice id") .RequiredArgument("ID") - .DefaultValue("control-plane") + .DefaultValue("data-plane") .StoreResult(µserviceId); - + opts.AddLongOption("resource-id", "Resource id") .Required() .RequiredArgument("ID") diff --git a/examples/executor/main.cpp b/examples/executor/main.cpp new file mode 100644 index 00000000000..2d314403fe8 --- /dev/null +++ b/examples/executor/main.cpp @@ -0,0 +1,52 @@ +#include +#include +#include + +#include + +#include + +#include + + +void ExecutorExample(const std::string& endpoint, const std::string& database) { + auto driverConfig = NYdb::CreateFromEnvironment(endpoint + "/?database=" + database) + .SetExecutor(NYdb::CreateThreadPoolExecutorAdapter( + std::make_shared(TThreadPool::TParams() + .SetBlocking(true) + .SetCatching(false) + .SetForkAware(false)), + std::thread::hardware_concurrency()) + ); + + NYdb::TDriver driver(driverConfig); + NYdb::NQuery::TQueryClient client(driver); + + try { + auto result = client.ExecuteQuery("SELECT 1", NYdb::NQuery::TTxControl::NoTx()).GetValueSync(); + NYdb::NStatusHelpers::ThrowOnError(result); + auto parser = result.GetResultSetParser(0); + parser.TryNextRow(); + std::cout << "Result: " << parser.ColumnParser(0).GetInt32() << std::endl; + } catch (const std::exception& e) { + std::cerr << "Execution failed: " << e.what() << std::endl; + } + + driver.Stop(true); +} + +int main(int argc, char** argv) { + std::string endpoint; + std::string database; + + NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); + + opts.AddLongOption('e', "endpoint", "YDB endpoint").Required().RequiredArgument("HOST:PORT").StoreResult(&endpoint); + opts.AddLongOption('d', "database", "YDB database").Required().RequiredArgument("DATABASE").StoreResult(&database); + + opts.SetFreeArgsMin(0); + NLastGetopt::TOptsParseResult result(&opts, argc, argv); + + ExecutorExample(endpoint, database); + return 0; +} diff --git a/include/ydb-cpp-sdk/client/driver/driver.h b/include/ydb-cpp-sdk/client/driver/driver.h index 6cb1a3a42a1..459bb01eb8d 100644 --- a/include/ydb-cpp-sdk/client/driver/driver.h +++ b/include/ydb-cpp-sdk/client/driver/driver.h @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -54,7 +55,7 @@ class TDriverConfig { //! Enable Ssl. //! caCerts - The buffer containing the PEM encoded root certificates for SSL/TLS connections. //! If this parameter is empty, the default roots will be used. - TDriverConfig& UseSecureConnection(const std::string& caCerts = std::string()); + TDriverConfig& UseSecureConnection(const std::string& caCerts = ""); TDriverConfig& SetUsePerChannelTcpConnection(bool usePerChannel); TDriverConfig& UseClientCertificate(const std::string& clientCert, const std::string& clientPrivateKey); @@ -107,8 +108,7 @@ class TDriverConfig { //! Set policy for balancing //! Params is a optionally field to set policy settings //! default: EBalancingPolicy::UsePreferableLocation - TDriverConfig& SetBalancingPolicy(EBalancingPolicy policy, const std::string& params = std::string()); - + TDriverConfig& SetBalancingPolicy(EBalancingPolicy policy, const std::string& params = ""); //! Set grpc level keep alive. If keepalive ping was delayed more than given timeout //! internal grpc routine fails request with TRANSIENT_FAILURE or TRANSPORT_UNAVAILABLE error //! Note: this timeout should not be too small to prevent fail due to @@ -142,6 +142,11 @@ class TDriverConfig { //! Log backend. TDriverConfig& SetLog(std::unique_ptr&& log); + + //! Set executor for async responses. + //! If not set, default executor will be used. + TDriverConfig& SetExecutor(std::shared_ptr executor); + private: class TImpl; std::shared_ptr Impl_; diff --git a/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h b/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h index 5bf3f4c4ede..52af62a9aad 100644 --- a/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h +++ b/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h @@ -394,7 +394,7 @@ struct TFederatedReadSessionSettings: public NTopic::TReadSessionSettings { //! Executor for handlers. //! If not set, default single threaded executor will be used. //! Shared between subsessions - FLUENT_SETTING(NTopic::IExecutor::TPtr, HandlersExecutor); + FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor); }; //! Federated event handlers. @@ -492,10 +492,10 @@ struct TFederatedTopicClientSettings : public TCommonClientSettingsBase -#include -#include - -#include -#include +#include namespace NYdb::inline V3::NTopic { -class IExecutor: public TThrRefBase { -public: - using TPtr = TIntrusivePtr; - using TFunction = std::function; - - // Is executor asynchronous. - virtual bool IsAsync() const = 0; - - // Post function to execute. - virtual void Post(TFunction&& f) = 0; - - // Start method. - // This method is idempotent. - // It can be called many times. Only the first one has effect. - void Start() { - std::lock_guard guard(StartLock); - if (!Started) { - DoStart(); - Started = true; - } - } - -private: - virtual void DoStart() = 0; - -private: - bool Started = false; - TAdaptiveLock StartLock; -}; -IExecutor::TPtr CreateThreadPoolExecutorAdapter( - std::shared_ptr threadPool); // Thread pool is expected to have been started. -IExecutor::TPtr CreateThreadPoolExecutor(size_t threads); - IExecutor::TPtr CreateSyncExecutor(); } // namespace NYdb::NTopic diff --git a/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h b/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h index 99a485acb5f..91339ecfbfb 100644 --- a/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h +++ b/include/ydb-cpp-sdk/client/types/credentials/oauth2_token_exchange/jwt_token_source.h @@ -36,7 +36,7 @@ struct TJwtTokenSourceParams { class ISigningAlgorithm { public: virtual ~ISigningAlgorithm() = default; -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS virtual std::string sign(const std::string& data, std::error_code& ec) const = 0; #else virtual std::string sign(const std::string& data) const = 0; @@ -54,7 +54,7 @@ struct TJwtTokenSourceParams { { } -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS std::string sign(const std::string& data, std::error_code& ec) const override { return Alg.sign(data, ec); } diff --git a/include/ydb-cpp-sdk/client/types/executor/executor.h b/include/ydb-cpp-sdk/client/types/executor/executor.h new file mode 100644 index 00000000000..977680869c6 --- /dev/null +++ b/include/ydb-cpp-sdk/client/types/executor/executor.h @@ -0,0 +1,58 @@ +#pragma once + +#ifndef YDB_SDK_OSS +#include +#endif + +#include +#include +#include + +namespace NYdb::inline V3 { + +class IExecutor { +public: + using TPtr = std::shared_ptr; + using TFunction = std::function; + + // Start method. + // This method is idempotent. + void Start() { + std::lock_guard guard(StartLock); + if (!Started) { + DoStart(); + Started = true; + } + } + + virtual void Stop() = 0; + + // Post function to execute. + virtual void Post(TFunction&& f) = 0; + + // Is executor asynchronous. + virtual bool IsAsync() const = 0; + + virtual ~IExecutor() = default; + +protected: + virtual void DoStart() = 0; + + bool Started = false; + std::mutex StartLock; +}; + +// Create default executor for thread pool. +IExecutor::TPtr CreateThreadPoolExecutor(std::size_t threadCount, std::size_t maxQueueSize = 0); + +#ifndef YDB_SDK_OSS +// Create executor adapter for util thread pool. +// Thread pool is started and stopped by SDK. +IExecutor::TPtr CreateThreadPoolExecutorAdapter(std::shared_ptr threadPool, std::size_t threadCount, std::size_t maxQueueSize = 0); + +// Create executor adapter for util thread pool. +// Thread pool is expected to have been started and stopped by user. +IExecutor::TPtr CreateExternalThreadPoolExecutorAdapter(std::shared_ptr threadPool); +#endif + +} // namespace NYdb diff --git a/include/ydb-cpp-sdk/client/types/ydb.h b/include/ydb-cpp-sdk/client/types/ydb.h index 53c767ebc93..cb89fa0a5ec 100644 --- a/include/ydb-cpp-sdk/client/types/ydb.h +++ b/include/ydb-cpp-sdk/client/types/ydb.h @@ -60,6 +60,13 @@ class TBalancingPolicy { //! Use pile with preferable state static TBalancingPolicy UsePreferablePileState(EPileState pileState = EPileState::PRIMARY); + TBalancingPolicy(const TBalancingPolicy&) = delete; + TBalancingPolicy(TBalancingPolicy&&) = default; + TBalancingPolicy& operator=(const TBalancingPolicy&) = delete; + TBalancingPolicy& operator=(TBalancingPolicy&&) = default; + + ~TBalancingPolicy(); + class TImpl; private: TBalancingPolicy(std::unique_ptr&& impl); diff --git a/include/ydb-cpp-sdk/client/value/value.h b/include/ydb-cpp-sdk/client/value/value.h index 7ac1b9f250c..543024ef1e2 100644 --- a/include/ydb-cpp-sdk/client/value/value.h +++ b/include/ydb-cpp-sdk/client/value/value.h @@ -18,9 +18,17 @@ namespace NYdb::inline V3 { class TResultSetParser; +// Forward declarations for friend access +namespace NTable { + +class TTableClient; + +} // namespace NTable + //! Representation of YDB type. class TType { friend class TProtoAccessor; + friend class NTable::TTableClient; public: TType(const Ydb::Type& typeProto); TType(Ydb::Type&& typeProto); @@ -264,12 +272,6 @@ struct TUuidValue { } Buf_; }; -namespace NTable { - -class TTableClient; - -} // namespace NTable - //! Representation of YDB value. class TValue { friend class TValueParser; diff --git a/src/api/protos/draft/ydb_maintenance.proto b/src/api/protos/draft/ydb_maintenance.proto index 39b7c67d325..2d3af6457fd 100644 --- a/src/api/protos/draft/ydb_maintenance.proto +++ b/src/api/protos/draft/ydb_maintenance.proto @@ -123,9 +123,21 @@ message LockAction { google.protobuf.Duration duration = 2; } +// Moving tablets away. +message DrainAction { + ActionScope scope = 1; +} + +// Ensuring no new tablets run there. +message CordonAction { + ActionScope scope = 1; +} + message Action { oneof action { LockAction lock_action = 1; + DrainAction drain_action = 2; + CordonAction cordon_action = 3; } } @@ -158,6 +170,8 @@ message ActionState { ACTION_STATUS_PENDING = 1; // Action performed: e.g. lock is taken. ACTION_STATUS_PERFORMED = 2; + // Action is currently in progress + ACTION_STATUS_IN_PROGRESS = 3; } enum ActionReason { @@ -184,7 +198,7 @@ message ActionState { ActionUid action_uid = 2; ActionStatus status = 3; ActionReason reason = 4; - string reason_details = 6; + string details = 6; google.protobuf.Timestamp deadline = 5; } diff --git a/src/api/protos/ydb_common.proto b/src/api/protos/ydb_common.proto index 9e865646b43..1be2cbe5727 100644 --- a/src/api/protos/ydb_common.proto +++ b/src/api/protos/ydb_common.proto @@ -27,3 +27,9 @@ message VirtualTimestamp { uint64 plan_step = 1; uint64 tx_id = 2; } + +enum MetricsLevel { + Database = 0; + Object = 1; + Detailed = 2; +} diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 544f432e1fa..953dca9b3b0 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -113,7 +113,149 @@ message GlobalVectorKMeansTreeIndex { KMeansTreeSettings vector_settings = 3; } -// Represent secondary index +message FulltextIndexSettings { + // Specifies the layout strategy for storing and updating the full-text index + enum Layout { + LAYOUT_UNSPECIFIED = 0; + + // Uses a single flat inverted index table (indexImplTable) + // Example source table: + // ┌────┬────────────────────────────┐ + // │ id │ text │ + // ├────┼────────────────────────────┤ + // │ 1 │ "The quick brown fox" │ + // │ 2 │ "The quick blue hare" │ + // └────┴────────────────────────────┘ + // Example inverted index table (indexImplTable): + // ┌──────────────┬────┐ + // │ __ydb_token │ id │ + // ├──────────────┼────┤ + // │ "blue" │ 2 │ + // │ "brown" │ 1 │ + // │ "fox" │ 1 │ + // │ "hare" │ 2 │ + // │ "quick" │ 1 │ + // │ "quick" │ 2 │ + // │ "The" │ 1 │ + // │ "The" │ 2 │ + // └──────────────┴────┘ + // Supports a single column only + FLAT = 1; + } + + // Specifies how text is tokenized during indexing + enum Tokenizer { + TOKENIZER_UNSPECIFIED = 0; + + // Splits text only by whitespace + // Does not split on punctuation + // Example: + // Text: "foo-bar baz_lorem ipsum" + // Tokens: ["foo-bar", "baz_lorem", "ipsum"] + WHITESPACE = 1; + + // Applies general language-aware tokenization + // Splits text on whitespace and punctuation + // Example: + // Text: "foo-bar baz_lorem ipsum" + // Tokens: ["foo", "bar", "baz", "lorem", "ipsum"] + STANDARD = 2; + + // Treats the entire input as a single token + // No splitting is performed + // Example: + // Text: "Hello World!" + // Tokens: ["Hello World!"] + KEYWORD = 3; + } + + // Represents text analyzers settings + message Analyzers { + // See Tokenizer enum + optional Tokenizer tokenizer = 1; + + // Language used for language-sensitive operations like stopword filtering + // Example: language = "english" + // By default is not specified and no language-specific logic is applied + optional string language = 2; + + // Whether to convert tokens to lowercase + // Example: + // Token: "Quick" + // Output: "quick" + optional bool use_filter_lowercase = 100; + + // Whether to remove common stopwords like "the", "a", "is" + // Example: language = "english" + // Tokens: ["the", "quick", "brown"] + // Output: ["quick", "brown"] + optional bool use_filter_stopwords = 110; + + // Whether to apply character n-gram indexing to each token + // Must be used with filter_ngram_min_length and filter_ngram_max_length + // Example: filter_ngram_min_length = 3, filter_ngram_max_length = 4 + // Token: "search" + // Output: ["sea", "ear", "arc", "rch", "sear", "earc", "arch"] + optional bool use_filter_ngram = 120; + + // Whether to apply edge n-gram indexing (prefix-based) to each token + // Used with filter_ngram_min_length and filter_ngram_max_length + // Example: filter_ngram_min_length = 3, filter_ngram_max_length = 4 + // Token: "search" + // Output: ["sea", "sear"] + optional bool use_filter_edge_ngram = 121; + + // Minimum length of n-grams to generate (inclusive) + // Must be used with use_filter_ngram or use_filter_edge_ngram + // Default value is 3 + optional int32 filter_ngram_min_length = 122 [(Ydb.value) = ">= 0"]; + + // Maximum length of n-grams to generate (inclusive) + // Must be used with use_filter_ngram or use_filter_edge_ngram + // Default value is 4 + optional int32 filter_ngram_max_length = 123 [(Ydb.value) = ">= 0"]; + + // Whether to filter tokens by their length + // Must be used with filter_length_min or filter_length_max + // Example: filter_length_min = 4, filter_length_max = 6 + // Tokens: ["foo", "fooba", "foobar", "foobarbaz"] + // Output: ["fooba", "foobar"] + optional bool use_filter_length = 130; + + // Minimum token length to keep (inclusive) + // Must be used with use_filter_length + optional int32 filter_length_min = 131 [(Ydb.value) = ">= 0"]; + + // Maximum token length to keep (inclusive) + // Must be used with use_filter_length + optional int32 filter_length_max = 132 [(Ydb.value) = ">= 0"]; + } + + // Represents text analyzers settings for a specific column + message ColumnAnalyzers { + // Name of the column to be indexed + optional string column = 1; + + // Analyzer settings specific to this column + Analyzers analyzers = 2; + } + + // See Layout enum + optional Layout layout = 1; + + // List of columns and their fulltext settings + // Currently, this list should contain a single entry with specified analyzers + // Later, some columns may not use analyzers and will be indexed as-is + // This list must always match TableIndex.index_columns + repeated ColumnAnalyzers columns = 2; +} + +message GlobalFulltextIndex { + GlobalIndexSettings settings = 1; + FulltextIndexSettings fulltext_settings = 2; +} + +// Represent table index message TableIndex { // Name of index string name = 1; @@ -125,12 +267,13 @@ message TableIndex { GlobalAsyncIndex global_async_index = 4; GlobalUniqueIndex global_unique_index = 6; GlobalVectorKMeansTreeIndex global_vector_kmeans_tree_index = 7; + GlobalFulltextIndex global_fulltext_index = 8; } // list of columns content to be copied in to index table repeated string data_columns = 5; } -// Represent secondary index with index state +// Represent table index with index state message TableIndexDescription { enum Status { STATUS_UNSPECIFIED = 0; @@ -149,6 +292,7 @@ message TableIndexDescription { GlobalAsyncIndex global_async_index = 5; GlobalUniqueIndex global_unique_index = 8; GlobalVectorKMeansTreeIndex global_vector_kmeans_tree_index = 9; + GlobalFulltextIndex global_fulltext_index = 10; } Status status = 4; // list of columns content to be copied in to index table @@ -648,7 +792,7 @@ message CreateTableRequest { // Table profile TableProfile profile = 5; Ydb.Operations.OperationParams operation_params = 6; - // List of secondary indexes + // List of table indexes repeated TableIndex indexes = 7; // Table rows time to live settings TtlSettings ttl_settings = 8; @@ -726,9 +870,9 @@ message AlterTableRequest { TtlSettings set_ttl_settings = 7; google.protobuf.Empty drop_ttl_settings = 8; } - // Add secondary indexes + // Add table indexes repeated TableIndex add_indexes = 9; - // Remove secondary indexes + // Remove table indexes repeated string drop_indexes = 10; // Change table storage settings StorageSettings alter_storage_settings = 11; diff --git a/src/client/common_client/CMakeLists.txt b/src/client/common_client/CMakeLists.txt index 468b6dc61fa..0515a219609 100644 --- a/src/client/common_client/CMakeLists.txt +++ b/src/client/common_client/CMakeLists.txt @@ -4,8 +4,8 @@ _ydb_sdk_add_library(client-ydb_common_client) target_link_libraries(client-ydb_common_client PUBLIC yutil - impl-ydb_internal-common - client-ydb_types-credentials + impl-internal-common + client-types-credentials ) target_sources(client-ydb_common_client PRIVATE diff --git a/src/client/common_client/impl/CMakeLists.txt b/src/client/common_client/impl/CMakeLists.txt index 63889462767..2be569fc25b 100644 --- a/src/client/common_client/impl/CMakeLists.txt +++ b/src/client/common_client/impl/CMakeLists.txt @@ -2,7 +2,7 @@ _ydb_sdk_add_library(client-ydb_common_client-impl) target_link_libraries(client-ydb_common_client-impl PUBLIC yutil - impl-ydb_internal-grpc_connections + impl-internal-grpc_connections ) target_sources(client-ydb_common_client-impl PRIVATE diff --git a/src/client/coordination/CMakeLists.txt b/src/client/coordination/CMakeLists.txt index 94277bae8bc..e8152b6dd7c 100644 --- a/src/client/coordination/CMakeLists.txt +++ b/src/client/coordination/CMakeLists.txt @@ -4,13 +4,13 @@ target_link_libraries(client-ydb_coordination PUBLIC yutil enum_serialization_runtime api-grpc - impl-ydb_internal-make_request + impl-internal-make_request client-ydb_common_client client-ydb_common_client-impl client-ydb_driver client-ydb_proto - client-ydb_types - client-ydb_types-status + client-types + client-types-status ) target_sources(client-ydb_coordination PRIVATE diff --git a/src/client/coordination/coordination.cpp b/src/client/coordination/coordination.cpp index e6dc2a9cc1c..4c3cacfb689 100644 --- a/src/client/coordination/coordination.cpp +++ b/src/client/coordination/coordination.cpp @@ -1341,7 +1341,7 @@ class TSessionContext : public TThrRefBase { return; } - if (auto* op = FindSentRequest(SessionSelfPingReqId)) { + if (FindSentRequest(SessionSelfPingReqId)) { // We have an active request which has not been answered yet // Cancel everything, it's a timeout :( SessionSelfPingContext.reset(); diff --git a/src/client/datastreams/CMakeLists.txt b/src/client/datastreams/CMakeLists.txt index ec4305e1287..6f4cebdad98 100644 --- a/src/client/datastreams/CMakeLists.txt +++ b/src/client/datastreams/CMakeLists.txt @@ -6,7 +6,7 @@ target_link_libraries(client-ydb_datastreams PUBLIC string_utils-url api-grpc-draft library-operation_id - impl-ydb_internal-make_request + impl-internal-make_request client-ydb_driver ) diff --git a/src/client/driver/CMakeLists.txt b/src/client/driver/CMakeLists.txt index db487c67c6b..5c5b6136a37 100644 --- a/src/client/driver/CMakeLists.txt +++ b/src/client/driver/CMakeLists.txt @@ -3,10 +3,10 @@ _ydb_sdk_add_library(client-ydb_driver) target_link_libraries(client-ydb_driver PUBLIC client-ydb_common_client - client-ydb_types-status + client-types-status PRIVATE - impl-ydb_internal-common - impl-ydb_internal-grpc_connections + impl-internal-common + impl-internal-grpc_connections client-resources ) diff --git a/src/client/driver/driver.cpp b/src/client/driver/driver.cpp index a6df6e8789b..ba6a6559ab6 100644 --- a/src/client/driver/driver.cpp +++ b/src/client/driver/driver.cpp @@ -50,6 +50,7 @@ class TDriverConfig::TImpl : public IConnectionsParams { uint64_t GetMaxOutboundMessageSize() const override { return MaxOutboundMessageSize; } uint64_t GetMaxMessageSize() const override { return MaxMessageSize; } const TLog& GetLog() const override { return Log; } + std::shared_ptr GetExecutor() const override { return Executor; } std::string Endpoint; size_t NetworkThreadsNum = 2; @@ -69,7 +70,7 @@ class TDriverConfig::TImpl : public IConnectionsParams { TCP_KEEPALIVE_INTERVAL }; bool DrainOnDtors = true; - TBalancingPolicy::TImpl BalancingSettings = TBalancingPolicy::TImpl(""); + TBalancingPolicy::TImpl BalancingSettings = TBalancingPolicy::TImpl::UsePreferableLocation(""); TDuration GRpcKeepAliveTimeout = TDuration::Seconds(10); bool GRpcKeepAlivePermitWithoutCalls = true; TDuration SocketIdleTimeout = TDuration::Minutes(6); @@ -78,6 +79,7 @@ class TDriverConfig::TImpl : public IConnectionsParams { uint64_t MaxOutboundMessageSize = 0; uint64_t MaxMessageSize = 0; TLog Log; // Null by default. + std::shared_ptr Executor; }; TDriverConfig::TDriverConfig(const std::string& connectionString) @@ -214,6 +216,11 @@ TDriverConfig& TDriverConfig::SetLog(std::unique_ptr&& log) { return *this; } +TDriverConfig& TDriverConfig::SetExecutor(std::shared_ptr executor) { + Impl_->Executor = executor; + return *this; +} + //////////////////////////////////////////////////////////////////////////////// std::shared_ptr CreateInternalInterface(const TDriver connection) { diff --git a/src/client/export/CMakeLists.txt b/src/client/export/CMakeLists.txt index 7ae0ee465ed..6b802f37189 100644 --- a/src/client/export/CMakeLists.txt +++ b/src/client/export/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries(client-ydb_export PUBLIC client-ydb_common_client-impl client-ydb_driver client-ydb_proto - client-ydb_types-operation + client-types-operation ) target_sources(client-ydb_export PRIVATE diff --git a/src/client/federated_topic/impl/CMakeLists.txt b/src/client/federated_topic/impl/CMakeLists.txt index a2aefdda413..b8eb0b9b713 100644 --- a/src/client/federated_topic/impl/CMakeLists.txt +++ b/src/client/federated_topic/impl/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries(client-ydb_federated_topic-impl PUBLIC persqueue-obfuscate api-grpc-draft api-grpc - impl-ydb_internal-make_request + impl-internal-make_request client-ydb_common_client-impl client-ydb_driver client-ydb_topic-impl diff --git a/src/client/federated_topic/impl/federated_topic_impl.cpp b/src/client/federated_topic/impl/federated_topic_impl.cpp index 53e4f182047..45fc189643c 100644 --- a/src/client/federated_topic/impl/federated_topic_impl.cpp +++ b/src/client/federated_topic/impl/federated_topic_impl.cpp @@ -83,10 +83,10 @@ NThreading::TFuture> TFederated }); } -auto TFederatedTopicClient::TImpl::GetSubsessionHandlersExecutor() -> NTopic::IExecutor::TPtr { +auto TFederatedTopicClient::TImpl::GetSubsessionHandlersExecutor() -> IExecutor::TPtr { with_lock (Lock) { if (!SubsessionHandlersExecutor) { - SubsessionHandlersExecutor = NTopic::CreateThreadPoolExecutor(1); + SubsessionHandlersExecutor = CreateThreadPoolExecutor(1); } return SubsessionHandlersExecutor; } diff --git a/src/client/federated_topic/impl/federated_topic_impl.h b/src/client/federated_topic/impl/federated_topic_impl.h index 9ebd774566d..758e7fac6d3 100644 --- a/src/client/federated_topic/impl/federated_topic_impl.h +++ b/src/client/federated_topic/impl/federated_topic_impl.h @@ -76,8 +76,8 @@ class TFederatedTopicClient::TImpl { private: - // Use single-threaded executor to prevent deadlocks inside subsession event handlers. - NTopic::IExecutor::TPtr GetSubsessionHandlersExecutor(); + // Use single-threaded executor to prevent deadlocks inside subsession event handlers. + IExecutor::TPtr GetSubsessionHandlersExecutor(); private: std::shared_ptr Connections; @@ -86,7 +86,7 @@ class TFederatedTopicClient::TImpl { std::shared_ptr>> ProvidedCodecs = std::make_shared>>(); - NTopic::IExecutor::TPtr SubsessionHandlersExecutor; + IExecutor::TPtr SubsessionHandlersExecutor; TAdaptiveLock Lock; }; diff --git a/src/client/federated_topic/impl/federated_write_session.cpp b/src/client/federated_topic/impl/federated_write_session.cpp index b9c6b5f7c13..8f559f6076a 100644 --- a/src/client/federated_topic/impl/federated_write_session.cpp +++ b/src/client/federated_topic/impl/federated_write_session.cpp @@ -30,7 +30,7 @@ TFederatedWriteSessionImpl::TFederatedWriteSessionImpl( const TFederatedTopicClientSettings& clientSettings, std::shared_ptr observer, std::shared_ptr>> codecs, - NTopic::IExecutor::TPtr subsessionHandlersExecutor + IExecutor::TPtr subsessionHandlersExecutor ) : Settings(settings) , Connections(std::move(connections)) diff --git a/src/client/federated_topic/impl/federated_write_session.h b/src/client/federated_topic/impl/federated_write_session.h index 31c0ed289dc..77376740e90 100644 --- a/src/client/federated_topic/impl/federated_write_session.h +++ b/src/client/federated_topic/impl/federated_write_session.h @@ -27,7 +27,7 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, const TFederatedTopicClientSettings& clientSetttings, std::shared_ptr observer, std::shared_ptr>> codecs, - NTopic::IExecutor::TPtr subsessionHandlersExecutor); + IExecutor::TPtr subsessionHandlersExecutor); ~TFederatedWriteSessionImpl() = default; @@ -108,7 +108,7 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, std::shared_ptr Connections; const NTopic::TTopicClientSettings SubclientSettings; std::shared_ptr>> ProvidedCodecs; - NTopic::IExecutor::TPtr SubsessionHandlersExecutor; + IExecutor::TPtr SubsessionHandlersExecutor; NTopic::IRetryPolicy::IRetryState::TPtr RetryState; std::shared_ptr Observer; @@ -158,7 +158,7 @@ class TFederatedWriteSession : public NTopic::IWriteSession, const TFederatedTopicClientSettings& clientSettings, std::shared_ptr observer, std::shared_ptr>> codecs, - NTopic::IExecutor::TPtr subsessionHandlersExecutor) + IExecutor::TPtr subsessionHandlersExecutor) : TContextOwner(settings, std::move(connections), clientSettings, std::move(observer), codecs, subsessionHandlersExecutor) {} NThreading::TFuture WaitEvent() override { diff --git a/src/client/federated_topic/ut/basic_usage_ut.cpp b/src/client/federated_topic/ut/basic_usage_ut.cpp index 9e00dfcd4f2..ce87d140bdc 100644 --- a/src/client/federated_topic/ut/basic_usage_ut.cpp +++ b/src/client/federated_topic/ut/basic_usage_ut.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include @@ -275,7 +274,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + IExecutor::TPtr executor = NPersQueue::CreateSyncExecutor(); writeSettings.CompressionExecutor(executor); auto& client = setup->GetPersQueueClient(); @@ -534,7 +533,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + IExecutor::TPtr executor = NPersQueue::CreateSyncExecutor(); writeSettings.CompressionExecutor(executor); auto& client = setup->GetPersQueueClient(); @@ -640,7 +639,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + IExecutor::TPtr executor = NPersQueue::CreateSyncExecutor(); writeSettings.CompressionExecutor(executor); auto& client = setup->GetPersQueueClient(); @@ -660,7 +659,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(setup->GetTestTopic() + "-mirrored-from-dc2").MessageGroupId("src_id"); writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + IExecutor::TPtr executor = NPersQueue::CreateSyncExecutor(); writeSettings.CompressionExecutor(executor); auto& client = setup->GetPersQueueClient(); @@ -680,7 +679,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(setup->GetTestTopic() + "-mirrored-from-dc3").MessageGroupId("src_id"); writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + IExecutor::TPtr executor = NPersQueue::CreateSyncExecutor(); writeSettings.CompressionExecutor(executor); auto& client = setup->GetPersQueueClient(); diff --git a/src/client/helpers/CMakeLists.txt b/src/client/helpers/CMakeLists.txt index dc47a456af9..c9e604b1915 100644 --- a/src/client/helpers/CMakeLists.txt +++ b/src/client/helpers/CMakeLists.txt @@ -3,9 +3,9 @@ _ydb_sdk_add_library(client-helpers) target_link_libraries(client-helpers PUBLIC yutil - client-ydb_types-credentials-oauth2 + client-types-credentials-oauth2 client-iam - client-ydb_types-credentials + client-types-credentials ) target_sources(client-helpers diff --git a/src/client/iam_private/iam.cpp b/src/client/iam_private/iam.cpp index 054118e73af..e6271b68869 100644 --- a/src/client/iam_private/iam.cpp +++ b/src/client/iam_private/iam.cpp @@ -32,7 +32,7 @@ TCredentialsProviderFactoryPtr CreateIamServiceCredentialsProviderFactory(const CreateIamTokenForServiceRequest, CreateIamTokenResponse, IamTokenService - >>(std::move(params)); + >>(params); } } diff --git a/src/client/impl/CMakeLists.txt b/src/client/impl/CMakeLists.txt index 2cb7d4e142e..9e04f134b37 100644 --- a/src/client/impl/CMakeLists.txt +++ b/src/client/impl/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(endpoints) +add_subdirectory(executor) add_subdirectory(internal) add_subdirectory(session) add_subdirectory(stats) diff --git a/src/client/impl/executor/CMakeLists.txt b/src/client/impl/executor/CMakeLists.txt new file mode 100644 index 00000000000..2893232d497 --- /dev/null +++ b/src/client/impl/executor/CMakeLists.txt @@ -0,0 +1,11 @@ +_ydb_sdk_add_library(impl-executor) + +target_sources(impl-executor PRIVATE + executor_impl.cpp +) + +target_link_libraries(impl-executor PUBLIC + yutil +) + +_ydb_sdk_install_targets(TARGETS impl-executor) diff --git a/src/client/impl/executor/executor_impl.cpp b/src/client/impl/executor/executor_impl.cpp new file mode 100644 index 00000000000..ab6d29d54e5 --- /dev/null +++ b/src/client/impl/executor/executor_impl.cpp @@ -0,0 +1,46 @@ +#define INCLUDE_YDB_INTERNAL_H +#include "executor_impl.h" +#undef INCLUDE_YDB_INTERNAL_H + +namespace NYdb::inline V3 { + +TThreadPoolExecutor::TThreadPoolExecutor(std::shared_ptr threadPool) + : ThreadPool_(threadPool) + , IsStarted_(true) + , DontStop_(true) +{ +} + +TThreadPoolExecutor::TThreadPoolExecutor(std::shared_ptr threadPool, std::size_t threadCount, std::size_t maxQueueSize) + : ThreadPool_(threadPool) + , IsStarted_(false) + , DontStop_(false) + , ThreadCount_(threadCount) + , MaxQueueSize_(maxQueueSize) +{ +} + +void TThreadPoolExecutor::DoStart() { + if (IsStarted_) { + return; + } + ThreadPool_->Start(ThreadCount_, MaxQueueSize_); +} + +void TThreadPoolExecutor::Stop() { + if (DontStop_) { + return; + } + ThreadPool_->Stop(); +} + + +void TThreadPoolExecutor::Post(std::function&& f) { + ThreadPool_->SafeAddFunc(std::move(f)); +} + +bool TThreadPoolExecutor::IsAsync() const { + return true; +} + +} diff --git a/src/client/impl/executor/executor_impl.h b/src/client/impl/executor/executor_impl.h new file mode 100644 index 00000000000..34cfe5c5ca2 --- /dev/null +++ b/src/client/impl/executor/executor_impl.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include + +#include + + +namespace NYdb::inline V3 { + +class TThreadPoolExecutor : public IExecutor { +public: + TThreadPoolExecutor(std::shared_ptr threadPool); + TThreadPoolExecutor(std::shared_ptr threadPool, std::size_t threadCount, std::size_t maxQueueSize); + + void DoStart() override; + void Stop() override; + void Post(std::function&& f) override; + bool IsAsync() const override; + +private: + std::shared_ptr ThreadPool_; + + const bool IsStarted_; + const bool DontStop_; + const std::size_t ThreadCount_ = 0; + const std::size_t MaxQueueSize_ = 0; +}; + +} diff --git a/src/client/impl/internal/common/CMakeLists.txt b/src/client/impl/internal/common/CMakeLists.txt index c6220cfa832..d64f631c517 100644 --- a/src/client/impl/internal/common/CMakeLists.txt +++ b/src/client/impl/internal/common/CMakeLists.txt @@ -1,16 +1,16 @@ -_ydb_sdk_add_library(impl-ydb_internal-common) +_ydb_sdk_add_library(impl-internal-common) -target_link_libraries(impl-ydb_internal-common PUBLIC +target_link_libraries(impl-internal-common PUBLIC yutil grpc-client yql-public-issue ) -target_sources(impl-ydb_internal-common PRIVATE +target_sources(impl-internal-common PRIVATE balancing_policies.cpp parser.cpp getenv.cpp client_pid.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-common) +_ydb_sdk_install_targets(TARGETS impl-internal-common) diff --git a/src/client/impl/internal/common/balancing_policies.cpp b/src/client/impl/internal/common/balancing_policies.cpp index 48431810815..b3b943c63be 100644 --- a/src/client/impl/internal/common/balancing_policies.cpp +++ b/src/client/impl/internal/common/balancing_policies.cpp @@ -3,16 +3,24 @@ namespace NYdb::inline V3 { -std::unique_ptr TBalancingPolicy::TImpl::UseAllNodes() { - return std::make_unique(); +TBalancingPolicy::TImpl TBalancingPolicy::TImpl::UseAllNodes() { + TBalancingPolicy::TImpl impl; + impl.PolicyType = EPolicyType::UseAllNodes; + return impl; } -std::unique_ptr TBalancingPolicy::TImpl::UsePreferableLocation(const std::string& location) { - return std::make_unique(location); +TBalancingPolicy::TImpl TBalancingPolicy::TImpl::UsePreferableLocation(const std::string& location) { + TBalancingPolicy::TImpl impl; + impl.PolicyType = EPolicyType::UsePreferableLocation; + impl.Location = location; + return impl; } -std::unique_ptr TBalancingPolicy::TImpl::UsePreferablePileState(EPileState pileState) { - return std::make_unique(pileState); +TBalancingPolicy::TImpl TBalancingPolicy::TImpl::UsePreferablePileState(EPileState pileState) { + TBalancingPolicy::TImpl impl; + impl.PolicyType = EPolicyType::UsePreferablePileState; + impl.PileState = pileState; + return impl; } } diff --git a/src/client/impl/internal/common/balancing_policies.h b/src/client/impl/internal/common/balancing_policies.h index 01efe15c61c..6c7e1c0905d 100644 --- a/src/client/impl/internal/common/balancing_policies.h +++ b/src/client/impl/internal/common/balancing_policies.h @@ -17,25 +17,11 @@ class TBalancingPolicy::TImpl { UsePreferablePileState }; - static std::unique_ptr UseAllNodes(); + static TImpl UseAllNodes(); - static std::unique_ptr UsePreferableLocation(const std::string& location); + static TImpl UsePreferableLocation(const std::string& location); - static std::unique_ptr UsePreferablePileState(EPileState pileState); - - TImpl() - : PolicyType(EPolicyType::UseAllNodes) - {} - - TImpl(const std::string& location) - : PolicyType(EPolicyType::UsePreferableLocation) - , Location(location) - {} - - TImpl(EPileState pileState) - : PolicyType(EPolicyType::UsePreferablePileState) - , PileState(pileState) - {} + static TImpl UsePreferablePileState(EPileState pileState); EPolicyType PolicyType; diff --git a/src/client/impl/internal/db_driver_state/CMakeLists.txt b/src/client/impl/internal/db_driver_state/CMakeLists.txt index 910d497c462..09089e4c10c 100644 --- a/src/client/impl/internal/db_driver_state/CMakeLists.txt +++ b/src/client/impl/internal/db_driver_state/CMakeLists.txt @@ -1,19 +1,19 @@ -_ydb_sdk_add_library(impl-ydb_internal-db_driver_state) +_ydb_sdk_add_library(impl-internal-db_driver_state) -target_link_libraries(impl-ydb_internal-db_driver_state PUBLIC +target_link_libraries(impl-internal-db_driver_state PUBLIC yutil string_utils-quote threading-future client-impl-ydb_endpoints - impl-ydb_internal-logger - impl-ydb_internal-plain_status - client-ydb_types-credentials + impl-internal-logger + impl-internal-plain_status + client-types-credentials ) -target_sources(impl-ydb_internal-db_driver_state PRIVATE +target_sources(impl-internal-db_driver_state PRIVATE authenticator.cpp endpoint_pool.cpp state.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-db_driver_state) +_ydb_sdk_install_targets(TARGETS impl-internal-db_driver_state) diff --git a/src/client/impl/internal/grpc_connections/CMakeLists.txt b/src/client/impl/internal/grpc_connections/CMakeLists.txt index 5b3d03f673b..052bd17505d 100644 --- a/src/client/impl/internal/grpc_connections/CMakeLists.txt +++ b/src/client/impl/internal/grpc_connections/CMakeLists.txt @@ -1,20 +1,20 @@ -_ydb_sdk_add_library(impl-ydb_internal-grpc_connections) +_ydb_sdk_add_library(impl-internal-grpc_connections) -target_link_libraries(impl-ydb_internal-grpc_connections PUBLIC +target_link_libraries(impl-internal-grpc_connections PUBLIC yutil api-grpc api-protos - impl-ydb_internal-db_driver_state - impl-ydb_internal-plain_status - impl-ydb_internal-thread_pool + impl-internal-db_driver_state + impl-internal-plain_status client-impl-ydb_stats client-resources - client-ydb_types-exceptions + client-types-exceptions + client-types-executor ) -target_sources(impl-ydb_internal-grpc_connections PRIVATE +target_sources(impl-internal-grpc_connections PRIVATE actions.cpp grpc_connections.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-grpc_connections) +_ydb_sdk_install_targets(TARGETS impl-internal-grpc_connections) diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.cpp b/src/client/impl/internal/grpc_connections/grpc_connections.cpp index aa5bfc21b4d..ab12fb06e63 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/internal/grpc_connections/grpc_connections.cpp @@ -3,6 +3,7 @@ #include + namespace NYdb::inline V3 { bool IsTokenCorrect(const std::string& in) { @@ -137,7 +138,6 @@ class TScheduledFuture : public TScheduledObject { TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr params) : MetricRegistryPtr_(nullptr) , ClientThreadsNum_(params->GetClientThreadsNum()) - , ResponseQueue_(CreateThreadPool(ClientThreadsNum_)) , DefaultDiscoveryEndpoint_(params->GetEndpoint()) , SslCredentials_(params->GetSslCredentials()) , DefaultDatabase_(params->GetDatabase()) @@ -180,8 +180,14 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr p AddPeriodicTask(channelPoolUpdateWrapper, SocketIdleTimeout_ * 0.1); } #endif - //TAdaptiveThreadPool ignores params - ResponseQueue_->Start(ClientThreadsNum_, MaxQueuedResponses_); + if (params->GetExecutor()) { + ResponseQueue_ = params->GetExecutor(); + } else { + // TAdaptiveThreadPool ignores params + ResponseQueue_ = CreateThreadPoolExecutor(ClientThreadsNum_, MaxQueuedRequests_); + } + + ResponseQueue_->Start(); if (!DefaultDatabase_.empty()) { DefaultState_ = StateTracker_.GetDriverState( DefaultDatabase_, @@ -427,7 +433,9 @@ const TLog& TGRpcConnectionsImpl::GetLog() const { } void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) { - Y_ENSURE(ResponseQueue_->Add(action)); + ResponseQueue_->Post([action]() { + action->Process(nullptr); + }); } } // namespace NYdb diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.h b/src/client/impl/internal/grpc_connections/grpc_connections.h index 8008cb7c834..e28c7af322e 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/internal/grpc_connections/grpc_connections.h @@ -10,12 +10,12 @@ #include #include #include -#include #include #include #include + namespace NYdb::inline V3 { constexpr TDuration GRPC_KEEP_ALIVE_TIMEOUT_FOR_DISCOVERY = TDuration::Seconds(10); @@ -750,8 +750,8 @@ class TGRpcConnectionsImpl std::mutex ExtensionsLock_; ::NMonitoring::TMetricRegistry* MetricRegistryPtr_ = nullptr; - const size_t ClientThreadsNum_; - std::unique_ptr ResponseQueue_; + const std::size_t ClientThreadsNum_; + std::shared_ptr ResponseQueue_; const std::string DefaultDiscoveryEndpoint_; const TSslCredentials SslCredentials_; @@ -759,16 +759,16 @@ class TGRpcConnectionsImpl std::shared_ptr DefaultCredentialsProviderFactory_; TDbDriverStateTracker StateTracker_; const EDiscoveryMode DefaultDiscoveryMode_; - const i64 MaxQueuedRequests_; - const i64 MaxQueuedResponses_; + const std::int64_t MaxQueuedRequests_; + const std::int64_t MaxQueuedResponses_; const bool DrainOnDtors_; const TBalancingPolicy::TImpl BalancingSettings_; const TDuration GRpcKeepAliveTimeout_; const bool GRpcKeepAlivePermitWithoutCalls_; - const ui64 MemoryQuota_; - const ui64 MaxInboundMessageSize_; - const ui64 MaxOutboundMessageSize_; - const ui64 MaxMessageSize_; + const std::uint64_t MemoryQuota_; + const std::uint64_t MaxInboundMessageSize_; + const std::uint64_t MaxOutboundMessageSize_; + const std::uint64_t MaxMessageSize_; std::atomic_int64_t QueuedRequests_; const NYdbGrpc::TTcpKeepAliveSettings TcpKeepAliveSettings_; @@ -784,7 +784,7 @@ class TGRpcConnectionsImpl IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb; - const size_t NetworkThreadsNum_; + const std::size_t NetworkThreadsNum_; bool UsePerChannelTcpConnection_; // Must be the last member (first called destructor) NYdbGrpc::TGRpcClientLow GRpcClientLow_; diff --git a/src/client/impl/internal/grpc_connections/params.h b/src/client/impl/internal/grpc_connections/params.h index 1267c1fa01c..2bc9f4567c5 100644 --- a/src/client/impl/internal/grpc_connections/params.h +++ b/src/client/impl/internal/grpc_connections/params.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace NYdb::inline V3 { @@ -34,6 +35,7 @@ class IConnectionsParams { virtual uint64_t GetMaxInboundMessageSize() const = 0; virtual uint64_t GetMaxOutboundMessageSize() const = 0; virtual uint64_t GetMaxMessageSize() const = 0; + virtual std::shared_ptr GetExecutor() const = 0; }; } // namespace NYdb diff --git a/src/client/impl/internal/logger/CMakeLists.txt b/src/client/impl/internal/logger/CMakeLists.txt index f078e7ebe41..bf8952cbbf9 100644 --- a/src/client/impl/internal/logger/CMakeLists.txt +++ b/src/client/impl/internal/logger/CMakeLists.txt @@ -1,12 +1,12 @@ -_ydb_sdk_add_library(impl-ydb_internal-logger) +_ydb_sdk_add_library(impl-internal-logger) -target_link_libraries(impl-ydb_internal-logger PUBLIC +target_link_libraries(impl-internal-logger PUBLIC yutil logger ) -target_sources(impl-ydb_internal-logger PRIVATE +target_sources(impl-internal-logger PRIVATE log.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-logger) +_ydb_sdk_install_targets(TARGETS impl-internal-logger) diff --git a/src/client/impl/internal/make_request/CMakeLists.txt b/src/client/impl/internal/make_request/CMakeLists.txt index ce339e5cb7f..d8601d962f3 100644 --- a/src/client/impl/internal/make_request/CMakeLists.txt +++ b/src/client/impl/internal/make_request/CMakeLists.txt @@ -1,13 +1,13 @@ -_ydb_sdk_add_library(impl-ydb_internal-make_request) +_ydb_sdk_add_library(impl-internal-make_request) -target_link_libraries(impl-ydb_internal-make_request PUBLIC +target_link_libraries(impl-internal-make_request PUBLIC yutil protobuf::libprotobuf api-protos ) -target_sources(impl-ydb_internal-make_request PRIVATE +target_sources(impl-internal-make_request PRIVATE make.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-make_request) +_ydb_sdk_install_targets(TARGETS impl-internal-make_request) diff --git a/src/client/impl/internal/plain_status/CMakeLists.txt b/src/client/impl/internal/plain_status/CMakeLists.txt index 0e2dcc91ba3..17b4dfdca07 100644 --- a/src/client/impl/internal/plain_status/CMakeLists.txt +++ b/src/client/impl/internal/plain_status/CMakeLists.txt @@ -1,14 +1,14 @@ -_ydb_sdk_add_library(impl-ydb_internal-plain_status) +_ydb_sdk_add_library(impl-internal-plain_status) -target_link_libraries(impl-ydb_internal-plain_status PUBLIC +target_link_libraries(impl-internal-plain_status PUBLIC yutil protobuf::libprotobuf grpc-client yql-public-issue ) -target_sources(impl-ydb_internal-plain_status PRIVATE +target_sources(impl-internal-plain_status PRIVATE status.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-plain_status) +_ydb_sdk_install_targets(TARGETS impl-internal-plain_status) diff --git a/src/client/impl/internal/retry/CMakeLists.txt b/src/client/impl/internal/retry/CMakeLists.txt index 03440c81d2b..bde0f9e0aeb 100644 --- a/src/client/impl/internal/retry/CMakeLists.txt +++ b/src/client/impl/internal/retry/CMakeLists.txt @@ -1,12 +1,12 @@ -_ydb_sdk_add_library(impl-ydb_internal-retry) +_ydb_sdk_add_library(impl-internal-retry) -target_link_libraries(impl-ydb_internal-retry PUBLIC +target_link_libraries(impl-internal-retry PUBLIC yutil - impl-ydb_internal-grpc_connections + impl-internal-grpc_connections ) -target_sources(impl-ydb_internal-retry PRIVATE +target_sources(impl-internal-retry PRIVATE retry.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-retry) +_ydb_sdk_install_targets(TARGETS impl-internal-retry) diff --git a/src/client/impl/internal/thread_pool/CMakeLists.txt b/src/client/impl/internal/thread_pool/CMakeLists.txt index 386db7a454f..dd7f8ca0b13 100644 --- a/src/client/impl/internal/thread_pool/CMakeLists.txt +++ b/src/client/impl/internal/thread_pool/CMakeLists.txt @@ -1,11 +1,11 @@ -_ydb_sdk_add_library(impl-ydb_internal-thread_pool) +_ydb_sdk_add_library(impl-internal-thread_pool) -target_link_libraries(impl-ydb_internal-thread_pool PUBLIC +target_link_libraries(impl-internal-thread_pool PUBLIC yutil ) -target_sources(impl-ydb_internal-thread_pool PRIVATE +target_sources(impl-internal-thread_pool PRIVATE pool.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-thread_pool) +_ydb_sdk_install_targets(TARGETS impl-internal-thread_pool) diff --git a/src/client/impl/internal/value_helpers/CMakeLists.txt b/src/client/impl/internal/value_helpers/CMakeLists.txt index 00591fe67c9..c9d024d6d38 100644 --- a/src/client/impl/internal/value_helpers/CMakeLists.txt +++ b/src/client/impl/internal/value_helpers/CMakeLists.txt @@ -1,13 +1,13 @@ -_ydb_sdk_add_library(impl-ydb_internal-value_helpers) +_ydb_sdk_add_library(impl-internal-value_helpers) -target_link_libraries(impl-ydb_internal-value_helpers PUBLIC +target_link_libraries(impl-internal-value_helpers PUBLIC yutil api-protos - client-ydb_types-fatal_error_handlers + client-types-fatal_error_handlers ) -target_sources(impl-ydb_internal-value_helpers PRIVATE +target_sources(impl-internal-value_helpers PRIVATE helpers.cpp ) -_ydb_sdk_install_targets(TARGETS impl-ydb_internal-value_helpers) +_ydb_sdk_install_targets(TARGETS impl-internal-value_helpers) diff --git a/src/client/impl/session/CMakeLists.txt b/src/client/impl/session/CMakeLists.txt index d22a9cb3c4c..11d00a070e0 100644 --- a/src/client/impl/session/CMakeLists.txt +++ b/src/client/impl/session/CMakeLists.txt @@ -5,7 +5,7 @@ target_link_libraries(impl-session PUBLIC threading-future api-protos client-impl-ydb_endpoints - client-ydb_types-operation + client-types-operation ) target_sources(impl-session PRIVATE diff --git a/src/client/import/CMakeLists.txt b/src/client/import/CMakeLists.txt index 1a030da4a7b..6f4c0ddb9a6 100644 --- a/src/client/import/CMakeLists.txt +++ b/src/client/import/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries(client-ydb_import PUBLIC client-ydb_common_client-impl client-ydb_driver client-ydb_proto - client-ydb_types-operation + client-types-operation ) target_sources(client-ydb_import diff --git a/src/client/import/import.cpp b/src/client/import/import.cpp index 6ec8b99fd44..a53c780ae54 100644 --- a/src/client/import/import.cpp +++ b/src/client/import/import.cpp @@ -262,6 +262,10 @@ TAsyncImportFromS3Response TImportClient::ImportFromS3(const TImportFromS3Settin settingsProto.set_no_acl(settings.NoACL_.value()); } + if (settings.SkipChecksumValidation_) { + settingsProto.set_skip_checksum_validation(settings.SkipChecksumValidation_.value()); + } + if (settings.SourcePrefix_) { settingsProto.set_source_prefix(settings.SourcePrefix_.value()); } diff --git a/src/client/monitoring/CMakeLists.txt b/src/client/monitoring/CMakeLists.txt index 5c458ebabcf..e7930f59a5c 100644 --- a/src/client/monitoring/CMakeLists.txt +++ b/src/client/monitoring/CMakeLists.txt @@ -4,7 +4,7 @@ target_link_libraries(client-ydb_monitoring PUBLIC yutil enum_serialization_runtime client-ydb_proto - impl-ydb_internal-make_request + impl-internal-make_request client-ydb_common_client-impl client-ydb_driver ) diff --git a/src/client/operation/CMakeLists.txt b/src/client/operation/CMakeLists.txt index d34e2e92da8..c0b0c10d196 100644 --- a/src/client/operation/CMakeLists.txt +++ b/src/client/operation/CMakeLists.txt @@ -10,7 +10,7 @@ target_link_libraries(client-ydb_operation PUBLIC client-ydb_export client-ydb_import client-ss_tasks - client-ydb_types-operation + client-types-operation ) target_sources(client-ydb_operation PRIVATE diff --git a/src/client/params/CMakeLists.txt b/src/client/params/CMakeLists.txt index 798fb859743..ece10b09f35 100644 --- a/src/client/params/CMakeLists.txt +++ b/src/client/params/CMakeLists.txt @@ -3,7 +3,7 @@ _ydb_sdk_add_library(client-ydb_params) target_link_libraries(client-ydb_params PUBLIC yutil api-protos - client-ydb_types-fatal_error_handlers + client-types-fatal_error_handlers client-ydb_value ) diff --git a/src/client/persqueue_public/impl/CMakeLists.txt b/src/client/persqueue_public/impl/CMakeLists.txt index d8ab632a8b4..2f8bab2ef77 100644 --- a/src/client/persqueue_public/impl/CMakeLists.txt +++ b/src/client/persqueue_public/impl/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries(client-ydb_persqueue_public-impl PUBLIC string_utils-url persqueue-obfuscate api-grpc-draft - impl-ydb_internal-make_request + impl-internal-make_request client-ydb_common_client-impl client-ydb_driver client-ydb_persqueue_public-include diff --git a/src/client/persqueue_public/impl/write_session.cpp b/src/client/persqueue_public/impl/write_session.cpp index efafbfa2d5d..07e90296885 100644 --- a/src/client/persqueue_public/impl/write_session.cpp +++ b/src/client/persqueue_public/impl/write_session.cpp @@ -117,9 +117,9 @@ std::optional TSimpleBlockingWriteSession::WaitForToken(cons if (auto* readyEvent = std::get_if(&event)) { Y_ABORT_UNLESS(!token.has_value()); token = std::move(readyEvent->ContinuationToken); - } else if (auto* ackEvent = std::get_if(&event)) { + } else if (std::get_if(&event)) { // discard - } else if (auto* closeSessionEvent = std::get_if(&event)) { + } else if (std::get_if(&event)) { Closed.store(true); return std::nullopt; } diff --git a/src/client/persqueue_public/include/aliases.h b/src/client/persqueue_public/include/aliases.h index 0cd8c0c8354..69043cb685f 100644 --- a/src/client/persqueue_public/include/aliases.h +++ b/src/client/persqueue_public/include/aliases.h @@ -36,9 +36,6 @@ using NTopic::TSessionClosedHandler; // TODO reuse TPrintable // executor -using NTopic::IExecutor; -using NTopic::CreateThreadPoolExecutorAdapter; -using NTopic::CreateThreadPoolExecutor; using NTopic::CreateSyncExecutor; // retry policy diff --git a/src/client/persqueue_public/ut/compress_executor_ut.cpp b/src/client/persqueue_public/ut/compress_executor_ut.cpp index baed9bd81aa..a850901de86 100644 --- a/src/client/persqueue_public/ut/compress_executor_ut.cpp +++ b/src/client/persqueue_public/ut/compress_executor_ut.cpp @@ -26,7 +26,7 @@ Y_UNIT_TEST_SUITE(CompressExecutor) { Y_UNIT_TEST(TestExecutorMemUsage) { auto queue = std::make_shared>(); auto setup = std::make_shared(TEST_CASE_NAME); - auto executor = MakeIntrusive(queue); + auto executor = std::make_shared(queue); auto config = setup->GetWriteSessionSettings(); auto memUsageLimit = 1_KB; config.MaxMemoryUsage(memUsageLimit); diff --git a/src/client/persqueue_public/ut/read_session_ut.cpp b/src/client/persqueue_public/ut/read_session_ut.cpp index 5fc21e1749a..32d8d55149c 100644 --- a/src/client/persqueue_public/ut/read_session_ut.cpp +++ b/src/client/persqueue_public/ut/read_session_ut.cpp @@ -17,7 +17,7 @@ using namespace NYdb; using namespace NYdb::NPersQueue; -using IExecutor = NYdb::NPersQueue::IExecutor; +using IExecutor = NYdb::IExecutor; using namespace ::testing; // Google mock. #define UNIT_ASSERT_EVENT_TYPE(event, type) \ @@ -550,6 +550,10 @@ class TReorderingExecutor : public ::IExecutor { Executor->Start(); } + void Stop() override { + Executor->Stop(); + } + size_t GetTasksAdded() { with_lock (Lock) { return TasksAdded; @@ -564,19 +568,6 @@ class TReorderingExecutor : public ::IExecutor { std::vector Functions; }; -class TSynchronousExecutor : public ::IExecutor { - bool IsAsync() const override { - return false; - } - - void Post(TFunction&& f) override { - f(); - } - - void DoStart() override { - } -}; - extern TLogFormatter NYdb::GetPrefixLogFormatter(const std::string& prefix); // Defined in ydb.cpp. TReadSessionImplTestSetup::TReadSessionImplTestSetup() { @@ -608,7 +599,7 @@ ::IExecutor::TPtr TReadSessionImplTestSetup::GetDefaultExecutor() { if (!DefaultExecutor) { ThreadPool = std::make_shared(); ThreadPool->Start(1); - DefaultExecutor = CreateThreadPoolExecutorAdapter(ThreadPool); + DefaultExecutor = CreateExternalThreadPoolExecutorAdapter(ThreadPool); } return DefaultExecutor; } @@ -1184,7 +1175,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { Y_UNIT_TEST(ProperlyOrdersDecompressedData) { TReadSessionImplTestSetup setup; - setup.Settings.DecompressionExecutor(new TReorderingExecutor()); + setup.Settings.DecompressionExecutor(std::make_shared()); setup.SuccessfulInit(); TPartitionStream::TPtr stream = setup.CreatePartitionStream(); for (ui64 i = 1; i <= 2; ++i) { @@ -1209,7 +1200,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { Y_UNIT_TEST(BrokenCompressedData) { TReadSessionImplTestSetup setup; - setup.Settings.DecompressionExecutor(new TReorderingExecutor(1)); + setup.Settings.DecompressionExecutor(std::make_shared(1)); setup.SuccessfulInit(); TPartitionStream::TPtr stream = setup.CreatePartitionStream(); setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() @@ -1286,7 +1277,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { } Y_UNIT_TEST(DecompressWithSynchronousExecutor) { - DecompressImpl(Ydb::PersQueue::V1::CODEC_ZSTD, "msg", new TSynchronousExecutor()); + DecompressImpl(Ydb::PersQueue::V1::CODEC_ZSTD, "msg", CreateSyncExecutor()); } TString GenerateMessageData(size_t size) { @@ -1313,7 +1304,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { if (memoryLimit) { setup.Settings.MaxMemoryUsageBytes(memoryLimit); } - auto executor = MakeIntrusive(reorderedCycleSize); + auto executor = std::make_shared(reorderedCycleSize); setup.Settings.DecompressionExecutor(executor); setup.SuccessfulInit(); TPartitionStream::TPtr stream = setup.CreatePartitionStream(); @@ -1548,7 +1539,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { Y_UNIT_TEST(HoleBetweenOffsets) { TReadSessionImplTestSetup setup; - setup.Settings.DecompressionExecutor(MakeIntrusive(2ull)); + setup.Settings.DecompressionExecutor(std::make_shared(2ull)); setup.SuccessfulInit(); TPartitionStream::TPtr stream = setup.CreatePartitionStream(); setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() @@ -1656,7 +1647,7 @@ Y_UNIT_TEST_SUITE(ReadSessionImplTest) { Y_UNIT_TEST(DataReceivedCallback) { TReadSessionImplTestSetup setup; - setup.Settings.DecompressionExecutor(MakeIntrusive(2ull)); + setup.Settings.DecompressionExecutor(std::make_shared(2ull)); auto calledPromise = NThreading::NewPromise(); int time = 0; setup.Settings.EventHandlers_.DataReceivedHandler([&](TReadSessionEvent::TDataReceivedEvent& event) { diff --git a/src/client/persqueue_public/ut/ut_utils/ut_utils.h b/src/client/persqueue_public/ut/ut_utils/ut_utils.h index 804e5a447aa..57379111e99 100644 --- a/src/client/persqueue_public/ut/ut_utils/ut_utils.h +++ b/src/client/persqueue_public/ut/ut_utils/ut_utils.h @@ -357,29 +357,29 @@ struct TYdbPqTestRetryPolicy : IRetryPolicy { class TYdbPqTestExecutor : public IAsyncExecutor { public: TYdbPqTestExecutor(std::shared_ptr> idsQueue) - : Stop() - , ExecIdsQueue(idsQueue) - , Thread([idsQueue, this]() { - while(!Stop) { + : Stop_(false) + , ExecIdsQueue_(idsQueue) + , Thread_([idsQueue, this]() { + while(!Stop_) { TFunction f; - while (TasksQueue.Dequeue(&f)) { - ++CurrentTaskId; - Cerr << "Enqueue task with id " << CurrentTaskId << Endl; - Tasks[CurrentTaskId] = f; + while (TasksQueue_.Dequeue(&f)) { + ++CurrentTaskId_; + Cerr << "Enqueue task with id " << CurrentTaskId_ << Endl; + Tasks_[CurrentTaskId_] = f; } ui64 id = 0; - while (ExecIdsQueue->Dequeue(&id)) { - ExecIds.push(id); + while (ExecIdsQueue_->Dequeue(&id)) { + ExecIds_.push(id); Cerr << "Got ok to execute task with id " << id << Endl; } - while (!ExecIds.empty()) { - auto id = ExecIds.front(); - auto iter = Tasks.find(id); - if (iter == Tasks.end()) + while (!ExecIds_.empty()) { + auto id = ExecIds_.front(); + auto iter = Tasks_.find(id); + if (iter == Tasks_.end()) break; Cerr << "Executing compression of " << id << Endl; - ExecIds.pop(); + ExecIds_.pop(); try { (iter->second)(); } catch (...) { @@ -387,7 +387,7 @@ class TYdbPqTestExecutor : public IAsyncExecutor { Y_ABORT(); } Cerr << "Compression of " << id << " Done\n"; - Tasks.erase(iter); + Tasks_.erase(iter); } } @@ -395,31 +395,34 @@ class TYdbPqTestExecutor : public IAsyncExecutor { { } ~TYdbPqTestExecutor() { - Stop = true; - Thread.Join(); + Stop_ = true; + Thread_.Join(); } void PostImpl(std::vector&& fs) override { for (auto& f : fs) { - TasksQueue.Enqueue(std::move(f)); + TasksQueue_.Enqueue(std::move(f)); } } void PostImpl(TFunction&& f) override { - TasksQueue.Enqueue(std::move(f)); + TasksQueue_.Enqueue(std::move(f)); } void DoStart() override { - Thread.Start(); + Thread_.Start(); + } + + void Stop() override { } private: - std::atomic_bool Stop; - TLockFreeQueue TasksQueue; - std::shared_ptr> ExecIdsQueue; - THashMap Tasks; - TQueue ExecIds; - ui64 CurrentTaskId = 0; - TThread Thread; + std::atomic_bool Stop_; + TLockFreeQueue TasksQueue_; + std::shared_ptr> ExecIdsQueue_; + THashMap Tasks_; + TQueue ExecIds_; + ui64 CurrentTaskId_ = 0; + TThread Thread_; }; @@ -428,7 +431,7 @@ struct TYdbPqWriterTestHelper { std::shared_ptr Setup; std::shared_ptr Policy; std::unique_ptr EventLoop; - TIntrusivePtr CompressExecutor; + std::shared_ptr CompressExecutor; TAutoEvent MessagesWrittenToBuffer; ui64 SeqNo = 1; @@ -445,8 +448,9 @@ struct TYdbPqWriterTestHelper { : Setup(setup ? setup : std::make_shared(name)) , Policy(std::make_shared()) { - if (executorQueue) - CompressExecutor = MakeIntrusive(executorQueue); + if (executorQueue) { + CompressExecutor = std::make_shared(executorQueue); + } EventLoop = std::make_unique(Setup, Policy, CompressExecutor, preferredCluster, sourceId, autoSeqNo); } diff --git a/src/client/query/CMakeLists.txt b/src/client/query/CMakeLists.txt index fe59888d193..6677d402d4d 100644 --- a/src/client/query/CMakeLists.txt +++ b/src/client/query/CMakeLists.txt @@ -4,14 +4,14 @@ _ydb_sdk_add_library(client-ydb_query) target_link_libraries(client-ydb_query PUBLIC yutil - impl-ydb_internal-make_request + impl-internal-make_request impl-session - impl-ydb_internal-retry + impl-internal-retry client-ydb_common_client client-ydb_driver client-ydb_query-impl client-ydb_result - client-ydb_types-operation + client-types-operation api-protos api-grpc ) diff --git a/src/client/result/CMakeLists.txt b/src/client/result/CMakeLists.txt index 9779b1bc489..282aa44796b 100644 --- a/src/client/result/CMakeLists.txt +++ b/src/client/result/CMakeLists.txt @@ -3,7 +3,7 @@ _ydb_sdk_add_library(client-ydb_result) target_link_libraries(client-ydb_result PUBLIC yutil api-protos - client-ydb_types-fatal_error_handlers + client-types-fatal_error_handlers client-ydb_value client-ydb_proto ) diff --git a/src/client/scheme/CMakeLists.txt b/src/client/scheme/CMakeLists.txt index 184e96bfdd6..cb84b44bb23 100644 --- a/src/client/scheme/CMakeLists.txt +++ b/src/client/scheme/CMakeLists.txt @@ -3,7 +3,7 @@ _ydb_sdk_add_library(client-ydb_scheme) target_link_libraries(client-ydb_scheme PUBLIC yutil enum_serialization_runtime - impl-ydb_internal-make_request + impl-internal-make_request client-ydb_common_client-impl client-ydb_driver ) diff --git a/src/client/ss_tasks/CMakeLists.txt b/src/client/ss_tasks/CMakeLists.txt index 61092e60f84..b4c8a9abaf5 100644 --- a/src/client/ss_tasks/CMakeLists.txt +++ b/src/client/ss_tasks/CMakeLists.txt @@ -7,7 +7,7 @@ target_link_libraries(client-ss_tasks PUBLIC client-ydb_common_client-impl client-ydb_driver client-ydb_proto - client-ydb_types-operation + client-types-operation ) target_sources(client-ss_tasks PRIVATE diff --git a/src/client/table/CMakeLists.txt b/src/client/table/CMakeLists.txt index 3c390383698..18e607cf6a0 100644 --- a/src/client/table/CMakeLists.txt +++ b/src/client/table/CMakeLists.txt @@ -7,9 +7,9 @@ target_link_libraries(client-ydb_table PUBLIC yutil enum_serialization_runtime api-protos - impl-ydb_internal-make_request + impl-internal-make_request impl-session - impl-ydb_internal-retry + impl-internal-retry client-ydb_driver client-ydb_params client-ydb_proto @@ -17,7 +17,7 @@ target_link_libraries(client-ydb_table PUBLIC client-ydb_scheme client-ydb_table-impl client-ydb_table-query_stats - client-ydb_types-operation + client-types-operation client-ydb_value string_utils-misc ) diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index 1df7dc34ee5..d68a1b633a5 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -1019,7 +1019,7 @@ void TTableClient::TImpl::SetStatCollector(const NSdkStats::TStatCollector::TCli SessionRemovedDueBalancing.Set(collector.SessionRemovedDueBalancing); } -TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings, bool canMove) { +TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings) { Ydb::Table::BulkUpsertRequest* request = nullptr; std::unique_ptr holder; @@ -1031,11 +1031,16 @@ TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, } request->set_table(TStringType{table}); - if (canMove) { + + if (rows.GetType().Impl_.use_count() == 1) { request->mutable_rows()->mutable_type()->Swap(&rows.GetType().GetProto()); + } else { + *request->mutable_rows()->mutable_type() = rows.GetType().GetProto(); + } + + if (rows.Impl_.use_count() == 1) { request->mutable_rows()->mutable_value()->Swap(&rows.GetProto()); } else { - *request->mutable_rows()->mutable_type() = TProtoAccessor::GetProto(rows.GetType()); *request->mutable_rows()->mutable_value() = rows.GetProto(); } diff --git a/src/client/table/impl/table_client.h b/src/client/table/impl/table_client.h index e26ef4d4784..18dd8720e97 100644 --- a/src/client/table/impl/table_client.h +++ b/src/client/table/impl/table_client.h @@ -139,7 +139,7 @@ class TTableClient::TImpl: public TClientImplCommon, public void SetStatCollector(const NSdkStats::TStatCollector::TClientStatCollector& collector); - TAsyncBulkUpsertResult BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings, bool canMove); + TAsyncBulkUpsertResult BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings); TAsyncBulkUpsertResult BulkUpsert(const std::string& table, EDataFormat format, const std::string& data, const std::string& schema, const TBulkUpsertSettings& settings); diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index c53c59e3579..6e3332b2d28 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -1496,7 +1496,7 @@ NThreading::TFuture TTableClient::Stop() { TAsyncBulkUpsertResult TTableClient::BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings) { - return Impl_->BulkUpsert(table, std::move(rows), settings, rows.Impl_.use_count() == 1); + return Impl_->BulkUpsert(table, std::move(rows), settings); } TAsyncBulkUpsertResult TTableClient::BulkUpsert(const std::string& table, EDataFormat format, diff --git a/src/client/topic/common/CMakeLists.txt b/src/client/topic/common/CMakeLists.txt index 7605cf26c02..609a8854076 100644 --- a/src/client/topic/common/CMakeLists.txt +++ b/src/client/topic/common/CMakeLists.txt @@ -2,7 +2,7 @@ _ydb_sdk_add_library(client-ydb_topic-common) target_link_libraries(client-ydb_topic-common PUBLIC client-ydb_common_client-impl - client-ydb_types + client-types monlib-dynamic_counters retry ) diff --git a/src/client/topic/common/executor_impl.cpp b/src/client/topic/common/executor_impl.cpp index 39c56acb329..40abae64d6d 100644 --- a/src/client/topic/common/executor_impl.cpp +++ b/src/client/topic/common/executor_impl.cpp @@ -1,25 +1,13 @@ #include "executor_impl.h" +#include + namespace NYdb::inline V3::NTopic { void IAsyncExecutor::Post(TFunction&& f) { PostImpl(std::move(f)); } -IAsyncExecutor::TPtr CreateDefaultExecutor() { - return CreateThreadPoolExecutor(1); -} - -void TThreadPoolExecutor::PostImpl(std::vector&& fs) { - for (auto& f : fs) { - ThreadPool->SafeAddFunc(std::move(f)); - } -} - -void TThreadPoolExecutor::PostImpl(TFunction&& f) { - ThreadPool->SafeAddFunc(std::move(f)); -} - TSerialExecutor::TSerialExecutor(IAsyncExecutor::TPtr executor) : Executor(executor) { @@ -65,34 +53,9 @@ void TSerialExecutor::PostNext() { Busy = true; } -IExecutor::TPtr CreateThreadPoolExecutor(size_t threads) { - return MakeIntrusive(threads); -} - -IExecutor::TPtr CreateGenericExecutor() { - return CreateThreadPoolExecutor(1); -} - -IExecutor::TPtr CreateThreadPoolExecutorAdapter(std::shared_ptr threadPool) { - return MakeIntrusive(std::move(threadPool)); -} - -TThreadPoolExecutor::TThreadPoolExecutor(std::shared_ptr threadPool) - : ThreadPool(std::move(threadPool)) -{ - IsFakeThreadPool = dynamic_cast(ThreadPool.get()) != nullptr; -} - -TThreadPoolExecutor::TThreadPoolExecutor(size_t threadsCount) - : TThreadPoolExecutor(CreateThreadPool(threadsCount)) -{ - Y_ABORT_UNLESS(threadsCount > 0); - ThreadsCount = threadsCount; -} - IExecutor::TPtr CreateSyncExecutor() { - return MakeIntrusive(); + return std::make_shared(); } } // namespace NYdb::NTopic \ No newline at end of file diff --git a/src/client/topic/common/executor_impl.h b/src/client/topic/common/executor_impl.h index cd06370e667..53407d329c8 100644 --- a/src/client/topic/common/executor_impl.h +++ b/src/client/topic/common/executor_impl.h @@ -22,37 +22,6 @@ class IAsyncExecutor : public IExecutor { void Post(TFunction&& f) final; }; -IExecutor::TPtr CreateDefaultExecutor(); - - -class TThreadPoolExecutor : public IAsyncExecutor { -private: - std::shared_ptr ThreadPool; - -public: - TThreadPoolExecutor(std::shared_ptr threadPool); - TThreadPoolExecutor(size_t threadsCount); - ~TThreadPoolExecutor() = default; - - bool IsAsync() const override { - return !IsFakeThreadPool; - } - - void DoStart() override { - if (ThreadsCount) { - ThreadPool->Start(ThreadsCount); - } - } - -private: - void PostImpl(std::vector&& fs) override; - void PostImpl(TFunction&& f) override; - -private: - bool IsFakeThreadPool = false; - size_t ThreadsCount = 0; -}; - class TSerialExecutor : public IAsyncExecutor, public std::enable_shared_from_this { private: IAsyncExecutor::TPtr Executor; //!< Wrapped executor that is actually doing the job @@ -80,8 +49,8 @@ class TSyncExecutor : public IExecutor { } void DoStart() override { } + void Stop() override { + } }; -IExecutor::TPtr CreateGenericExecutor(); - } // namespace NYdb::NTopic diff --git a/src/client/topic/impl/CMakeLists.txt b/src/client/topic/impl/CMakeLists.txt index 6d1e4e1d203..6f3bc99989b 100644 --- a/src/client/topic/impl/CMakeLists.txt +++ b/src/client/topic/impl/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries(client-ydb_topic-impl PUBLIC persqueue-obfuscate api-grpc-draft api-grpc - impl-ydb_internal-make_request + impl-internal-make_request client-ydb_common_client-impl client-ydb_driver client-ydb_topic-common diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index 42e87e4c2c6..a05ff699300 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -85,11 +85,6 @@ using IARetryPolicy = std::conditional_t; -template -using IAExecutor = std::conditional_t; - template using TAReadSessionSettings = std::conditional_t callback); void DeferReadFromProcessor(const typename IProcessor::TPtr& processor, TServerMessage* dst, typename IProcessor::TReadCallback callback); - void DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& task); + void DeferStartExecutorTask(const typename IExecutor::TPtr& executor, typename IExecutor::TFunction&& task); void DeferAbortSession(TCallbackContextPtr cbContext, TASessionClosedEvent&& closeEvent); void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYdb::NIssue::TIssues&& issues); void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, const std::string& message); @@ -201,7 +196,7 @@ class TDeferredActions { } DirectReadActions; // Executor tasks. - std::vector::TPtr, typename IAExecutor::TFunction>> ExecutorsTasks; + std::vector> ExecutorsTasks; // Abort session. std::optional> SessionClosedEvent; @@ -234,7 +229,7 @@ class TDataDecompressionInfo : public std::enable_shared_from_this::TPtr& executor, + i64 StartDecompressionTasks(const typename IExecutor::TPtr& executor, i64 availableMemory, TDeferredActions& deferred); void PlanDecompressionTasks(double averageCompressionRatio, @@ -843,12 +838,12 @@ template class TReadSessionEventsQueue: public TBaseSessionEventsQueue, typename TAReadSessionEvent::TEvent, TASessionClosedEvent, - IAExecutor, + IExecutor, TReadSessionEventInfo> { using TParent = TBaseSessionEventsQueue, typename TAReadSessionEvent::TEvent, TASessionClosedEvent, - IAExecutor, + IExecutor, TReadSessionEventInfo>; public: @@ -1065,7 +1060,7 @@ class TReadSessionEventsQueue: public TBaseSessionEventsQueue::TPtr& executor, typename IAExecutor::TFunction&& f) override { + void Post(const typename IExecutor::TPtr& executor, typename IExecutor::TFunction&& f) override { Deferred.DeferStartExecutorTask(executor, std::move(f)); } diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 83ccb111be3..7a222ba4acb 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -2785,7 +2785,7 @@ std::exception_ptr TDataDecompressionInfo::GetDecompressio template i64 TDataDecompressionInfo::StartDecompressionTasks( - const typename IAExecutor::TPtr& executor, i64 availableMemory, + const typename IExecutor::TPtr& executor, i64 availableMemory, TDeferredActions& deferred) { auto session = CbContext->LockShared(); @@ -3172,7 +3172,7 @@ void TDeferredActions::DeferReadFromProcessor(const typena } template -void TDeferredActions::DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& task) { +void TDeferredActions::DeferStartExecutorTask(const typename IExecutor::TPtr& executor, typename IExecutor::TFunction&& task) { ExecutorsTasks.emplace_back(executor, std::move(task)); } diff --git a/src/client/topic/impl/write_session.cpp b/src/client/topic/impl/write_session.cpp index 5f2f54605b7..968ac4a6a0b 100644 --- a/src/client/topic/impl/write_session.cpp +++ b/src/client/topic/impl/write_session.cpp @@ -147,9 +147,9 @@ std::optional TSimpleBlockingWriteSession::WaitForToken(cons if (auto* readyEvent = std::get_if(&event)) { Y_ABORT_UNLESS(!token.has_value()); token = std::move(readyEvent->ContinuationToken); - } else if (auto* ackEvent = std::get_if(&event)) { + } else if (std::get_if(&event)) { // discard - } else if (auto* closeSessionEvent = std::get_if(&event)) { + } else if (std::get_if(&event)) { Closed.store(true); return std::nullopt; } diff --git a/src/client/topic/ut/basic_usage_ut.cpp b/src/client/topic/ut/basic_usage_ut.cpp index 99d9af7c42a..38fe6a2d51e 100644 --- a/src/client/topic/ut/basic_usage_ut.cpp +++ b/src/client/topic/ut/basic_usage_ut.cpp @@ -28,7 +28,7 @@ static const bool EnableDirectRead = !std::string{std::getenv("PQ_EXPERIMENTAL_D namespace NYdb::inline V3::NTopic::NTests { -void WriteAndReadToEndWithRestarts(TReadSessionSettings readSettings, TWriteSessionSettings writeSettings, const std::string& message, std::uint32_t count, TTopicSdkTestSetup& setup, TIntrusivePtr decompressor) { +void WriteAndReadToEndWithRestarts(TReadSessionSettings readSettings, TWriteSessionSettings writeSettings, const std::string& message, std::uint32_t count, TTopicSdkTestSetup& setup, std::shared_ptr decompressor) { auto client = setup.MakeClient(); auto session = client.CreateSimpleBlockingWriteSession(writeSettings); @@ -113,7 +113,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { return; } TTopicSdkTestSetup setup(TEST_CASE_NAME); - auto compressor = new TSyncExecutor(); + auto compressor = std::make_shared(); auto decompressor = CreateThreadPoolManagedExecutor(1); TReadSessionSettings readSettings; @@ -143,7 +143,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { Y_UNIT_TEST(ReadWithRestarts) { TTopicSdkTestSetup setup(TEST_CASE_NAME); - auto compressor = new TSyncExecutor(); + auto compressor = std::make_shared(); auto decompressor = CreateThreadPoolManagedExecutor(1); TReadSessionSettings readSettings; @@ -176,7 +176,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { writeSettings.Path(setup.GetTopicPath()).MessageGroupId(TEST_MESSAGE_GROUP_ID); writeSettings.Path(setup.GetTopicPath()).ProducerId(TEST_MESSAGE_GROUP_ID); writeSettings.Codec(ECodec::RAW); - IExecutor::TPtr executor = new TSyncExecutor(); + IExecutor::TPtr executor = std::make_shared(); writeSettings.CompressionExecutor(executor); std::uint64_t count = 100u; diff --git a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp index 1a64d8a0d78..30273f03ff8 100644 --- a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp +++ b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp @@ -151,8 +151,8 @@ std::string TTopicSdkTestSetup::GetDatabase() const { return Database_; } -std::string TTopicSdkTestSetup::GetFullTopicPath() const { - return GetDatabase() + "/" + GetTopicPath(); +std::string TTopicSdkTestSetup::GetFullTopicPath(const std::string& name) const { + return GetDatabase() + "/" + GetTopicPath(name); } std::vector TTopicSdkTestSetup::GetNodeIds() { diff --git a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h index 9e2c0a7c900..65c7f036ae7 100644 --- a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h +++ b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h @@ -51,7 +51,7 @@ class TTopicSdkTestSetup : public ITopicTestSetup { std::string GetEndpoint() const override; std::string GetDatabase() const override; - std::string GetFullTopicPath() const; + std::string GetFullTopicPath(const std::string& name = TEST_TOPIC) const; std::vector GetNodeIds() override; std::uint16_t GetPort() const override; diff --git a/src/client/types/CMakeLists.txt b/src/client/types/CMakeLists.txt index a7e2cea89cf..1d72d1b961b 100644 --- a/src/client/types/CMakeLists.txt +++ b/src/client/types/CMakeLists.txt @@ -1,34 +1,35 @@ add_subdirectory(credentials) add_subdirectory(exceptions) +add_subdirectory(executor) add_subdirectory(fatal_error_handlers) add_subdirectory(operation) add_subdirectory(status) -_ydb_sdk_add_library(client-ydb_types) +_ydb_sdk_add_library(client-types) -target_sources(client-ydb_types PRIVATE +target_sources(client-types PRIVATE ydb.cpp ) -target_link_libraries(client-ydb_types PUBLIC +target_link_libraries(client-types PUBLIC yutil protobuf::libprotobuf grpc-client yql-public-issue enum_serialization_runtime - impl-ydb_internal-common + impl-internal-common ) -generate_enum_serilization(client-ydb_types +generate_enum_serilization(client-types ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/types/s3_settings.h INCLUDE_HEADERS include/ydb-cpp-sdk/client/types/s3_settings.h ) -generate_enum_serilization(client-ydb_types +generate_enum_serilization(client-types ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/client/types/status_codes.h INCLUDE_HEADERS include/ydb-cpp-sdk/client/types/status_codes.h ) -_ydb_sdk_make_client_component(Types client-ydb_types) +_ydb_sdk_make_client_component(Types client-types) diff --git a/src/client/types/credentials/CMakeLists.txt b/src/client/types/credentials/CMakeLists.txt index 8c53cb0f8bf..de608b39882 100644 --- a/src/client/types/credentials/CMakeLists.txt +++ b/src/client/types/credentials/CMakeLists.txt @@ -1,19 +1,19 @@ add_subdirectory(login) add_subdirectory(oauth2_token_exchange) -_ydb_sdk_add_library(client-ydb_types-credentials) +_ydb_sdk_add_library(client-types-credentials) -target_link_libraries(client-ydb_types-credentials PUBLIC +target_link_libraries(client-types-credentials PUBLIC yutil api-grpc - client-ydb_types-status + client-types-status yql-public-issue - client-ydb_types-credentials-login - client-ydb_types-credentials-oauth2 + client-types-credentials-login + client-types-credentials-oauth2 ) -target_sources(client-ydb_types-credentials PRIVATE +target_sources(client-types-credentials PRIVATE credentials.cpp ) -_ydb_sdk_make_client_component(Credentials client-ydb_types-credentials) +_ydb_sdk_make_client_component(Credentials client-types-credentials) diff --git a/src/client/types/credentials/login/CMakeLists.txt b/src/client/types/credentials/login/CMakeLists.txt index 218685ff0da..ef490821dad 100644 --- a/src/client/types/credentials/login/CMakeLists.txt +++ b/src/client/types/credentials/login/CMakeLists.txt @@ -1,18 +1,18 @@ -_ydb_sdk_add_library(client-ydb_types-credentials-login) +_ydb_sdk_add_library(client-types-credentials-login) -target_link_libraries(client-ydb_types-credentials-login +target_link_libraries(client-types-credentials-login PUBLIC yutil PRIVATE api-grpc - client-ydb_types-status - impl-ydb_internal-grpc_connections + client-types-status + impl-internal-grpc_connections yql-public-issue jwt-cpp::jwt-cpp ) -target_sources(client-ydb_types-credentials-login PRIVATE +target_sources(client-types-credentials-login PRIVATE login.cpp ) -_ydb_sdk_install_targets(TARGETS client-ydb_types-credentials-login) +_ydb_sdk_install_targets(TARGETS client-types-credentials-login) diff --git a/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt b/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt index 5865f03a1be..8828ea14c39 100644 --- a/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt +++ b/src/client/types/credentials/oauth2_token_exchange/CMakeLists.txt @@ -1,6 +1,6 @@ -_ydb_sdk_add_library(client-ydb_types-credentials-oauth2) +_ydb_sdk_add_library(client-types-credentials-oauth2) -target_link_libraries(client-ydb_types-credentials-oauth2 +target_link_libraries(client-types-credentials-oauth2 PUBLIC yutil jwt-cpp::jwt-cpp @@ -11,20 +11,20 @@ target_link_libraries(client-ydb_types-credentials-oauth2 retry string_utils-base64 uri - client-ydb_types-credentials - client-ydb_types + client-types-credentials + client-types ) -target_compile_definitions(client-ydb_types-credentials-oauth2 +target_compile_definitions(client-types-credentials-oauth2 PUBLIC YDB_SDK_USE_NEW_JWT ) -target_sources(client-ydb_types-credentials-oauth2 +target_sources(client-types-credentials-oauth2 PRIVATE credentials.cpp from_file.cpp jwt_token_source.cpp ) -_ydb_sdk_install_targets(TARGETS client-ydb_types-credentials-oauth2) +_ydb_sdk_install_targets(TARGETS client-types-credentials-oauth2) diff --git a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp index 31d9225709d..57cbc427600 100644 --- a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp +++ b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp @@ -98,7 +98,7 @@ ERetryErrorClass SyncRetryPolicyClass(const std::exception* ex, bool retryAllErr return ERetryErrorClass::ShortRetry; } } - if (const TSystemError* err = dynamic_cast(ex)) { + if (dynamic_cast(ex)) { return ERetryErrorClass::ShortRetry; } diff --git a/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp b/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp index 09bca9a42d9..5e8bfe5c26d 100644 --- a/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp +++ b/src/client/types/credentials/oauth2_token_exchange/jwt_token_source.cpp @@ -8,7 +8,7 @@ namespace NYdb::inline V3 { -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS using TJwtCppStorage = std::vector; #else using TJwtCppStorage = std::set; diff --git a/src/client/types/exceptions/CMakeLists.txt b/src/client/types/exceptions/CMakeLists.txt index 365cc219592..1ecdc12611c 100644 --- a/src/client/types/exceptions/CMakeLists.txt +++ b/src/client/types/exceptions/CMakeLists.txt @@ -1,11 +1,11 @@ -_ydb_sdk_add_library(client-ydb_types-exceptions) +_ydb_sdk_add_library(client-types-exceptions) -target_link_libraries(client-ydb_types-exceptions PUBLIC +target_link_libraries(client-types-exceptions PUBLIC yutil ) -target_sources(client-ydb_types-exceptions PRIVATE +target_sources(client-types-exceptions PRIVATE exceptions.cpp ) -_ydb_sdk_install_targets(TARGETS client-ydb_types-exceptions) +_ydb_sdk_install_targets(TARGETS client-types-exceptions) diff --git a/src/client/types/executor/CMakeLists.txt b/src/client/types/executor/CMakeLists.txt new file mode 100644 index 00000000000..38972310f0b --- /dev/null +++ b/src/client/types/executor/CMakeLists.txt @@ -0,0 +1,14 @@ +_ydb_sdk_add_library(client-types-executor) + +target_sources(client-types-executor PRIVATE + executor.cpp +) + +target_link_libraries(client-types-executor PUBLIC + yutil + impl-internal-thread_pool + impl-executor + client-types-exceptions +) + +_ydb_sdk_install_targets(TARGETS client-types-executor) diff --git a/src/client/types/executor/executor.cpp b/src/client/types/executor/executor.cpp new file mode 100644 index 00000000000..a5e0f8f85cc --- /dev/null +++ b/src/client/types/executor/executor.cpp @@ -0,0 +1,25 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#include +#undef INCLUDE_YDB_INTERNAL_H + + +namespace NYdb::inline V3 { + +std::shared_ptr CreateThreadPoolExecutor(std::size_t threadCount, std::size_t maxQueueSize) { + return std::make_shared(CreateThreadPool(threadCount), threadCount, maxQueueSize); +} + +#ifndef YDB_SDK_OSS +std::shared_ptr CreateThreadPoolExecutorAdapter(std::shared_ptr threadPool, std::size_t threadCount, std::size_t maxQueueSize) { + return std::make_shared(threadPool, threadCount, maxQueueSize); +} + +std::shared_ptr CreateExternalThreadPoolExecutorAdapter(std::shared_ptr threadPool) { + return std::make_shared(threadPool); +} +#endif + +} diff --git a/src/client/types/fatal_error_handlers/CMakeLists.txt b/src/client/types/fatal_error_handlers/CMakeLists.txt index bf19c1526d1..ff5742e77d3 100644 --- a/src/client/types/fatal_error_handlers/CMakeLists.txt +++ b/src/client/types/fatal_error_handlers/CMakeLists.txt @@ -1,12 +1,12 @@ -_ydb_sdk_add_library(client-ydb_types-fatal_error_handlers) +_ydb_sdk_add_library(client-types-fatal_error_handlers) -target_link_libraries(client-ydb_types-fatal_error_handlers PUBLIC +target_link_libraries(client-types-fatal_error_handlers PUBLIC yutil - client-ydb_types-exceptions + client-types-exceptions ) -target_sources(client-ydb_types-fatal_error_handlers PRIVATE +target_sources(client-types-fatal_error_handlers PRIVATE handlers.cpp ) -_ydb_sdk_install_targets(TARGETS client-ydb_types-fatal_error_handlers) +_ydb_sdk_install_targets(TARGETS client-types-fatal_error_handlers) diff --git a/src/client/types/operation/CMakeLists.txt b/src/client/types/operation/CMakeLists.txt index cb20314eb8b..6f1f0e9a388 100644 --- a/src/client/types/operation/CMakeLists.txt +++ b/src/client/types/operation/CMakeLists.txt @@ -1,16 +1,16 @@ -_ydb_sdk_add_library(client-ydb_types-operation) +_ydb_sdk_add_library(client-types-operation) -target_link_libraries(client-ydb_types-operation PUBLIC +target_link_libraries(client-types-operation PUBLIC yutil protobuf::libprotobuf threading-future library-operation_id - client-ydb_types + client-types ) -target_sources(client-ydb_types-operation PRIVATE +target_sources(client-types-operation PRIVATE operation.cpp out.cpp ) -_ydb_sdk_install_targets(TARGETS client-ydb_types-operation) +_ydb_sdk_install_targets(TARGETS client-types-operation) diff --git a/src/client/types/status/CMakeLists.txt b/src/client/types/status/CMakeLists.txt index 280206f8ce2..81b10c95145 100644 --- a/src/client/types/status/CMakeLists.txt +++ b/src/client/types/status/CMakeLists.txt @@ -1,16 +1,16 @@ -_ydb_sdk_add_library(client-ydb_types-status) +_ydb_sdk_add_library(client-types-status) -target_link_libraries(client-ydb_types-status PUBLIC +target_link_libraries(client-types-status PUBLIC yutil threading-future - impl-ydb_internal-plain_status - client-ydb_types - client-ydb_types-fatal_error_handlers + impl-internal-plain_status + client-types + client-types-fatal_error_handlers yql-public-issue ) -target_sources(client-ydb_types-status PRIVATE +target_sources(client-types-status PRIVATE status.cpp ) -_ydb_sdk_install_targets(TARGETS client-ydb_types-status) +_ydb_sdk_install_targets(TARGETS client-types-status) diff --git a/src/client/types/ydb.cpp b/src/client/types/ydb.cpp index 0951d0cda4a..332c03b6261 100644 --- a/src/client/types/ydb.cpp +++ b/src/client/types/ydb.cpp @@ -10,28 +10,30 @@ namespace NYdb::inline V3 { TBalancingPolicy::TBalancingPolicy(EBalancingPolicy policy, const std::string& params) { switch (policy) { case EBalancingPolicy::UsePreferableLocation: - Impl_ = TImpl::UsePreferableLocation(params); + Impl_ = std::make_unique(TImpl::UsePreferableLocation(params)); break; case EBalancingPolicy::UseAllNodes: - Impl_ = TImpl::UseAllNodes(); + Impl_ = std::make_unique(TImpl::UseAllNodes()); break; } } TBalancingPolicy TBalancingPolicy::UsePreferableLocation(const std::string& location) { - return TBalancingPolicy(TImpl::UsePreferableLocation(location)); + return TBalancingPolicy(std::make_unique(TImpl::UsePreferableLocation(location))); } TBalancingPolicy TBalancingPolicy::UseAllNodes() { - return TBalancingPolicy(TImpl::UseAllNodes()); + return TBalancingPolicy(std::make_unique(TImpl::UseAllNodes())); } TBalancingPolicy TBalancingPolicy::UsePreferablePileState(EPileState pileState) { - return TBalancingPolicy(TImpl::UsePreferablePileState(pileState)); + return TBalancingPolicy(std::make_unique(TImpl::UsePreferablePileState(pileState))); } TBalancingPolicy::TBalancingPolicy(std::unique_ptr&& impl) : Impl_(std::move(impl)) {} +TBalancingPolicy::~TBalancingPolicy() = default; + } diff --git a/src/client/value/CMakeLists.txt b/src/client/value/CMakeLists.txt index 38b4b603258..ca1998c9521 100644 --- a/src/client/value/CMakeLists.txt +++ b/src/client/value/CMakeLists.txt @@ -5,8 +5,8 @@ target_link_libraries(client-ydb_value PUBLIC enum_serialization_runtime containers-stack_vector api-protos - impl-ydb_internal-value_helpers - client-ydb_types-fatal_error_handlers + impl-internal-value_helpers + client-types-fatal_error_handlers yql-public-decimal library-uuid proto_output diff --git a/src/library/jwt/CMakeLists.txt b/src/library/jwt/CMakeLists.txt index 99a1153efc6..d43241716eb 100644 --- a/src/library/jwt/CMakeLists.txt +++ b/src/library/jwt/CMakeLists.txt @@ -4,7 +4,7 @@ target_link_libraries(library-jwt PUBLIC yutil jwt-cpp::jwt-cpp json - impl-ydb_internal-common + impl-internal-common ) target_sources(library-jwt PRIVATE diff --git a/src/library/operation_id/operation_id.cpp b/src/library/operation_id/operation_id.cpp index ea7e2e1bf3f..9e43e1c01d7 100644 --- a/src/library/operation_id/operation_id.cpp +++ b/src/library/operation_id/operation_id.cpp @@ -153,7 +153,7 @@ class TOperationId::TImpl { auto data = Proto.add_data(); data->set_key(it.first); data->set_value(it.second); -#ifdef YDB_SDK_USE_STD_STRING +#ifdef YDB_SDK_OSS Index[it.first].push_back(&data->value()); #else Index[it.first].push_back(&data->value().ConstRef()); @@ -206,7 +206,7 @@ class TOperationId::TImpl { private: void BuildIndex() { for (const auto& data : Proto.data()) { -#ifdef YDB_SDK_USE_STD_STRING +#ifdef YDB_SDK_OSS Index[data.key()].push_back(&data.value()); #else Index[data.key()].push_back(&data.value().ConstRef()); diff --git a/tests/integration/topic/basic_usage.cpp b/tests/integration/topic/basic_usage.cpp index 7152e511a1e..585c59bcb8b 100644 --- a/tests/integration/topic/basic_usage.cpp +++ b/tests/integration/topic/basic_usage.cpp @@ -179,7 +179,7 @@ TEST_F(BasicUsage, TEST_NAME(MaxByteSizeEqualZero)) { TEST_F(BasicUsage, TEST_NAME(WriteAndReadSomeMessagesWithSyncCompression)) { auto driver = MakeDriver(); - IExecutor::TPtr executor = new TSyncExecutor(); + IExecutor::TPtr executor = std::make_shared(); auto writeSettings = NPersQueue::TWriteSessionSettings() .Path(GetTopicPath()) .MessageGroupId(TEST_MESSAGE_GROUP_ID) @@ -507,7 +507,7 @@ TEST_F(BasicUsage, TEST_NAME(SessionNotDestroyedWhileUserEventHandlingInFlight)) RunTasks(stepByStepExecutor, {1}); futureRead.GetValueSync(); ASSERT_TRUE(futureRead.HasValue()); - std::cerr << ">>>TEST: future read has value " << std::endl; + std::cerr << ">>> TEST: future read has value " << std::endl; f.get(); @@ -520,7 +520,7 @@ TEST_F(BasicUsage, TEST_NAME(ReadSessionCorrectClose)) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(GetTopicPath()).MessageGroupId(TEST_MESSAGE_GROUP_ID); writeSettings.Codec(NPersQueue::ECodec::RAW); - IExecutor::TPtr executor = new TSyncExecutor(); + IExecutor::TPtr executor = std::make_shared(); writeSettings.CompressionExecutor(executor); NPersQueue::TPersQueueClient client(driver); diff --git a/tests/integration/topic/direct_read.cpp b/tests/integration/topic/direct_read.cpp index 2e3d1debb76..646487c36c4 100644 --- a/tests/integration/topic/direct_read.cpp +++ b/tests/integration/topic/direct_read.cpp @@ -8,6 +8,10 @@ #include #include +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + #include #include @@ -801,7 +805,7 @@ IExecutor::TPtr TDirectReadSessionImplTestSetup::GetDefaultExecutor() { if (!DefaultExecutor) { ThreadPool = std::make_shared(); ThreadPool->Start(1); - DefaultExecutor = CreateThreadPoolExecutorAdapter(ThreadPool); + DefaultExecutor = std::make_shared(ThreadPool); } return DefaultExecutor; } @@ -956,7 +960,7 @@ TEST_F(DirectReadWithClient, ManyMessages) { auto killer = std::thread([&]() { while (work.load()) { std::this_thread::sleep_for(std::chrono::seconds(5)); - // setup.GetServer().KillTopicPqrbTablet(setup.GetTopicPath()); + // setup.GetServer().KillTopicPqrbTablet(setup.GetTopicPath()); // TODO(qyryq) Uncomment! } }); diff --git a/tests/integration/topic/utils/managed_executor.cpp b/tests/integration/topic/utils/managed_executor.cpp index ece779d8dbc..3e72dbc5a12 100644 --- a/tests/integration/topic/utils/managed_executor.cpp +++ b/tests/integration/topic/utils/managed_executor.cpp @@ -21,6 +21,11 @@ void TManagedExecutor::Post(TFunction &&f) ++Planned; } +void TManagedExecutor::Stop() +{ + Executor->Stop(); +} + void TManagedExecutor::DoStart() { Executor->Start(); @@ -90,14 +95,14 @@ void TManagedExecutor::RunAllTasks() } } -TIntrusivePtr CreateThreadPoolManagedExecutor(size_t threads) +std::shared_ptr CreateThreadPoolManagedExecutor(size_t threads) { - return MakeIntrusive(NYdb::NTopic::CreateThreadPoolExecutor(threads)); + return std::make_shared(NYdb::CreateThreadPoolExecutor(threads)); } -TIntrusivePtr CreateSyncManagedExecutor() +std::shared_ptr CreateSyncManagedExecutor() { - return MakeIntrusive(NYdb::NTopic::CreateSyncExecutor()); + return std::make_shared(NYdb::NTopic::CreateSyncExecutor()); } } diff --git a/tests/integration/topic/utils/managed_executor.h b/tests/integration/topic/utils/managed_executor.h index 003a23a1469..d379f469a3d 100644 --- a/tests/integration/topic/utils/managed_executor.h +++ b/tests/integration/topic/utils/managed_executor.h @@ -17,6 +17,8 @@ class TManagedExecutor : public IExecutor { bool IsAsync() const override; void Post(TFunction&& f) override; + void Stop() override; + void StartFuncs(const std::vector& indicies); size_t GetFuncsCount() const; @@ -41,7 +43,7 @@ class TManagedExecutor : public IExecutor { std::atomic Executed = 0; }; -TIntrusivePtr CreateThreadPoolManagedExecutor(size_t threads); -TIntrusivePtr CreateSyncManagedExecutor(); +std::shared_ptr CreateThreadPoolManagedExecutor(size_t threads); +std::shared_ptr CreateSyncManagedExecutor(); } diff --git a/tests/slo_workloads/key_value/create.cpp b/tests/slo_workloads/key_value/create.cpp new file mode 100644 index 00000000000..a79db76a123 --- /dev/null +++ b/tests/slo_workloads/key_value/create.cpp @@ -0,0 +1,39 @@ +#include "key_value.h" + +#include + +using namespace NYdb; +using namespace NYdb::NTable; + + +namespace { + void CreateTable(TTableClient& client, const std::string& prefix) { + RetryBackoff(client, 5, [prefix](TSession session) { + auto desc = TTableBuilder() + .AddNullableColumn("object_id_key", EPrimitiveType::Uint32) + .AddNullableColumn("object_id", EPrimitiveType::Uint32) + .AddNullableColumn("timestamp", EPrimitiveType::Uint64) + .AddNullableColumn("payload", EPrimitiveType::Utf8) + .SetPrimaryKeyColumns({ "object_id_key", "object_id" }) + .Build(); + + auto tableSettings = TCreateTableSettings() + .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(PartitionsCount)) + .CancelAfter(DefaultReactionTime) + .ClientTimeout(DefaultReactionTime + TDuration::Seconds(5)); + + return session.CreateTable( + JoinPath(prefix, TableName) + , std::move(desc) + , std::move(tableSettings) + ).ExtractValueSync(); + }); + } +} //namespace + +int CreateTable(TDatabaseOptions& dbOptions) { + TTableClient client(dbOptions.Driver); + CreateTable(client, dbOptions.Prefix); + Cout << "Table created." << Endl; + return EXIT_SUCCESS; +} diff --git a/tests/slo_workloads/key_value/drop.cpp b/tests/slo_workloads/key_value/drop.cpp new file mode 100644 index 00000000000..ae749bc9033 --- /dev/null +++ b/tests/slo_workloads/key_value/drop.cpp @@ -0,0 +1,18 @@ +#include "key_value.h" + +using namespace NLastGetopt; +using namespace NYdb; +using namespace NYdb::NTable; + +static void DropTable(TTableClient& client, const std::string& path) { + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([path](TSession session) { + return session.DropTable(path).ExtractValueSync(); + })); +} + +int DropTable(TDatabaseOptions& dbOptions) { + TTableClient client(dbOptions.Driver); + DropTable(client, JoinPath(dbOptions.Prefix, TableName)); + Cout << "Table dropped." << Endl; + return EXIT_SUCCESS; +} diff --git a/tests/slo_workloads/key_value/generate.cpp b/tests/slo_workloads/key_value/generate.cpp new file mode 100644 index 00000000000..1486c9649aa --- /dev/null +++ b/tests/slo_workloads/key_value/generate.cpp @@ -0,0 +1,85 @@ +#include "key_value.h" + +#include + +using namespace NLastGetopt; +using namespace NYdb; +using namespace NYdb::NTable; + + +TGenerateInitialContentJob::TGenerateInitialContentJob(const TCreateOptions& createOpts, std::uint32_t maxId) + : TThreadJob(createOpts.CommonOptions) + , Executor(createOpts.CommonOptions, Stats, TExecutor::ModeBlocking) + , PackGenerator( + createOpts.CommonOptions + , createOpts.PackSize + , [](const TKeyValueRecordData& recordData) { return BuildValueFromRecord(recordData); } + , createOpts.Count + , maxId + ) + , Total(createOpts.Count) +{} + +void TGenerateInitialContentJob::ShowProgress(TStringBuilder& report) { + report << Endl << "======- GenerateInitialContentJob report -======" << Endl; + Executor.Report(report); + TDuration timePassed = TInstant::Now() - Stats.GetStartTime(); + std::uint64_t rps = (Total - PackGenerator.GetRemain()) * 1000000 / timePassed.MicroSeconds(); + report << "Generated " << Total - PackGenerator.GetRemain() << " new elements." << Endl + << "With pack_size=" << PackGenerator.GetPackSize() << " its " << rps << " rows/sec" << Endl + << "Generator compute time: " << PackGenerator.GetComputeTime() << Endl; + Stats.PrintStatistics(report); + report << "========================================" << Endl; +} + +void TGenerateInitialContentJob::DoJob() { + std::vector pack; + while (!ShouldStop.load() && PackGenerator.GetNextPack(pack)) { + auto upload = [pack{ std::move(pack) }, this](TSession session)->TAsyncStatus { + static const TString query = Sprintf(R"( +--!syntax_v1 +PRAGMA TablePathPrefix("%s"); + +DECLARE $items AS + List>; + +UPSERT INTO `%s` SELECT * FROM AS_TABLE($items); + +)", Prefix.c_str(), TableName.c_str()); + auto promise = NThreading::NewPromise(); + auto params = PackValuesToParamsAsList(pack); + + auto resultFuture = session.ExecuteDataQuery( + query + , TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx() + , std::move(params) + , TExecDataQuerySettings() + .KeepInQueryCache(true) + .OperationTimeout(MaxDelay + ReactionTimeDelay) + .ClientTimeout(MaxDelay + ReactionTimeDelay) + ); + + resultFuture.Subscribe([promise](TAsyncDataQueryResult queryFuture) mutable { + Y_ABORT_UNLESS(queryFuture.HasValue()); + TDataQueryResult queryResult = queryFuture.GetValue(); + promise.SetValue(std::move(queryResult)); + }); + + return promise.GetFuture(); + }; + + if (!Executor.Execute(upload)) { + break; + } + } +} + +void TGenerateInitialContentJob::OnFinish() { + Executor.Finish(); + Executor.Wait(); + Stats.Flush(); +} diff --git a/tests/slo_workloads/key_value/key_value.cpp b/tests/slo_workloads/key_value/key_value.cpp new file mode 100644 index 00000000000..9f48597019f --- /dev/null +++ b/tests/slo_workloads/key_value/key_value.cpp @@ -0,0 +1,108 @@ +#include "key_value.h" + +#include +#include +#include + +using namespace NYdb; + +const std::string TableName = "key_value"; + +NYdb::TValue BuildValueFromRecord(const TKeyValueRecordData& recordData) { + NYdb::TValueBuilder value; + value.BeginStruct(); + value.AddMember("object_id_key").Uint32(GetHash(recordData.ObjectId)); + value.AddMember("object_id").Uint32(recordData.ObjectId); + value.AddMember("timestamp").Uint64(recordData.Timestamp); + value.AddMember("payload").Utf8(recordData.Payload); + value.EndStruct(); + return value.Build(); +} + +int DoCreate(TDatabaseOptions& dbOptions, int argc, char** argv) { + TCreateOptions createOptions{ {dbOptions} }; + if (!ParseOptionsCreate(argc, argv, createOptions)) { + return EXIT_FAILURE; + } + + createOptions.CommonOptions.MaxInfly = createOptions.CommonOptions.MaxInputThreads; + + int result = CreateTable(dbOptions); + if (result) { + return result; + } + + std::uint32_t maxId = GetTableStats(dbOptions, TableName).MaxId; + + createOptions.CommonOptions.ReactionTime = TDuration::Seconds(20); + + Cout << TInstant::Now().ToRfc822StringLocal() << " Uploading initial content... do 'kill -USR1 " << GetPID() + << "' for progress details or Ctrl/Cmd+C to interrupt" << Endl; + + std::shared_ptr& jobs = *Singleton>(); + TJobGC gc(jobs); + jobs = std::make_shared(); + + jobs->Add(new TGenerateInitialContentJob(createOptions, maxId)); + + SetUpInteraction(); + + jobs->Start(); + jobs->Wait(); + jobs->ShowProgress(); + + return EXIT_SUCCESS; +} + +int DoRun(TDatabaseOptions& dbOptions, int argc, char** argv) { + TRunOptions runOptions{ {dbOptions} }; + if (!ParseOptionsRun(argc, argv, runOptions)) { + return EXIT_FAILURE; + } + + Cout << TInstant::Now().ToRfc822StringLocal() << " Creating and initializing jobs..." << Endl; + + std::uint32_t maxId = GetTableStats(dbOptions, TableName).MaxId; + + std::shared_ptr& jobs = *Singleton>(); + TJobGC gc(jobs); + jobs = std::make_shared(); + + if (!runOptions.DontRunA) { + runOptions.CommonOptions.Rps = runOptions.Read_rps; + runOptions.CommonOptions.ReactionTime = TDuration::MilliSeconds(runOptions.CommonOptions.A_ReactionTime); + jobs->Add(new TReadJob(runOptions.CommonOptions, maxId)); + } + if (!runOptions.DontRunB) { + runOptions.CommonOptions.Rps = runOptions.Write_rps; + runOptions.CommonOptions.ReactionTime = DefaultReactionTime; + jobs->Add(new TWriteJob(runOptions.CommonOptions, maxId)); + } + + TInstant start = TInstant::Now(); + TInstant deadline = start + TDuration::Seconds(runOptions.CommonOptions.SecondsToRun); + + jobs->Start(deadline); + + SetUpInteraction(); + Cout << "Jobs launched. Do 'kill -USR1 " << GetPID() + << "' for progress details or 'kill -INT " << GetPID() << "' (Ctrl/Cmd + C) to interrupt" << Endl + << " Start time: " << start.ToRfc822StringLocal() << Endl + << "Estimated finish time: " << deadline.ToRfc822StringLocal() << Endl; + + jobs->Wait(); + + Cout << "All jobs finished: " << TInstant::Now().ToRfc822StringLocal() << Endl; + + jobs->ShowProgress(); + + return EXIT_SUCCESS; +} + +int DoCleanup(TDatabaseOptions& dbOptions, int argc) { + if (argc > 1) { + Cerr << "Unexpected arguments after cleanup" << Endl; + return EXIT_FAILURE; + } + return DropTable(dbOptions); +} diff --git a/tests/slo_workloads/key_value/key_value.h b/tests/slo_workloads/key_value/key_value.h new file mode 100644 index 00000000000..10c0e876925 --- /dev/null +++ b/tests/slo_workloads/key_value/key_value.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include + +extern const std::string TableName; + +NYdb::TValue BuildValueFromRecord(const TKeyValueRecordData& recordData); + +// Initial content generation +class TGenerateInitialContentJob : public TThreadJob { +public: + TGenerateInitialContentJob(const TCreateOptions& createOpts, std::uint32_t maxId); + void ShowProgress(TStringBuilder& report) override; + void DoJob() override; + void OnFinish() override; + +private: + TExecutor Executor; + TPackGenerator PackGenerator; + std::uint64_t Total; +}; + +// Write workload job +class TWriteJob : public TThreadJob { +public: + TWriteJob(const TCommonOptions& opts, std::uint32_t maxId); + void ShowProgress(TStringBuilder& report) override; + void DoJob() override; + void OnFinish() override; + +private: + TExecutor Executor; + TKeyValueGenerator Generator; + std::atomic ValuesGenerated = 0; +}; + +// Read workload job +class TReadJob : public TThreadJob { +public: + TReadJob(const TCommonOptions& opts, std::uint32_t maxId); + void ShowProgress(TStringBuilder& report) override; + void DoJob() override; + void OnFinish() override; + +private: + std::unique_ptr Executor; + std::uint32_t ObjectIdRange; + bool SaveResult; +}; + +int CreateTable(TDatabaseOptions& dbOptions); +int DropTable(TDatabaseOptions& dbOptions); + +// Creates a table and fills it with initial content +int DoCreate(TDatabaseOptions& dbOptions, int argc, char** argv); +// Not implemented +int DoRun(TDatabaseOptions& dbOptions, int argc, char** argv); +// Drops the table +int DoCleanup(TDatabaseOptions& dbOptions, int argc); diff --git a/tests/slo_workloads/key_value/main.cpp b/tests/slo_workloads/key_value/main.cpp new file mode 100644 index 00000000000..e37a9770838 --- /dev/null +++ b/tests/slo_workloads/key_value/main.cpp @@ -0,0 +1,6 @@ +#include "key_value.h" + + +int main(int argc, char** argv) { + return DoMain(argc, argv, DoCreate, DoRun, DoCleanup); +} diff --git a/tests/slo_workloads/key_value/run.cpp b/tests/slo_workloads/key_value/run.cpp new file mode 100644 index 00000000000..fa2393c6b95 --- /dev/null +++ b/tests/slo_workloads/key_value/run.cpp @@ -0,0 +1,161 @@ +#include "key_value.h" + +#include +#include + +using namespace NYdb; +using namespace NYdb::NTable; + +TWriteJob::TWriteJob(const TCommonOptions& opts, std::uint32_t maxId) + : TThreadJob(opts) + , Executor(opts, Stats, TExecutor::ModeNonBlocking) + , Generator(opts, maxId) +{} + +void TWriteJob::ShowProgress(TStringBuilder& report) { + report << Endl << "=====- WriteJob report (Thread B) -=====" << Endl; + Executor.Report(report); + report << "Generated " << ValuesGenerated.load() << " new elements." << Endl + << "Generator compute time: " << Generator.GetComputeTime() << Endl; + Stats.PrintStatistics(report); + report << "==================================================" << Endl; +} + +void TWriteJob::DoJob() { + while (!ShouldStop.load() && TInstant::Now() < Deadline) { + TKeyValueRecordData recordData = Generator.Get(); + auto value = BuildValueFromRecord(recordData); + + ValuesGenerated.fetch_add(1); + + auto upload = [value{ std::move(value) }, this](TSession session)->TAsyncStatus { + static const TString query = Sprintf(R"( +--!syntax_v1 +PRAGMA TablePathPrefix("%s"); + +DECLARE $items AS + List>; + +UPSERT INTO `%s` SELECT * FROM AS_TABLE($items); + +)", Prefix.c_str(), TableName.c_str()); + + auto promise = NThreading::NewPromise(); + auto params = PackValuesToParamsAsList({value}); + + auto resultFuture = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + std::move(params), + TExecDataQuerySettings() + .KeepInQueryCache(true) + .OperationTimeout(MaxDelay + ReactionTimeDelay) + .ClientTimeout(MaxDelay + ReactionTimeDelay) + ); + + resultFuture.Subscribe([promise](TAsyncDataQueryResult queryFuture) mutable { + Y_ABORT_UNLESS(queryFuture.HasValue()); + TDataQueryResult queryResult = queryFuture.GetValue(); + promise.SetValue(std::move(queryResult)); + }); + + return promise.GetFuture(); + }; + + RpsProvider.Use(); + + if (!Executor.Execute(upload)) { + break; + } + } +} + +void TWriteJob::OnFinish() { + Executor.Finish(); + Executor.Wait(); + Stats.Flush(); +} + +// Implementation of TReadJob +TReadJob::TReadJob(const TCommonOptions& opts, std::uint32_t maxId) + : TThreadJob(opts) + , Executor(opts.RetryMode ? new TExecutorWithRetry(opts, Stats) : new TExecutor(opts, Stats)) + , ObjectIdRange(static_cast(maxId * 1.25)) // 20% of requests with no result + , SaveResult(opts.SaveResult) +{} + +void TReadJob::ShowProgress(TStringBuilder& report) { + report << Endl << "======- ReadJob report (Thread A) -======" << Endl; + Executor->Report(report); + Stats.PrintStatistics(report); + report << "========================================" << Endl; +} + +void TReadJob::DoJob() { + while (!ShouldStop.load() && TInstant::Now() < Deadline) { + std::uint32_t idToSelect = RandomNumber() % ObjectIdRange; + + auto read = [idToSelect, this](TSession session) -> TAsyncStatus { + static const TString query = Sprintf(R"( +--!syntax_v1 +PRAGMA TablePathPrefix("%s"); + +DECLARE $object_id_key AS Uint32; +DECLARE $object_id AS Uint32; + +SELECT * FROM `%s` WHERE `object_id_key` = $object_id_key AND `object_id` = $object_id; + +)", Prefix.c_str(), TableName.c_str()); + + auto promise = NThreading::NewPromise(); + auto params = TParamsBuilder() + .AddParam("$object_id_key") + .Uint32(GetHash(idToSelect)) + .Build() + .AddParam("$object_id") + .Uint32(idToSelect) + .Build() + .Build(); + + auto resultFuture = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + std::move(params), + TExecDataQuerySettings() + .KeepInQueryCache(true) + .OperationTimeout(MaxDelay + ReactionTimeDelay) + .ClientTimeout(MaxDelay + ReactionTimeDelay) + ); + + resultFuture.Subscribe([promise](TAsyncDataQueryResult queryFuture) mutable { + Y_ABORT_UNLESS(queryFuture.HasValue()); + TDataQueryResult queryResult = queryFuture.GetValue(); + promise.SetValue(std::move(queryResult)); + }); + + return promise.GetFuture(); + }; + + RpsProvider.Use(); + + if (!Executor->Execute(read)) { + break; + } + } +} + +void TReadJob::OnFinish() { + Executor->Finish(); + std::uint32_t infly = Executor->Wait(WaitTimeout); + if (infly) { + Cerr << "Warning: thread A finished while having " << infly << " infly requests." << Endl; + } + Stats.Flush(); + if (SaveResult) { + Stats.SaveResult(); + } +} diff --git a/tests/slo_workloads/utils/executor.cpp b/tests/slo_workloads/utils/executor.cpp new file mode 100644 index 00000000000..5e45cef31ca --- /dev/null +++ b/tests/slo_workloads/utils/executor.cpp @@ -0,0 +1,534 @@ +#include "executor.h" + +const TDuration WaitTimeout = TDuration::Seconds(10); + +// Debug use only: +std::atomic ReadPromises = 0; +std::atomic ExecutorPromises = 0; + + +TInsistentClient::TInsistentClient(const TCommonOptions& opts) + : Client( + opts.DatabaseOptions.Driver, + NYdb::NTable::TClientSettings() + .SessionPoolSettings(NYdb::NTable::TSessionPoolSettings().MaxActiveSessions(opts.MaxInfly)) + .MinSessionCV(8) + .AllowRequestMigration(true) + ) + , ClientMaxRetries(opts.MaxRetries) + , Timeout(opts.ReactionTime) + , RetryTimeout(Timeout / 2) + , SessionTimeout(Timeout + ReactionTimeDelay) + , UseApplicationTimeout(opts.UseApplicationTimeout) + , SendPreventiveRequest(opts.SendPreventiveRequest) +{ + if (UseApplicationTimeout || SendPreventiveRequest) { + CallbackQueue.Start(opts.MaxCallbackThreads); + // Thread that executes timeout callbacks + auto threadFunc = [this]() { + TDuration timeToSleep; + while (!ShouldStop.WaitT(timeToSleep)) { + TInstant wakeupTime; + TInstant now; + with_lock(CallbacksLock) { + now = TInstant::Now(); + while (!TimeoutCallbacks.empty() && now >= TimeoutCallbacks.front().ExecucionTime) { + Y_UNUSED(CallbackQueue.AddFunc(TimeoutCallbacks.front().Callback)); + RemoveTimeoutIter(TimeoutCallbacks.front().context); + } + while (!RetryCallbacks.empty() && now >= RetryCallbacks.front().ExecucionTime) { + Y_UNUSED(CallbackQueue.AddFunc(RetryCallbacks.front().Callback)); + RemoveRetryIter(RetryCallbacks.front().context); + } + if (RetryCallbacks.empty()) { + wakeupTime = now + RetryTimeout; + } else { + wakeupTime = RetryCallbacks.front().ExecucionTime; + } + if (!TimeoutCallbacks.empty()) { + wakeupTime = Min(wakeupTime, TimeoutCallbacks.front().ExecucionTime); + } + } + timeToSleep = wakeupTime - now; + } + }; + WorkThread.reset(SystemThreadFactory()->Run(threadFunc).Release()); + } +} + +TInsistentClient::~TInsistentClient() { + ShouldStop.Signal(); + if (UseApplicationTimeout || SendPreventiveRequest) { + if (WorkThread) { + WorkThread->Join(); + } else { + Cerr << (TStringBuilder() << "TInsistentClient::~TINsistentClient Error: WorkThread is not running." << Endl); + } + CallbackQueue.Stop(); + } + Client.Stop().Wait(WaitTimeout); +} + +void TInsistentClient::Report(TStringBuilder& out) const { + out << "Client retries sent: total " << CounterSStart.load() + << ", successful " << CounterSOk.load() << Endl; +} + +std::uint64_t TInsistentClient::GetActiveSessions() const { + std::int64_t sessions = Client.GetActiveSessionCount(); + return static_cast(sessions); +} + +void TInsistentClient::ClearContext(std::shared_ptr& context) { + if (SendPreventiveRequest) { + RemoveRetryIter(context); + } + if (UseApplicationTimeout) { + RemoveTimeoutIter(context); + } +} + +void TInsistentClient::RemoveRetryIter(std::shared_ptr& context) { + if (context->RetryIter.Valid) { + context->RetryIter.Valid = false; + RetryCallbacks.erase(context->RetryIter.RealIter); + } +} + +void TInsistentClient::RemoveTimeoutIter(std::shared_ptr& context) { + if (context->TimeoutIter.Valid) { + context->TimeoutIter.Valid = false; + TimeoutCallbacks.erase(context->TimeoutIter.RealIter); + } +} + +TAsyncFinalStatus TInsistentClient::ExecuteWithRetry(const NYdb::NTable::TTableClient::TOperationFunc& operation) { + TTracedPromise promise = TTracedPromise( + NThreading::NewPromise(), + &ExecutorPromises + ); + std::shared_ptr context = std::make_shared(); + + auto launchOperation = [this, operation, promise, context](bool firstTime) mutable { + with_lock(context->Lock) { + if (context->Finished) { + return; + } + } + auto callback = [promise, context, firstTime, this](const NYdb::TAsyncStatus& future) mutable { + Y_ABORT_UNLESS(future.HasValue()); + // Not setting promise under lock to avoid deadlock + bool firstCallback = false; + with_lock(context->Lock) { + if (!context->Finished) { + context->Finished = true; + firstCallback = true; + } + } + if (firstCallback) { + promise.SetValue(future.GetValue()); + with_lock(CallbacksLock) { + if (firstTime) { + CounterFOk.fetch_add(1); + } else { + CounterSOk.fetch_add(1); + } + ClearContext(context); + } + } + }; + if (firstTime) { + CounterFStart.fetch_add(1); + } else { + CounterSStart.fetch_add(1); + } + NYdb::NTable::TRetryOperationSettings settings; + settings.MaxRetries(ClientMaxRetries); + settings.GetSessionClientTimeout(SessionTimeout); + auto future = Client.RetryOperation(operation, settings); + future.Subscribe(std::move(callback)); + }; + + with_lock(CallbacksLock) { + TInstant now = TInstant::Now(); + + if (SendPreventiveRequest) { + auto onRetryTimeout = [launchOperation]() mutable { + launchOperation(false); + }; + + RetryCallbacks.push_back({ now + RetryTimeout, onRetryTimeout, context }); + context->RetryIter = { --RetryCallbacks.end() }; + } + + if (UseApplicationTimeout) { + auto onTimeout = [this, promise, context]() mutable { + // Not setting promise under lock to avoid deadlock + bool firstCallback = false; + with_lock(context->Lock) { + if (!context->Finished) { + context->Finished = true; + firstCallback = true; + } + } + if (firstCallback) { + promise.SetValue(TFinalStatus()); + with_lock(CallbacksLock) { + ClearContext(context); + } + } + }; + + TimeoutCallbacks.push_back({ now + Timeout, onTimeout, context }); + context->TimeoutIter = { --TimeoutCallbacks.end() }; + } + } + + launchOperation(true); + + return promise.GetFuture(); +} + +TExecutor::TExecutor(const TCommonOptions& opts, TStat& stats, EMode mode) + : Opts(opts) + , Stats(stats) + , InsistentClient(opts) + , Semaphore(opts.MaxInputThreads) + , Mode(mode) +{ + InputQueue = std::make_unique(); + InputQueue->Start(opts.MaxInputThreads); + + auto threadFunc = [this]() { + TInstant wakeupTime = TInstant::Now(); + while (!ShouldStop.load()) { + with_lock(Lock) { + UpdateStats(); + } + wakeupTime += TDuration::Seconds(1); + SleepUntil(wakeupTime); + } + }; + MetricsPusherThread.reset(SystemThreadFactory()->Run(threadFunc).Release()); +} + +TExecutor::~TExecutor() { + std::uint32_t infly = StopAndWait(WaitTimeout); + if (MetricsPusherThread) { + MetricsPusherThread->Join(); + } else { + Cerr << (TStringBuilder() << "TExecutor::~TExecutor Error: MetricsPusherThread is not running." << Endl); + } + if (infly) { + Cerr << "Warning: destroying TExecutor while having " << infly << " infly requests." << Endl; + } +} + +namespace { + class TSemaphoreWrapper : public TThrRefBase { + public: + TSemaphoreWrapper(TFastSemaphore& semaphore) + : Semaphore(semaphore) + {} + + ~TSemaphoreWrapper() { + if (Acquired) { + Semaphore.Release(); + } + } + + void Acquire() { + Semaphore.Acquire(); + Acquired = true; + } + + private: + TFastSemaphore& Semaphore; + bool Acquired = false; + }; +} + +bool TExecutor::Execute(const NYdb::NTable::TTableClient::TOperationFunc& func) { + TIntrusivePtr SemaphoreWrapper; + if (Mode == ModeBlocking) { + SemaphoreWrapper = new TSemaphoreWrapper(Semaphore); + } + auto threadFunc = [this, func, SemaphoreWrapper]() { + if (IsStopped()) { + DecrementWaiting(); + return; + } + + with_lock(Lock) { + --Waiting; + if (Infly < Opts.MaxInfly) { + ++Infly; + UpdateStats(); + if (Infly > MaxInfly) { + MaxInfly = Infly; + } + } else { + Stats.ReportMaxInfly(); + UpdateStats(); + return; + } + } + + TStatUnit stat = Stats.CreateStatUnit(); + + auto future = InsistentClient.ExecuteWithRetry(func); + future.Subscribe([this, stat, SemaphoreWrapper](const TAsyncFinalStatus& future) mutable { + Y_ABORT_UNLESS(future.HasValue()); + TFinalStatus resultStatus = future.GetValue(); + Stats.Report(stat, resultStatus); + if (resultStatus) { + CheckForError(*resultStatus); + } + DecrementInfly(); + }); + }; + + if (IsStopped()) { + return false; + } + + bool CanLaunchJob = false; + + with_lock(Lock) { + if (!AllJobsLaunched) { + CanLaunchJob = true; + ++Waiting; + } + } + + if (CanLaunchJob) { + if (Mode == ModeBlocking) { + SemaphoreWrapper->Acquire(); + } + if (!InputQueue->AddFunc(threadFunc)) { + DecrementWaiting(); + } + } + ++InProgressCount; + InProgressSum += InputQueue->Size(); + return true; +} + +void TExecutor::Start(TInstant deadline) { + Deadline = deadline; +} + +void TExecutor::Stop() { + if (!IsStopped()) { + ShouldStop.store(true); + Finish(); + } +} + +void TExecutor::Wait() { + AllJobsFinished.WaitI(); + InputQueue->Stop(); +} + +std::uint32_t TExecutor::Wait(TDuration waitTimeout) { + AllJobsFinished.WaitT(waitTimeout); + InputQueue->Stop(); + return Infly; +} + +void TExecutor::StopAndWait() { + Stop(); + Wait(); +} + +std::uint32_t TExecutor::StopAndWait(TDuration waitTimeout) { + Stop(); + return Wait(waitTimeout); +} + +bool TExecutor::IsStopped() { + return ShouldStop.load(); +} + +void TExecutor::Finish() { + // Stats.UpdateSessionStats(InsistentClient.GetSessionStats()); + with_lock(Lock) { + if (!AllJobsLaunched) { + AllJobsLaunched = true; + CheckForFinish(); + } + } +} + +void TExecutor::UpdateStats() { + if (Infly > MaxSecInfly) { + MaxSecInfly = Infly; + } + std::uint64_t activeSessions = InsistentClient.GetActiveSessions(); + if (activeSessions > MaxSecSessions) { + MaxSecSessions = activeSessions; + } + + // Debug use only: + std::uint64_t readPromises = ReadPromises.load(); + if (readPromises > MaxSecReadPromises) { + MaxSecReadPromises = readPromises; + } + std::uint64_t executorPromises = ExecutorPromises.load(); + if (executorPromises > MaxSecExecutorPromises) { + MaxSecExecutorPromises = executorPromises; + } + ReportStats(); +} + +void TExecutor::ReportStats() { + TInstant now = TInstant::Now(); + if (now.Seconds() > LastReportSec) { + Stats.ReportStats(MaxSecInfly, MaxSecSessions, MaxSecReadPromises, MaxSecExecutorPromises); + MaxSecInfly = 0; + MaxSecSessions = 0; + MaxSecReadPromises = 0; + MaxSecExecutorPromises = 0; + LastReportSec = now.Seconds(); + } +} + +void TExecutor::DecrementInfly() { + with_lock(Lock) { + --Infly; + UpdateStats(); + if (!Infly) { + CheckForFinish(); + } + } +} + +void TExecutor::DecrementWaiting() { + with_lock(Lock) { + --Waiting; + CheckForFinish(); + } +} + +void TExecutor::CheckForFinish() { + if (AllJobsLaunched && !Infly && !Waiting) { + AllJobsFinished.Signal(); + } +} + +void TExecutor::CheckForError(const NYdb::TStatus& status) { + if (!status.IsSuccess()) { + with_lock(ErrorLock) { + auto it = Errors.find(status.GetStatus()); + if (it == Errors.end()) { + Errors.insert({ status.GetStatus() , { status.GetIssues().ToString() , 1 } }); + } else { + ++it->second.Counter; + } + } + } +} + +void TExecutor::Report(TStringBuilder& out) const { + out << MaxInfly << " maxInfly" << Endl + << (InProgressCount ? InProgressSum / InProgressCount : 0) << " average Inprogress threads in input queue" << Endl; + InsistentClient.Report(out); + with_lock(ErrorLock) { + if (Errors.size()) { + out << "Errors:" << Endl; + for (auto& error : Errors) { + out << error.second.Counter << " errors with status " << error.first << ": " << error.second.Message << Endl; + } + } + } +} + + +TExecutorWithRetry::TExecutorWithRetry(const TCommonOptions& opts, TStat& stats) + : TExecutor(opts, stats) +{} + +bool TExecutorWithRetry::Execute(const NYdb::NTable::TTableClient::TOperationFunc& func) { + auto threadFunc = [this, func]() { + if (IsStopped()) { + DecrementWaiting(); + return; + } + + with_lock(Lock) { + --Waiting; + if (Infly < Opts.MaxInfly) { + ++Infly; + if (Infly > MaxInfly) { + MaxInfly = Infly; + } + UpdateStats(); + } else { + Stats.ReportMaxInfly(); + UpdateStats(); + return; + } + } + + std::shared_ptr context = std::make_shared(Stats); + + auto executeOperation = [this, func]() { + return InsistentClient.ExecuteWithRetry(func); + }; + + context->HandleStatusFunc = std::make_unique>( + [this, executeOperation, context](const TAsyncFinalStatus& future) mutable { + Y_ABORT_UNLESS(future.HasValue()); + TFinalStatus resultStatus = future.GetValue(); + if (resultStatus) { + // Reply received + CheckForError(*resultStatus); + if (resultStatus->IsSuccess()) { + //Ok received + Stats.Report(context->LifeTimeStat, resultStatus->GetStatus()); + DecrementInfly(); + context->HandleStatusFunc.reset(); + return; + } + } + if (IsStopped() || TInstant::Now() - context->LifeTimeStat.Start > GlobalTimeout) { + // Application stopped working or global timeout reached. Ok reply hasn't received yet + Stats.Report(context->LifeTimeStat, TInnerStatus::StatusNotFinished); + DecrementInfly(); + context->HandleStatusFunc.reset(); + return; + } + Stats.Report(context->PerRequestStat, resultStatus); + context->PerRequestStat = Stats.CreateStatUnit(); + // Retrying: + executeOperation().Subscribe(*context->HandleStatusFunc); + }); + + context->Retries.fetch_add(1); + Y_ABORT_UNLESS(context->Retries.load() < 500, "Too much retries"); + + executeOperation().Subscribe(*context->HandleStatusFunc); + }; + + if (IsStopped()) { + return false; + } + + bool CanLaunchJob = false; + + with_lock(Lock) { + if (!AllJobsLaunched) { + CanLaunchJob = true; + ++Waiting; + } + } + + if (CanLaunchJob) { + if (!InputQueue->AddFunc(threadFunc)) { + DecrementWaiting(); + } + } + ++InProgressCount; + InProgressSum += InputQueue->Size(); + return true; +} diff --git a/tests/slo_workloads/utils/executor.h b/tests/slo_workloads/utils/executor.h new file mode 100644 index 00000000000..5dddb336b6d --- /dev/null +++ b/tests/slo_workloads/utils/executor.h @@ -0,0 +1,198 @@ +#pragma once + +#include "utils.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +extern const TDuration WaitTimeout; + +// Debug use only +extern std::atomic ReadPromises; + +template +class TTracedPromise : public NThreading::TPromise { +public: + TTracedPromise(NThreading::TPromise promise, std::atomic* counter) + : NThreading::TPromise(promise) + , Counter(counter) + { + Counter->fetch_add(1); + } + + TTracedPromise(const TTracedPromise& other) + : NThreading::TPromise(other) + , Counter(other.Counter) + { + other.Counter = nullptr; + } + + TTracedPromise& operator=(const TTracedPromise&) = delete; + + ~TTracedPromise() { + if (Counter) { + Counter->fetch_sub(1); + } + } + +private: + mutable std::atomic* Counter; +}; + +class TInsistentClient { +public: + struct TDelayedCallback; + + struct TCheckedIterator { + std::list::iterator RealIter; + bool Valid = true; + }; + + struct TOperationContext : public TThrRefBase { + bool Finished = false; + TAdaptiveLock Lock; + TCheckedIterator RetryIter; + TCheckedIterator TimeoutIter; + }; + + struct TDelayedCallback { + TInstant ExecucionTime; + std::function Callback; + std::shared_ptr context; + }; + + TInsistentClient(const TCommonOptions& opts); + ~TInsistentClient(); + void Report(TStringBuilder& out) const; + TAsyncFinalStatus ExecuteWithRetry(const NYdb::NTable::TTableClient::TOperationFunc& operation); + std::uint64_t GetActiveSessions() const; + +private: + void ClearContext(std::shared_ptr& context); + void RemoveRetryIter(std::shared_ptr& context); + void RemoveTimeoutIter(std::shared_ptr& context); + + TThreadPool CallbackQueue; + NYdb::NTable::TTableClient Client; + std::uint32_t ClientMaxRetries; + TDuration Timeout; + TDuration RetryTimeout; + TDuration SessionTimeout; + TAdaptiveLock CallbacksLock; + std::unique_ptr WorkThread; + TManualEvent ShouldStop; + std::list RetryCallbacks; + std::list TimeoutCallbacks; + bool UseApplicationTimeout; + bool SendPreventiveRequest; + + // Ok received on the First try + std::atomic CounterFOk = 0; + // Ok received on the Second try + std::atomic CounterSOk = 0; + // First try launches (= total) + std::atomic CounterFStart = 0; + // Second try launches + std::atomic CounterSStart = 0; +}; + +class TExecutor { +public: + enum EMode { + ModeBlocking, + ModeNonBlocking + }; + + struct TErrorData { + std::string Message; + std::uint64_t Counter; + }; + + TExecutor(const TCommonOptions& opts, TStat& stats, EMode mode = ModeNonBlocking); + virtual ~TExecutor(); + virtual bool Execute(const NYdb::NTable::TTableClient::TOperationFunc& func); + + void Start(TInstant deadline); + // Abort all waiting jobs and do not accept new ones + void Stop(); + // Wait for all jobs to finish + void Wait(); + // Signal that there will be no more new jobs + void StopAndWait(); + std::uint32_t StopAndWait(TDuration waitTimeout); + std::uint32_t Wait(TDuration waitTimeout); + bool IsStopped(); + void Finish(); + std::uint32_t GetTotal() const; + void Report(TStringBuilder& out) const; + +protected: + void DecrementInfly(); + void DecrementWaiting(); + // Checks if all jobs are done + void CheckForFinish(); + void UpdateStats(); + void ReportStats(); + void CheckForError(const NYdb::TStatus& status); + + const TCommonOptions& Opts; + TStat& Stats; + TInsistentClient InsistentClient; + TFastSemaphore Semaphore; + EMode Mode; + std::unique_ptr InputQueue; + std::unique_ptr MetricsPusherThread; + std::atomic ShouldStop = false; + std::atomic Total = 0; + std::atomic Succeeded = 0; + std::atomic Failed = 0; + TAdaptiveLock ErrorLock; + std::unordered_map Errors; + TAdaptiveLock Lock; + // Jobs put in queue but haven't started yet + std::uint32_t Waiting = 0; + // Jobs started executing + std::uint32_t Infly = 0; + std::uint32_t MaxInfly = 0; + bool AllJobsLaunched = false; + TManualEvent AllJobsFinished; + TInstant Deadline; + // Last second we reported Infly + std::uint64_t LastReportSec = 0; + // Max infly for current second + std::uint64_t MaxSecInfly = 0; + // Max Active sessions for current second + std::uint64_t MaxSecSessions = 0; + + //(Debug usage) Monitoring the number of jobs waiting for rps limiter + std::size_t InProgressCount = 0; + std::size_t InProgressSum = 0; + std::uint64_t MaxSecReadPromises = 0; + std::uint64_t MaxSecExecutorPromises = 0; +}; + +class TExecutorWithRetry : public TExecutor { +public: + struct TRetryContext { + TRetryContext(TStat& stat) + : LifeTimeStat(stat.CreateStatUnit()) + , PerRequestStat(stat.CreateStatUnit()) + {} + + TStatUnit LifeTimeStat; + TStatUnit PerRequestStat; + std::unique_ptr> HandleStatusFunc; + std::atomic Retries = 0; + }; + + TExecutorWithRetry(const TCommonOptions& opts, TStat& stats); + bool Execute(const NYdb::NTable::TTableClient::TOperationFunc& func) override; +}; diff --git a/tests/slo_workloads/utils/generator.cpp b/tests/slo_workloads/utils/generator.cpp new file mode 100644 index 00000000000..85433478388 --- /dev/null +++ b/tests/slo_workloads/utils/generator.cpp @@ -0,0 +1,47 @@ +#include "generator.h" + +#include + + +TValueGenerator::TValueGenerator(const TCommonOptions& opts, ui32 startId) + : Opts(opts) + , CurrentObjectId(startId) +{ +} + +TRecordData TValueGenerator::Get() { + TDurationMeter computeTime(ComputeTime); + if (RandomNumber() % 10 > 2) { + ++CurrentObjectId; + } + return { + CurrentObjectId, + TInstant::Now().MicroSeconds(), + CreateGuidAsString(), + GenerateRandomString(Opts.MinLength, Opts.MaxLength) + }; +} + +TDuration TValueGenerator::GetComputeTime() const { + return ComputeTime; +} + +TKeyValueGenerator::TKeyValueGenerator(const TCommonOptions& opts, ui32 startId) + : Opts(opts) + , CurrentObjectId(startId) +{ +} + +TKeyValueRecordData TKeyValueGenerator::Get() { + TDurationMeter computeTime(ComputeTime); + ++CurrentObjectId; + return { + CurrentObjectId, + TInstant::Now().MicroSeconds(), + GenerateRandomString(Opts.MinLength, Opts.MaxLength) + }; +} + +TDuration TKeyValueGenerator::GetComputeTime() const { + return ComputeTime; +} diff --git a/tests/slo_workloads/utils/generator.h b/tests/slo_workloads/utils/generator.h new file mode 100644 index 00000000000..584ecd68e96 --- /dev/null +++ b/tests/slo_workloads/utils/generator.h @@ -0,0 +1,90 @@ +#pragma once + +#include "utils.h" + +#include + + +class TValueGenerator { +public: + TValueGenerator(const TCommonOptions& opts, std::uint32_t startId = 0); + TRecordData Get(); + TDuration GetComputeTime() const; + +private: + const TCommonOptions& Opts; + std::uint32_t CurrentObjectId = 0; + TDuration ComputeTime; +}; + +class TKeyValueGenerator { +public: + TKeyValueGenerator(const TCommonOptions& opts, std::uint32_t startId = 0); + TKeyValueRecordData Get(); + TDuration GetComputeTime() const; + +private: + const TCommonOptions& Opts; + std::uint32_t CurrentObjectId = 0; + TDuration ComputeTime; +}; + +template +class TPackGenerator { +public: + TPackGenerator( + const TCommonOptions& opts, + std::uint32_t packSize, + NYdb::TValue(*buildValueFromRecordFunc)(const TRecordType&), + std::uint64_t remain, + std::uint32_t startId = 0 + ) + : BuildValueFromRecordFunc(buildValueFromRecordFunc) + , Generator(opts, startId) + , PackSize(packSize) + , Remain(remain) + { + } + + // Returns One-shard pack + bool GetNextPack(std::vector& pack) { + pack.clear(); + while (Remain) { + TRecordType record = Generator.Get(); + --Remain; + std::uint32_t specialId = GetSpecialId(GetHash(record.ObjectId)); + auto& existingPack = Packs[specialId]; + existingPack.emplace_back(BuildValueFromRecordFunc(record)); + if (existingPack.size() >= PackSize) { + existingPack.swap(pack); + return true; + } + } + for (auto& it : Packs) { + if (it.second.size()) { + it.second.swap(pack); + return true; + } + } + return false; + } + + std::uint32_t GetPackSize() const { + return PackSize; + } + + TDuration GetComputeTime() const { + return Generator.GetComputeTime(); + } + + std::uint64_t GetRemain() const { + return Remain; + } + +private: + NYdb::TValue(*BuildValueFromRecordFunc)(const TRecordType&); + TGeneratorType Generator; + std::uint32_t PackSize; + std::unordered_map> Packs; + std::uint64_t Remain; +}; diff --git a/tests/slo_workloads/utils/job.cpp b/tests/slo_workloads/utils/job.cpp new file mode 100644 index 00000000000..23421bcb747 --- /dev/null +++ b/tests/slo_workloads/utils/job.cpp @@ -0,0 +1,110 @@ +#include "job.h" + +#include + +const std::string LogFileName = "benchmark.log"; + + +TThreadJob::TThreadJob(const TCommonOptions& opts) + : RpsProvider(opts.Rps) + , Prefix(opts.DatabaseOptions.Prefix) + , Stats( + opts.ReactionTime, + opts.ResultFileName, + !opts.DontPushMetrics, + opts.RetryMode + ) + , StopOnError(opts.StopOnError) + , MaxDelay(opts.ReactionTime) + , UseFollowers(opts.UseFollowers) +{ +} + +// virtual +void TThreadJob::Start(TInstant deadline) { + Deadline = deadline; + StartThread(); +} + +void TThreadJob::StartThread() { + auto threadFunc = [this]() { + Stats.Reset(); + RpsProvider.Reset(); + DoJob(); + Stats.Finish(); + OnFinish(); + }; + WorkThread.reset(SystemThreadFactory()->Run(threadFunc).Release()); +} + +void TThreadJob::Stop() { + if (WorkThread) { + ShouldStop.store(true); + } else { + Cerr << (TStringBuilder() << "TGenerateInitialContentJob::Stop Error: Thread is not running." << Endl); + } +} + +void TThreadJob::Wait() { + if (WorkThread) { + WorkThread->Join(); + } +} +void TThreadJob::OnFinish() { +} + +void TJobContainer::Add(TThreadJob* job) { + Y_ABORT_UNLESS(job); + Jobs.push_back(std::unique_ptr(job)); +} + +void TJobContainer::Start(TInstant deadline) { + for (auto& job : Jobs) { + job->Start(deadline); + } +} + +void TJobContainer::Stop() { + for (auto& job : Jobs) { + job->Stop(); + } +} + +void TJobContainer::Wait() { + for (auto& job : Jobs) { + job->Wait(); + } +} + +void TJobContainer::ShowProgress() { + TStringBuilder report; + for (auto& job : Jobs) { + job->ShowProgress(report); + } + Cout << report; + TFile logFile(LogFileName, OpenAlways | WrOnly | Seq | ForAppend); + TFileOutput out(logFile); + out << report; +} + +void SetUpInteraction() { + signal(SIGUSR1, [](int) -> void { + Cout << (TStringBuilder() << TInstant::Now().ToRfc822StringLocal() << " " << "SIGUSR1 handle" << Endl); + std::shared_ptr jobs = *Singleton>(); + if (jobs) { + jobs->ShowProgress(); + } else { + Cerr << "Jobs are already destroyed" << Endl; + } + }); + + signal(SIGINT, [](int) -> void { + Cerr << (TStringBuilder() << TInstant::Now().ToRfc822StringLocal() << " " << "SIGINT handle received. Stop signal sent." << Endl); + std::shared_ptr jobs = *Singleton>(); + if (jobs) { + jobs->Stop(); + } else { + Cerr << "Jobs are already destroyed" << Endl; + } + }); +} diff --git a/tests/slo_workloads/utils/job.h b/tests/slo_workloads/utils/job.h new file mode 100644 index 00000000000..7e6e48bede5 --- /dev/null +++ b/tests/slo_workloads/utils/job.h @@ -0,0 +1,66 @@ +#pragma once + +#include "utils.h" + +#include +#include + +class TThreadJob { +public: + TThreadJob(const TCommonOptions& opts); + virtual ~TThreadJob() = default; + + virtual void Start(TInstant deadline); + void Stop(); + void Wait(); + virtual void OnFinish(); + + virtual void ShowProgress(TStringBuilder& report) = 0; + virtual void DoJob() = 0; + +protected: + void StartThread(); + + TInstant Deadline; + TRpsProvider RpsProvider; + const std::string& Prefix; + std::atomic ShouldStop = false; + std::atomic ErrorOccured = false; + std::unique_ptr WorkThread; + TInstant StartTime; + TStat Stats; + bool StopOnError; + TDuration MaxDelay; + bool UseFollowers; +}; + +class TJobContainer : public TThrRefBase { +public: + void Add(TThreadJob* job); + void Start(TInstant deadline = TInstant()); + void Stop(); + void Wait(); + void ShowProgress(); + +private: + std::vector> Jobs; +}; + +class TJobGC { +public: + TJobGC(std::shared_ptr& jobs) + : Jobs(jobs) + {} + + ~TJobGC() { + if (Jobs) { + Jobs->Wait(); + Jobs.reset(); + } + } + +private: + std::shared_ptr& Jobs; +}; + +void SetUpInteraction(); diff --git a/tests/slo_workloads/utils/statistics.cpp b/tests/slo_workloads/utils/statistics.cpp new file mode 100644 index 00000000000..7ffcc04babf --- /dev/null +++ b/tests/slo_workloads/utils/statistics.cpp @@ -0,0 +1,520 @@ +#include "statistics.h" + +#include +#include + +#include + + +TStatUnit::TStatUnit(const std::shared_ptr& periodData, TInstant startTime) + : PeriodData(periodData) + , Start(startTime) +{ +} + +void TStatUnit::Report(const TInnerStatus& status) { + PeriodData->AddStat(Delay(), status); +} + +TDuration TStatUnit::Delay() const { + return End - Start; +} + +TPeriodData::TPeriodData( + TStat* stats, + bool retryMode, + const TDuration maxDelay, + std::uint64_t currentSecond, + std::uint64_t currInfly, + std::uint64_t currSessions, + std::uint64_t currPromises, + std::uint64_t currExecutorPromises +) + : Stats(stats) + , RetryMode(retryMode) + , CurrentSecond(currentSecond) + , MaxDelay(maxDelay) +{ + Counters.Infly = currInfly; + Counters.ActiveSessions = currSessions; + Counters.ReadPromises = currPromises; + Counters.ExecutorPromises = currExecutorPromises; +} + +TPeriodData::~TPeriodData() { + Stats->ReportLatencyData(CurrentSecond, std::move(Counters), std::move(Replies), std::move(OkDelays), NotOkDelays); +} + +TStatUnit TPeriodData::CreateStatUnit(TInstant startTime) { + return TStatUnit(shared_from_this(), startTime); +} + +std::uint64_t TPeriodData::GetCurrentSecond() const { + return CurrentSecond; +} + +void TPeriodData::ReportMaxInfly() { + ++Replies.CountMaxInfly; +} + +void TPeriodData::ReportInfly(std::uint64_t infly) { + if (infly > Counters.Infly) { + Counters.Infly = infly; + } +} + +void TPeriodData::ReportActiveSessions(std::uint64_t sessions) { + if (sessions > Counters.ActiveSessions) { + Counters.ActiveSessions = sessions; + } +} + +// Debug use only: +void TPeriodData::ReportReadPromises(std::uint64_t promises) { + if (promises > Counters.ReadPromises) { + Counters.ReadPromises = promises; + } +} +void TPeriodData::ReportExecutorPromises(std::uint64_t promises) { + if (promises > Counters.ExecutorPromises) { + Counters.ExecutorPromises = promises; + } +} + +void TPeriodData::AddStat(TDuration delay, const TInnerStatus& status) { + if (status.InnerStatus == TInnerStatus::StatusReceived && status.YdbStatus == NYdb::EStatus::SUCCESS) { + OkDelays.push_back(delay); + } else { + NotOkDelays.push_back(delay); + } + + switch (status.InnerStatus) { + case TInnerStatus::StatusReceived: + if (status.YdbStatus != NYdb::EStatus::SUCCESS || !RetryMode || delay <= MaxDelay) { + ++Replies.Statuses[status.YdbStatus]; + } else { + ++Replies.CountHighLatency; + } + break; + case TInnerStatus::StatusApplicationTimeout: + ++Replies.ApplicationTimeout; + break; + case TInnerStatus::StatusNotFinished: + ++Replies.NotFinished; + break; + default: + Y_ABORT_UNLESS(status.InnerStatus); + break; + } +} + +namespace { + void CalculatePercentiles(TPercentile& p, std::vector& delays) { + size_t count = delays.size(); + if (count) { + std::sort(delays.begin(), delays.end()); + p.P50 = delays[(count - 1) * 50 / 100]; + p.P90 = delays[(count - 1) * 90 / 100]; + p.P95 = delays[(count - 1) * 95 / 100]; + p.P99 = delays[(count - 1) * 99 / 100]; + p.P99_9 = delays[(count - 1) * 999 / 1000]; + p.P100 = delays[count - 1]; + } + } +} + +TStat::TStat( + TDuration maxDelay, + const std::string& resultFileName, + bool pushMetrics, + bool retryMode +) + : MaxDelay(maxDelay) + , StartTime(TInstant::Now()) + , PushMetrics(pushMetrics) + , RetryMode(retryMode) + , ResultFileName(resultFileName) +{ + MetricsPushQueue.Start(20); +} + +TStat::~TStat() { + Flush(); +} + +void TStat::Flush() { + std::uint64_t lastSecMeasured = TInstant::Now().Seconds(); + if (ActivePeriod) { + lastSecMeasured = ActivePeriod->GetCurrentSecond(); + ActivePeriod.reset(); + } + if (PushMetrics) { + ResetMetricsPusher(lastSecMeasured + 1); + MetricsPushQueue.Stop(); + } +} + +void TStat::Reset() { + if (PushMetrics) { + ResetMetricsPusher(TInstant::Now().Seconds() - 1); + } + StartTime = TInstant::Now(); +} + +void TStat::Finish() { + FinishTime = TInstant::Now(); +} + +TStatUnit TStat::CreateStatUnit() { + std::lock_guard lock(Mutex); + + TInstant Now = TInstant::Now(); + CheckCurrentSecond(Now); + return ActivePeriod->CreateStatUnit(Now); +} + +void TStat::Report(TStatUnit& unit, const TInnerStatus& status) { + unit.End = TInstant::Now(); + OnReport(unit, status); +} + +void TStat::Report(TStatUnit& unit, TInnerStatus::EInnerStatus innerStatus) { + Report(unit, TInnerStatus(innerStatus)); +} + +void TStat::Report(TStatUnit& unit, const TFinalStatus& status) { + Report( + unit, + status + ? TInnerStatus(TInnerStatus::StatusReceived, status->GetStatus()) + : TInnerStatus(TInnerStatus::StatusApplicationTimeout) + ); +} + +void TStat::Report(TStatUnit& unit, NYdb::EStatus status) { + Report(unit, TInnerStatus(TInnerStatus::StatusReceived, status)); +} + +void TStat::ReportMaxInfly() { + std::lock_guard lock(Mutex); + + ++Replies.CountMaxInfly; + TInstant Now = TInstant::Now(); + CheckCurrentSecond(Now); + ActivePeriod->ReportMaxInfly(); +} + +void TStat::ReportStats(std::uint64_t infly, std::uint64_t sessions, std::uint64_t readPromises, std::uint64_t executorPromises) { + std::lock_guard lock(Mutex); + + Counters.Infly = infly; + Counters.ActiveSessions = sessions; + Counters.ReadPromises = readPromises; + Counters.ExecutorPromises = executorPromises; + TInstant Now = TInstant::Now(); + CheckCurrentSecond(Now); + ActivePeriod->ReportInfly(infly); + ActivePeriod->ReportActiveSessions(sessions); + ActivePeriod->ReportReadPromises(readPromises); + ActivePeriod->ReportExecutorPromises(executorPromises); +} + +void TStat::Reserve(size_t size) { + std::lock_guard lock(Mutex); + + LatencyStats.reserve(size); + LatencyData.reserve(size); +} + +void TStat::ReportLatencyData( + std::uint64_t currSecond, + TCounters&& counters, + TReplies&& replies, + std::vector&& oks, + std::vector& notOks +) { + std::lock_guard lock(Mutex); + + LatencyStats.emplace_back(); + TPeriodStat& p = LatencyStats.back(); + p.Seconds = currSecond; + std::swap(p.Counters, counters); + std::swap(p.Replies, replies); + CalculatePercentiles(p.Oks, oks); + CalculatePercentiles(p.NotOks, notOks); + LatencyData.emplace_back(std::move(oks)); + PushMetricsData(p); +} + +void TStat::UpdateSessionStats( + const std::unordered_map& sessionStats +) { + std::lock_guard lock(Mutex); + + SessionStats = sessionStats; +} + +void TStat::PrintStatistics(TStringBuilder& out) { + std::lock_guard lock(Mutex); + + TInstant now = TInstant::Now(); + CheckCurrentSecond(now); + std::uint64_t total = GetTotal(); + + TDuration timePassed; + if (FinishTime < StartTime) { + // If we ask for current progress + timePassed = now - StartTime; + } else { + timePassed = FinishTime - StartTime; + } + + std::uint64_t rps = total * 1000000 / timePassed.MicroSeconds(); + out << total << " requests total" << Endl + << Replies.Statuses[NYdb::EStatus::SUCCESS] << " succeeded"; + if (total) { + out << " (" << Replies.Statuses[NYdb::EStatus::SUCCESS] * 100 / total << "%)"; + } + for (const auto&[status, counter] : Replies.Statuses) { + out << Endl << counter << " replies with status " << YdbStatusToString(status) << Endl; + } + out << Endl << Replies.CountMaxInfly << " failed due to max infly" << Endl + << Replies.CountHighLatency << " OK results exceeded latency limit of " << MaxDelay << Endl + << Replies.ApplicationTimeout << " application timeouts" << Endl + << Replies.NotFinished << " requests not finished within program lifetime" << Endl + << "Time passed: " << timePassed.ToString() << Endl + << "Real rps: " << rps << Endl; + + if (LatencyData.size()) { + CalculateGlobalPercentile(); + TPercentile& p = *GlobalPercentile; + out << "Global latency percentiles (" << LatencyData.size() << " seconds measured):" << Endl + << "P50: " << p.P50 << "\tP90: " << p.P90 << "\tP95: " << p.P95 << "\tP99: " << p.P99 + << "\tP99.9: " << p.P99_9 << "\tP100: " << p.P100 << Endl; + CalculateFailSeconds(); + out << *FailSeconds << " seconds where p99 reached max delay of " << MaxDelay << Endl; + } else { + out << "Can't calculate latency percentiles: No data (zero requests measured)" << Endl; + } +} + +void TStat::SaveResult() { + std::lock_guard lock(Mutex); + + if (LatencyData.size()) { + CalculateGlobalPercentile(); + CalculateFailSeconds(); + NJson::TJsonValue root; + root["Oks"] = Replies.Statuses[NYdb::EStatus::SUCCESS]; + root["Total"] = GetTotal(); + root["P99"] = GlobalPercentile->P99.MilliSeconds(); + root["FailSeconds"] = *FailSeconds; + root["StartTime"] = StartTime.Seconds(); + root["FinishTime"] = FinishTime.Seconds(); + NJson::TJsonValue& items = root["SessionCountsAtFinish"]; + items.SetType(NJson::JSON_ARRAY); + for (const auto& s : SessionStats) { + items.AppendValue(s.second); + } + TFileOutput resultFile(ResultFileName); + NJsonWriter::TBuf buf; + buf.WriteJsonValue(&root); + resultFile << buf.Str(); + Cout << "Result saved to file " << ResultFileName << Endl; + } +} + +void TStat::CalculateGlobalPercentile() { + if (GlobalPercentile) { + return; + } + std::vector fullData; + size_t totalSize = 0; + for (auto& periodData : LatencyData) { + totalSize += periodData.size(); + } + fullData.reserve(totalSize); + for (auto& periodData : LatencyData) { + fullData.insert(fullData.end(), periodData.begin(), periodData.end()); + } + GlobalPercentile = std::make_unique(); + CalculatePercentiles(*GlobalPercentile, fullData); +} + +namespace { + bool IsGoodInterval(const TPeriodStat& stat, const TDuration& maxDelay) { + for (const auto& [status, counter] : stat.Replies.Statuses) { + if (status != NYdb::EStatus::SUCCESS && counter) { + return false; + } + } + return stat.Oks.P99 <= maxDelay && !stat.Replies.CountMaxInfly && !stat.Replies.ApplicationTimeout; + } +} + +void TStat::CalculateFailSeconds() { + if (FailSeconds) { + return; + } + std::sort(LatencyStats.begin(), LatencyStats.end(), [&](const TPeriodStat& a, const TPeriodStat& b) { + return a.Seconds < b.Seconds; + }); + FailSeconds = std::make_unique(0); + size_t& failSeconds = *FailSeconds; + std::uint64_t lastSecChecked = LatencyStats[0].Seconds - 1; + for (auto& stat : LatencyStats) { + failSeconds += stat.Seconds - lastSecChecked - 1; + lastSecChecked = stat.Seconds; + if (!IsGoodInterval(stat, MaxDelay)) { + ++failSeconds; + } + } +} + +std::uint64_t TStat::GetTotal() { + std::uint64_t total = Replies.CountMaxInfly + Replies.CountHighLatency + Replies.NotFinished; + if (!RetryMode) { + for (const auto& [status, counter] : Replies.Statuses) { + total += counter; + } + total += Replies.ApplicationTimeout; + } else { + total += Replies.Statuses[NYdb::EStatus::SUCCESS]; + } + return total; +} + +TInstant TStat::GetStartTime() const { + return StartTime; +} + +void TStat::OnReport(TStatUnit& unit, const TInnerStatus& status) { + std::lock_guard lock(Mutex); + + TDuration delay = unit.Delay(); + switch (status.InnerStatus) { + case TInnerStatus::StatusReceived: + if (status.YdbStatus != NYdb::EStatus::SUCCESS || !RetryMode || delay <= MaxDelay) { + ++Replies.Statuses[status.YdbStatus]; + } else { + ++Replies.CountHighLatency; + } + break; + case TInnerStatus::StatusApplicationTimeout: + ++Replies.ApplicationTimeout; + break; + case TInnerStatus::StatusNotFinished: + ++Replies.NotFinished; + break; + default: + Y_ABORT_UNLESS(status.InnerStatus); + break; + } + // Saving delay stats + unit.Report(status); +} + +void TStat::CheckCurrentSecond(TInstant now) { + std::uint64_t currSecond = now.Seconds(); + if (currSecond > CurrentSecond) { + CurrentSecond = currSecond; + ActivePeriod = std::make_shared( + this, + RetryMode, + MaxDelay, + currSecond, + Counters.Infly, + Counters.ActiveSessions, + Counters.ReadPromises, + Counters.ExecutorPromises + ); + } +} + +void TStat::PushMetricsData(const TPeriodStat& p) { + if (!PushMetrics) { + return; + } + auto threadFunc = [this, p]() { + MetricsPusher->PushData(p); + }; + if (!MetricsPushQueue.AddFunc(threadFunc)) { + Cerr << TInstant::Now().ToRfc822StringLocal() << ": Failed to push data to solomon" << Endl; + } +} + +void TStat::ResetMetricsPusher(std::uint64_t timestamp) { + while (timestamp >= TInstant::Now().Seconds()) { + Sleep(TDuration::Seconds(1)); + } + TPeriodStat pStat; + pStat.Seconds = timestamp; + PushMetricsData(pStat); +} + +std::string YdbStatusToString(NYdb::EStatus status) { + switch (status) { + case NYdb::EStatus::SUCCESS: + return "SUCCESS"; + case NYdb::EStatus::BAD_REQUEST: + return "BAD_REQUEST"; + case NYdb::EStatus::UNAUTHORIZED: + return "UNAUTHORIZED"; + case NYdb::EStatus::INTERNAL_ERROR: + return "INTERNAL_ERROR"; + case NYdb::EStatus::ABORTED: + return "ABORTED"; + case NYdb::EStatus::UNAVAILABLE: + return "UNAVAILABLE"; + case NYdb::EStatus::OVERLOADED: + return "OVERLOADED"; + case NYdb::EStatus::SCHEME_ERROR: + return "SCHEME_ERROR"; + case NYdb::EStatus::GENERIC_ERROR: + return "GENERIC_ERROR"; + case NYdb::EStatus::TIMEOUT: + return "TIMEOUT"; + case NYdb::EStatus::BAD_SESSION: + return "BAD_SESSION"; + case NYdb::EStatus::PRECONDITION_FAILED: + return "PRECONDITION_FAILED"; + case NYdb::EStatus::ALREADY_EXISTS: + return "ALREADY_EXISTS"; + case NYdb::EStatus::NOT_FOUND: + return "NOT_FOUND"; + case NYdb::EStatus::SESSION_EXPIRED: + return "SESSION_EXPIRED"; + case NYdb::EStatus::CANCELLED: + return "CANCELLED"; + case NYdb::EStatus::UNDETERMINED: + return "UNDETERMINED"; + case NYdb::EStatus::UNSUPPORTED: + return "UNSUPPORTED"; + case NYdb::EStatus::SESSION_BUSY: + return "SESSION_BUSY"; + case NYdb::EStatus::EXTERNAL_ERROR: + return "EXTERNAL_ERROR"; + case NYdb::EStatus::STATUS_UNDEFINED: + return "STATUS_UNDEFINED"; + case NYdb::EStatus::TRANSPORT_UNAVAILABLE: + return "TRANSPORT_UNAVAILABLE"; + case NYdb::EStatus::CLIENT_RESOURCE_EXHAUSTED: + return "CLIENT_RESOURCE_EXHAUSTED"; + case NYdb::EStatus::CLIENT_DEADLINE_EXCEEDED: + return "CLIENT_DEADLINE_EXCEEDED"; + case NYdb::EStatus::CLIENT_INTERNAL_ERROR: + return "CLIENT_INTERNAL_ERROR"; + case NYdb::EStatus::CLIENT_CANCELLED: + return "CLIENT_CANCELLED"; + case NYdb::EStatus::CLIENT_UNAUTHENTICATED: + return "CLIENT_UNAUTHENTICATED"; + case NYdb::EStatus::CLIENT_CALL_UNIMPLEMENTED: + return "CLIENT_CALL_UNIMPLEMENTED"; + case NYdb::EStatus::CLIENT_OUT_OF_RANGE: + return "CLIENT_OUT_OF_RANGE"; + case NYdb::EStatus::CLIENT_DISCOVERY_FAILED: + return "CLIENT_DISCOVERY_FAILED"; + case NYdb::EStatus::CLIENT_LIMITS_REACHED: + return "CLIENT_LIMITS_REACHED"; + } +} diff --git a/tests/slo_workloads/utils/statistics.h b/tests/slo_workloads/utils/statistics.h new file mode 100644 index 00000000000..e3c08ee6109 --- /dev/null +++ b/tests/slo_workloads/utils/statistics.h @@ -0,0 +1,200 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include + +inline std::string GetMillisecondsStr(const TDuration& d) { + return TStringBuilder() << d.MilliSeconds() << '.' << Sprintf("%03" PRIu64, d.MicroSeconds() % 1000); +} + +inline double GetMillisecondsDouble(const TDuration& d) { + return static_cast(d.MicroSeconds()) / 1000; +} + +using TFinalStatus = std::optional; +using TAsyncFinalStatus = NThreading::TFuture; + +struct TInnerStatus { + enum EInnerStatus { + StatusUnknown, + StatusReceived, + StatusApplicationTimeout, + StatusNotFinished + }; + + TInnerStatus(EInnerStatus innerStatus = StatusUnknown, NYdb::EStatus ydbStatus = NYdb::EStatus::STATUS_UNDEFINED) + : InnerStatus(innerStatus) + , YdbStatus(ydbStatus) + {} + + EInnerStatus InnerStatus; + NYdb::EStatus YdbStatus; +}; + +class TPeriodData; + +// Primitive latency unit +struct TStatUnit { + TStatUnit(const std::shared_ptr& periodData, TInstant startTime); + void Report(const TInnerStatus& status); + TDuration Delay() const; + + std::shared_ptr PeriodData; + TInstant Start; + TInstant End; +}; + +struct TCounters { + std::uint64_t Infly = 0; + std::uint64_t ActiveSessions = 0; + + // Debug use only: + std::uint64_t ReadPromises = 0; + std::uint64_t ExecutorPromises = 0; +}; + +struct TReplies { + std::map Statuses; + std::uint64_t CountMaxInfly = 0; + std::uint64_t CountHighLatency = 0; + std::uint64_t ApplicationTimeout = 0; + std::uint64_t NotFinished = 0; +}; + +// Calculated percentiles for a period of time +struct TPercentile { + TDuration P50; + TDuration P90; + TDuration P95; + TDuration P99; + TDuration P99_9; + TDuration P100; +}; + +// Accumulated statistics for a period of time +struct TPeriodStat { + std::uint64_t Seconds = 0; + TCounters Counters; + TReplies Replies; + TPercentile Oks; + TPercentile NotOks; +}; + +class TStat; + +// Full latency data for a period of time +class TPeriodData : public std::enable_shared_from_this { +public: + TPeriodData( + TStat* stats, + bool RetryMode, + const TDuration maxDelay, + std::uint64_t currentSecond, + std::uint64_t currInfly, + std::uint64_t currSessions, + std::uint64_t currPromises, + std::uint64_t currExecutorPromises + ); + ~TPeriodData(); + TStatUnit CreateStatUnit(TInstant startTime); + void AddStat(TDuration delay, const TInnerStatus& status); + std::uint64_t GetCurrentSecond() const; + void ReportMaxInfly(); + void ReportInfly(std::uint64_t infly); + void ReportActiveSessions(std::uint64_t sessions); + + // Debug use only: + void ReportReadPromises(std::uint64_t promises); + void ReportExecutorPromises(std::uint64_t promises); + +private: + TStat* Stats; + bool RetryMode; + std::uint64_t CurrentSecond; + std::vector OkDelays; + std::vector NotOkDelays; + TCounters Counters; + TReplies Replies; + TDuration MaxDelay; +}; + +class IMetricsPusher { +public: + virtual ~IMetricsPusher() = default; + virtual void PushData(const TPeriodStat& p) = 0; +}; + +class TStat { + friend class TPeriodData; +public: + TStat( + TDuration maxDelay, + const std::string& resultFileName, + bool pushMetrics, + bool retryMode + ); + ~TStat(); + void Reset(); + void Finish(); + void Flush(); + TStatUnit CreateStatUnit(); + void Report(TStatUnit& unit, const TInnerStatus& status); + void Report(TStatUnit& unit, TInnerStatus::EInnerStatus innerStatus); + void Report(TStatUnit& unit, const TFinalStatus& status); + void Report(TStatUnit& unit, NYdb::EStatus status); + void ReportMaxInfly(); + void ReportStats(std::uint64_t infly, std::uint64_t sessions, std::uint64_t readPromises, std::uint64_t executorPromises); + void Reserve(size_t size); + void ReportLatencyData( + std::uint64_t currSecond, + TCounters&& counters, + TReplies&& replies, + std::vector&& oks, + std::vector& notOks + ); + void UpdateSessionStats(const std::unordered_map& sessionStats); + + void PrintStatistics(TStringBuilder& out); + void SaveResult(); + + TInstant GetStartTime() const; + +private: + void OnReport(TStatUnit& unit, const TInnerStatus& status); + void CheckCurrentSecond(TInstant now); + void PushMetricsData(const TPeriodStat& p); + void ResetMetricsPusher(std::uint64_t timestamp); + void CalculateGlobalPercentile(); + void CalculateFailSeconds(); + std::uint64_t GetTotal(); + + std::recursive_mutex Mutex; + TThreadPool MetricsPushQueue; + // Current period we collect stats for + std::shared_ptr ActivePeriod; + std::vector LatencyStats; + std::vector> LatencyData; + TDuration MaxDelay; + TInstant StartTime; + TInstant FinishTime; + std::unordered_map SessionStats; + std::uint64_t CurrentSecond = 0; + // program lifetime + TCounters Counters; + TReplies Replies; + bool PushMetrics; + bool RetryMode; + std::string ResultFileName; + std::unique_ptr MetricsPusher; + + std::unique_ptr GlobalPercentile; + std::unique_ptr FailSeconds; +}; + +std::string YdbStatusToString(NYdb::EStatus status); diff --git a/tests/slo_workloads/utils/utils.cpp b/tests/slo_workloads/utils/utils.cpp new file mode 100644 index 00000000000..2388bf4caca --- /dev/null +++ b/tests/slo_workloads/utils/utils.cpp @@ -0,0 +1,457 @@ +#include "utils.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include + +using namespace NLastGetopt; +using namespace NYdb; + +const TDuration DefaultReactionTime = TDuration::Minutes(2); +const TDuration ReactionTimeDelay = TDuration::MilliSeconds(5); +const TDuration GlobalTimeout = TDuration::Minutes(2); +const std::uint64_t PartitionsCount = 64; + +Y_DECLARE_OUT_SPEC(, NYdb::TStatus, stream, value) { + stream << "Status: " << value.GetStatus() << Endl; + value.GetIssues().PrintTo(stream); +} + +TDurationMeter::TDurationMeter(TDuration& value) + : Value(value) + , StartTime(TInstant::Now()) +{ +} + +TDurationMeter::~TDurationMeter() { + Value += TInstant::Now() - StartTime; +} + +TRpsProvider::TRpsProvider(std::uint64_t rps) + : Rps(rps) + , Period(Max(TDuration::MilliSeconds(10), TDuration::MicroSeconds(1000000 / Rps))) + , ProcessedTime(TInstant::Now()) +{ +} + +void TRpsProvider::Reset() { + ProcessedTime = TInstant::Now() - Period - Period; +} + +void TRpsProvider::Use() { + if (Allowed) { + --Allowed; + return; + } + + while (!TryUse()) { + SleepUntil(TInstant::Now() + Period); + } +} + +bool TRpsProvider::TryUse() { + TInstant now = TInstant::Now(); + // Number of objects to process since ProcessedTime + Allowed = Rps * TDuration(now - ProcessedTime).MicroSeconds() / 1000000; + if (Allowed) { + ProcessedTime += TDuration::MicroSeconds(1000000 * Allowed / Rps); + --Allowed; + return true; + } else { + return false; + } +} + +std::uint64_t TRpsProvider::GetRps() const { + return Rps; +} + +bool ParseToken(std::string& token, std::string& tokenFile) { + if (!tokenFile.empty()) { + if (!token.empty()) { + Cerr << "Both token and token_file provided. Choose one." << Endl; + } else { + TFsPath path(tokenFile); + if (path.Exists()) { + token = Strip(TUnbufferedFileInput(path).ReadAll()); + return true; + } + Cerr << "Wrong path provided for token_file." << Endl; + } + } else if (!token.empty()) { + return true; + } else { + token = GetEnv("YDB_TOKEN"); + return true; + } + return false; +} + +void StartStatCollecting([[maybe_unused]] TDriver& driver, const std::string& statConfigFile) { + if (statConfigFile.empty()) { + return; + } + + // TODO: Implement +} + +std::string GetDatabase(const std::string& connectionString) { + constexpr std::string_view databaseFlag = "/?database="; + size_t pathIndex = connectionString.find(databaseFlag); + if (pathIndex != std::string::npos) { + return connectionString.substr(pathIndex + databaseFlag.size()); + } + return {}; +} + +int DoMain(int argc, char** argv, TCreateCommand create, TRunCommand run, TCleanupCommand cleanup) { + TOpts opts = TOpts::Default(); + + std::string connectionString; + std::string prefix; + std::string token; + std::string tokenFile; + std::string iamSaKeyFile; + std::string statConfigFile; + std::string balancingPolicy; + + opts.AddLongOption('c', "connection-string", "YDB connection string").Required().RequiredArgument("SCHEMA://HOST:PORT/?DATABASE=DATABASE") + .StoreResult(&connectionString); + opts.AddLongOption('p', "prefix", "Base prefix for tables").RequiredArgument("PATH") + .StoreResult(&prefix); + opts.AddLongOption('k', "token", "security token").RequiredArgument("TOKEN") + .StoreResult(&token); + opts.AddLongOption('f', "token-file", "security token file").RequiredArgument("PATH") + .StoreResult(&tokenFile); + opts.AddLongOption("iam-sa-key-file", "IAM service account key file").RequiredArgument("SECRET") + .StoreResult(&iamSaKeyFile); + opts.AddLongOption('s', "stat-config", "statistics config file").Optional().RequiredArgument("PATH") + .StoreResult(&statConfigFile); + opts.AddLongOption('b', "balancing-policy", "Balancing policy").Optional().DefaultValue("use-all-nodes").RequiredArgument("(use-all-nodes|prefer-local-dc|prefer-primary-pile)") + .StoreResult(&balancingPolicy); + opts.AddHelpOption('h'); + opts.SetFreeArgsMin(1); + opts.SetFreeArgTitle(0, "", GetCmdList()); + opts.ArgPermutation_ = NLastGetopt::REQUIRE_ORDER; + + TOptsParseResult res(&opts, argc, argv); + size_t freeArgsPos = res.GetFreeArgsPos(); + argc -= freeArgsPos; + argv += freeArgsPos; + ECommandType command = ParseCommand(*argv); + if (command == ECommandType::Unknown) { + Cerr << "Unknown command '" << *argv << "'" << Endl; + return EXIT_FAILURE; + } + + if (prefix.empty()) { + prefix = GetDatabase(connectionString); + } + + if (!ParseToken(token, tokenFile)) { + return EXIT_FAILURE; + } + + auto config = TDriverConfig(connectionString); + + if (!iamSaKeyFile.empty()) { + Cout << "Enabling IAM authentication..." << Endl; + TIamJwtFilename iamJwtFilename{ .JwtFilename = iamSaKeyFile }; + config.SetCredentialsProviderFactory(CreateIamJwtFileCredentialsProviderFactory(iamJwtFilename)); + } else if (!token.empty()) { + Cout << "Enabling OAuth authentication..." << Endl; + config.SetCredentialsProviderFactory(CreateOAuthCredentialsProviderFactory(token)); + } else { + Cerr << "Warning: No authentication methods provided." << Endl; + } + + if (balancingPolicy == "use-all-nodes") { + config.SetBalancingPolicy(TBalancingPolicy::UseAllNodes()); + } else if (balancingPolicy == "prefer-local-dc") { + config.SetBalancingPolicy(TBalancingPolicy::UsePreferableLocation()); + } else if (balancingPolicy == "prefer-primary-pile") { + config.SetBalancingPolicy(TBalancingPolicy::UsePreferablePileState()); + } else { + Cerr << "Unknown balancing policy: " << balancingPolicy << Endl; + return EXIT_FAILURE; + } + + TDriver driver(config); + + StartStatCollecting(driver, statConfigFile); + + TDatabaseOptions dbOptions{ driver, prefix }; + int result; + try { + switch (command) { + case ECommandType::Create: + Cout << "Launching create command..." << Endl; + result = create(dbOptions, argc, argv); + break; + case ECommandType::Run: + Cout << "Launching run command..." << Endl; + result = run(dbOptions, argc, argv); + break; + case ECommandType::Cleanup: + Cout << "Launching cleanup command..." << Endl; + result = cleanup(dbOptions, argc); + break; + default: + Cerr << "Unknown command" << Endl; + return EXIT_FAILURE; + } + } + catch (const NYdb::NStatusHelpers::TYdbErrorException& e) { + Cerr << "Exception caught: " << e << Endl; + return EXIT_FAILURE; + } + driver.Stop(true); + return result; +} + +std::string GetCmdList() { + return "create, run, cleanup"; +} + +ECommandType ParseCommand(const char* cmd) { + if (!strcmp(cmd, "create")) { + return ECommandType::Create; + } + if (!strcmp(cmd, "run")) { + return ECommandType::Run; + } + if (!strcmp(cmd, "cleanup")) { + return ECommandType::Cleanup; + } + return ECommandType::Unknown; +} + +std::string JoinPath(const std::string& prefix, const std::string& path) { + if (prefix.empty()) { + return path; + } + + TPathSplitUnix prefixPathSplit(prefix); + prefixPathSplit.AppendComponent(path); + + return prefixPathSplit.Reconstruct(); +} + +std::string GenerateRandomString(std::uint32_t minLength, std::uint32_t maxLength) { + std::uint32_t length = minLength + RandomNumber() % (maxLength - minLength); + static const char* symbols = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::string result; + result.reserve(length); + for (size_t i = 0; i < length; ++i) { + result.push_back(symbols[RandomNumber(61)]); + } + return result; +} + +using namespace NYdb; +using namespace NYdb::NTable; + +TParams PackValuesToParamsAsList(const std::vector& items, const std::string name) { + TValueBuilder itemsAsList; + itemsAsList.BeginList(); + for (const TValue& item : items) { + itemsAsList.AddListItem(item); + } + itemsAsList.EndList(); + + TParamsBuilder paramsBuilder; + paramsBuilder.AddParam(name, itemsAsList.Build()); + return paramsBuilder.Build(); +} + +static double shardSize = (static_cast(Max()) + 1) / PartitionsCount; + +std::uint32_t GetSpecialId(std::uint32_t id) { + return static_cast(id / shardSize) * shardSize + 1; +} + +std::uint32_t GetShardSpecialId(std::uint64_t shardNo) { + return shardNo * shardSize + 1; +} + +std::uint32_t GetHash(std::uint32_t value) { + std::uint32_t result = NumericHash(value); + if (result == GetSpecialId(result)) { + ++result; + } + return result; +} + +TTableStats GetTableStats(TDatabaseOptions& dbOptions, const std::string& tableName) { + Cout << TInstant::Now().ToRfc822StringLocal() + << " Getting table stats (maxId and count of rows) with ReadTable... " << Endl; + TInstant start_time = TInstant::Now(); + NYdb::NTable::TTableClient client( + dbOptions.Driver, + NYdb::NTable::TClientSettings() + .MinSessionCV(8) + .AllowRequestMigration(true) + ); + + std::optional tableIterator; + NYdb::NStatusHelpers::ThrowOnError(client.RetryOperationSync([&tableIterator, &dbOptions, &tableName](TSession session) { + auto result = session.ReadTable( + JoinPath(dbOptions.Prefix, tableName), + TReadTableSettings().AppendColumns("object_id") + ).GetValueSync(); + + if (result.IsSuccess()) { + tableIterator = result; + } + + return result; + })); + Y_ENSURE(tableIterator); + TSimpleThreadPool pool; + std::vector> futures; + pool.Start(10); + for (;;) { + auto tablePart = tableIterator->ReadNext().GetValueSync(); + if (!tablePart.IsSuccess()) { + if (tablePart.EOS()) { + break; + } + + NYdb::NStatusHelpers::ThrowOnError(tablePart); + } + futures.push_back( + NThreading::Async( + [extractedPart = tablePart.ExtractPart()]{ + auto rsParser = TResultSetParser(extractedPart); + std::uint32_t partMax = 0; + while (rsParser.TryNextRow()) { + auto& idParser = rsParser.ColumnParser("object_id"); + idParser.OpenOptional(); + std::uint32_t id = idParser.GetUint32(); + if (id > partMax) { + partMax = id; + } + } + return TTableStats{ rsParser.RowsCount(), partMax }; + }, + pool + ) + ); + } + TTableStats result; + for (auto future : futures) { + TTableStats partStats = future.GetValueSync(); + if (partStats.MaxId > result.MaxId) { + result.MaxId = partStats.MaxId; + } + result.RowCount += partStats.RowCount; + } + Cout << TInstant::Now().ToRfc822StringLocal() << " Done. maxId=" << result.MaxId << ", row count=" << result.RowCount + << ". Calculations took " << TInstant::Now() - start_time << Endl; + return result; +} + +void ParseOptionsCommon(TOpts& opts, TCommonOptions& options, bool followers) { + opts.AddLongOption("threads", "Number of threads to use").RequiredArgument("NUM") + .DefaultValue(options.MaxInputThreads).StoreResult(&options.MaxInputThreads); + opts.AddLongOption("stop_on_error", "Stop thread if an error occured").NoArgument() + .SetFlag(&options.StopOnError).DefaultValue(options.StopOnError); + opts.AddLongOption("payload-min", "Minimum length of payload string").RequiredArgument("NUM") + .DefaultValue(options.MinLength).StoreResult(&options.MinLength); + opts.AddLongOption("payload-max", "Maximum length of payload string").RequiredArgument("NUM") + .DefaultValue(options.MaxLength).StoreResult(&options.MaxLength); + opts.AddLongOption("timeout", "Read requests execution timeout [ms]").RequiredArgument("NUM") + .DefaultValue(options.A_ReactionTime).StoreResult(&options.A_ReactionTime); + opts.AddLongOption("dont-push", "Do not push metrics").NoArgument() + .SetFlag(&options.DontPushMetrics).DefaultValue(options.DontPushMetrics); + opts.AddLongOption("retry", "Retry each request until Ok reply or global timeout").NoArgument() + .SetFlag(&options.RetryMode).DefaultValue(options.RetryMode); + opts.AddLongOption("save-result", "Save result to file").NoArgument() + .SetFlag(&options.SaveResult).DefaultValue(options.SaveResult); + opts.AddLongOption("result-file-name", "Set result json file name").RequiredArgument("String") + .DefaultValue(options.ResultFileName).StoreResult(&options.ResultFileName); + opts.AddLongOption("app-timeout", "Use application timeout (over SDK)").NoArgument() + .SetFlag(&options.UseApplicationTimeout).DefaultValue(options.UseApplicationTimeout); + opts.AddLongOption("prevention-request", "Send prevention request at 1/2 of timeout").NoArgument() + .SetFlag(&options.SendPreventiveRequest).DefaultValue(options.SendPreventiveRequest); + if (followers) { + opts.AddLongOption("followers", "Use followers").NoArgument() + .SetFlag(&options.UseFollowers).DefaultValue(options.UseFollowers); + } +} + +bool CheckOptionsCommon(TCommonOptions& options) { + if (options.MinLength > options.MaxLength) { + Cerr << "--payload-min should be less than --payload-max" << Endl; + return false; + } + if (!options.MaxInputThreads) { + Cerr << "--threads should be more than 0" << Endl; + return false; + } + if (!options.DontPushMetrics) { + Cerr << "Push metrics is not supported yet" << Endl; + return false; + } + return true; +} + +bool ParseOptionsCreate(int argc, char** argv, TCreateOptions& createOptions, bool followers) { + TOpts opts = TOpts::Default(); + ParseOptionsCommon(opts, createOptions.CommonOptions, followers); + opts.AddLongOption("count", "Total number of records to generate").RequiredArgument("NUM") + .DefaultValue(createOptions.Count).StoreResult(&createOptions.Count); + opts.AddLongOption("pack-size", "Number of new records in each create request").RequiredArgument("NUM") + .DefaultValue(createOptions.PackSize).StoreResult(&createOptions.PackSize); + + TOptsParseResult res(&opts, argc, argv); + + if (!CheckOptionsCommon(createOptions.CommonOptions)) { + return false; + } + if (!createOptions.Count) { + Cerr << "--count should be more than 0" << Endl; + return false; + } + if (!createOptions.PackSize) { + Cerr << "--pack-size should be more than 0" << Endl; + return false; + } + return true; +} + +bool ParseOptionsRun(int argc, char** argv, TRunOptions& runOptions, bool followers) { + TOpts opts = TOpts::Default(); + ParseOptionsCommon(opts, runOptions.CommonOptions, followers); + opts.AddLongOption("time", "Time to run (Seconds)").RequiredArgument("Seconds") + .DefaultValue(runOptions.CommonOptions.SecondsToRun).StoreResult(&runOptions.CommonOptions.SecondsToRun); + opts.AddLongOption("read-rps", "Request generation rate for read requests (Thread A)").RequiredArgument("NUM") + .DefaultValue(runOptions.Read_rps).StoreResult(&runOptions.Read_rps); + opts.AddLongOption("write-rps", "Request generation rate for write requests (Thread B)").RequiredArgument("NUM") + .DefaultValue(runOptions.Write_rps).StoreResult(&runOptions.Write_rps); + opts.AddLongOption("no-read", "Do not run reading requests (thread A)").NoArgument() + .SetFlag(&runOptions.DontRunA).DefaultValue(runOptions.DontRunA); + opts.AddLongOption("no-write", "Do not run writing requests (thread B)").NoArgument() + .SetFlag(&runOptions.DontRunB).DefaultValue(runOptions.DontRunB); + opts.AddLongOption("no-c", "Do not run thread C").NoArgument() + .SetFlag(&runOptions.DontRunC).DefaultValue(runOptions.DontRunC); + opts.AddLongOption("infly", "Maximum number of running jobs").RequiredArgument("NUM") + .DefaultValue(runOptions.CommonOptions.MaxInfly).StoreResult(&runOptions.CommonOptions.MaxInfly); + TOptsParseResult res(&opts, argc, argv); + + if (!CheckOptionsCommon(runOptions.CommonOptions)) { + return false; + } + if (!runOptions.CommonOptions.SecondsToRun) { + Cerr << "Time to run should be more than 0" << Endl; + return false; + } + return true; +} diff --git a/tests/slo_workloads/utils/utils.h b/tests/slo_workloads/utils/utils.h new file mode 100644 index 00000000000..32b1ea3b9cb --- /dev/null +++ b/tests/slo_workloads/utils/utils.h @@ -0,0 +1,165 @@ +#pragma once + +#include "statistics.h" + +#include +#include +#include + +#include + +#include + +extern const TDuration DefaultReactionTime; +extern const TDuration ReactionTimeDelay; +extern const TDuration GlobalTimeout; +extern const std::uint64_t PartitionsCount; + +struct TRecordData { + std::uint32_t ObjectId; + std::uint64_t Timestamp; + std::string Guid; + std::string Payload; +}; + +struct TKeyValueRecordData { + std::uint32_t ObjectId; + std::uint64_t Timestamp; + std::string Payload; +}; + +struct TDurationMeter { + TDurationMeter(TDuration& value); + ~TDurationMeter(); + + TDuration& Value; + TInstant StartTime; +}; + +struct TDatabaseOptions { + NYdb::TDriver& Driver; + const std::string& Prefix; +}; + +struct TCommonOptions { + // Executor options: + TDatabaseOptions DatabaseOptions; + std::uint32_t SecondsToRun = 10; + std::uint32_t Rps = 10; + std::uint32_t MaxInputThreads = 50; + std::uint32_t MaxCallbackThreads = 50; + std::uint32_t MaxInfly = 500; + std::uint32_t MaxRetries = 50; + std::uint64_t A_ReactionTime = 70; //ms + TDuration ReactionTime = DefaultReactionTime; + bool StopOnError = false; + bool UseApplicationTimeout = false; + bool SendPreventiveRequest = false; + + //Generator options: + std::uint32_t MinLength = 20; + std::uint32_t MaxLength = 200; + + //Output options: + bool DontPushMetrics = true; + std::string ResultFileName = "slo_result.json"; + + bool UseFollowers = false; + bool RetryMode = false; + bool SaveResult = false; +}; + +struct TCreateOptions { + TCommonOptions CommonOptions; + std::uint32_t Count = 10000; + std::uint32_t PackSize = 100; +}; + +struct TRunOptions { + TCommonOptions CommonOptions; + bool DontRunA = false; + bool DontRunB = false; + bool DontRunC = false; + std::uint32_t Read_rps = 1000; + std::uint32_t Write_rps = 10; +}; + +class TRpsProvider { +public: + TRpsProvider(std::uint64_t rps); + void Reset(); + void Use(); + bool TryUse(); + std::uint64_t GetRps() const; + +private: + std::uint64_t Rps; + TDuration Period; + TInstant ProcessedTime; + TInstant LastCheck; + std::uint32_t Allowed = 0; + TInstant StartTime; +}; + +enum class ECommandType { + Unknown, + Create, + Run, + Cleanup +}; + +struct TTableStats { + std::uint64_t RowCount = 0; + std::uint32_t MaxId = 0; +}; + +using TCreateCommand = std::function; +using TRunCommand = std::function; +using TCleanupCommand = std::function; + +int DoMain(int argc, char** argv, TCreateCommand create, TRunCommand run, TCleanupCommand cleanup); + +std::string GetCmdList(); +ECommandType ParseCommand(const char* cmd); + +std::string JoinPath(const std::string& prefix, const std::string& path); + +inline void RetryBackoff( + NYdb::NTable::TTableClient& client, + std::uint32_t retries, + const NYdb::NTable::TTableClient::TOperationSyncFunc& func +) { + TDuration delay = TDuration::Seconds(5); + while (retries) { + NYdb::TStatus status = client.RetryOperationSync(func); + if (status.IsSuccess()) { + return; + } + --retries; + if (!retries) { + Cerr << "Create request failed after all retries." << Endl; + Cerr << status << Endl; + NYdb::NStatusHelpers::ThrowOnError(status); + } + Cerr << "Create request failed. Sleeping for " << delay << Endl; + Sleep(delay); + delay *= 2; + } +} + +std::string GenerateRandomString(std::uint32_t minLength, std::uint32_t maxLength); + +NYdb::TParams PackValuesToParamsAsList(const std::vector& items, const std::string name = "$items"); + +// Returns special object_id within the same shard as given id +std::uint32_t GetSpecialId(std::uint32_t id); + +// Returns special object_id for given shard +std::uint32_t GetShardSpecialId(std::uint64_t shardNo); + +std::uint32_t GetHash(std::uint32_t value); + +TTableStats GetTableStats(TDatabaseOptions& dbOptions, const std::string& tableName); + +bool ParseOptionsCreate(int argc, char** argv, TCreateOptions& createOptions, bool followers = false); +bool ParseOptionsRun(int argc, char** argv, TRunOptions& runOptions, bool followers = false); diff --git a/tests/unit/client/CMakeLists.txt b/tests/unit/client/CMakeLists.txt index 33752c8a6ee..8c3b142ee74 100644 --- a/tests/unit/client/CMakeLists.txt +++ b/tests/unit/client/CMakeLists.txt @@ -54,7 +54,7 @@ add_ydb_test(NAME client-oauth2_ut http-server json string_utils-base64 - client-ydb_types-credentials-oauth2 + client-types-credentials-oauth2 client-oauth2-ut-helpers LABELS unit diff --git a/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp b/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp index 496c371b240..04d47619998 100644 --- a/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp +++ b/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp @@ -1006,7 +1006,7 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { .Build() .Build() ); -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS server.Check.ExpectedErrorPart = "failed to load key"; #else server.Check.ExpectedErrorPart = "failed to load private key"; @@ -1034,7 +1034,7 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { .Build() ); -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS server.Check.ExpectedErrorPart = "invalid key size"; #else server.Check.ExpectedErrorPart = "failed to load private key"; @@ -1050,7 +1050,7 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { .Build() ); -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS server.Check.ExpectedErrorPart = "failed to load key"; #else server.Check.ExpectedErrorPart = "failed to load private key"; diff --git a/tests/unit/client/oauth2_token_exchange/helpers/CMakeLists.txt b/tests/unit/client/oauth2_token_exchange/helpers/CMakeLists.txt index 486fa605abe..c54f51a3ba6 100644 --- a/tests/unit/client/oauth2_token_exchange/helpers/CMakeLists.txt +++ b/tests/unit/client/oauth2_token_exchange/helpers/CMakeLists.txt @@ -13,5 +13,5 @@ target_link_libraries(client-oauth2-ut-helpers http-server json string_utils-base64 - client-ydb_types-credentials-oauth2 + client-types-credentials-oauth2 ) diff --git a/tests/unit/client/oauth2_token_exchange/helpers/jwt_check_helper.h b/tests/unit/client/oauth2_token_exchange/helpers/jwt_check_helper.h index 3b23e2fbe2d..c41c5afe8a1 100644 --- a/tests/unit/client/oauth2_token_exchange/helpers/jwt_check_helper.h +++ b/tests/unit/client/oauth2_token_exchange/helpers/jwt_check_helper.h @@ -8,7 +8,7 @@ extern const std::string TestRSAPublicKeyContent; struct TJwtCheck { using TSelf = TJwtCheck; -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS using TDecodedJwt = jwt::decoded_jwt; #else using TDecodedJwt = jwt::decoded_jwt; @@ -29,7 +29,7 @@ struct TJwtCheck { UNIT_ASSERT_VALUES_EQUAL(decoded.get_algorithm(), Alg.name()); const std::string data = decoded.get_header_base64() + "." + decoded.get_payload_base64(); const std::string signature = decoded.get_signature(); -#ifdef YDB_SDK_USE_NEW_JWT +#ifdef YDB_SDK_OSS std::error_code ec; Alg.verify(data, signature, ec); if (ec) {