diff --git a/ydb/core/blobstorage/base/blobstorage_console_events.h b/ydb/core/blobstorage/base/blobstorage_console_events.h index 37efc732f500..47d3932fa2af 100644 --- a/ydb/core/blobstorage/base/blobstorage_console_events.h +++ b/ydb/core/blobstorage/base/blobstorage_console_events.h @@ -32,8 +32,14 @@ namespace NKikimr { NKikimrBlobStorage::TEvControllerConsoleCommitRequest, EvControllerConsoleCommitRequest> { TEvControllerConsoleCommitRequest() = default; - TEvControllerConsoleCommitRequest(const TString& yamlConfig) { + TEvControllerConsoleCommitRequest( + const TString& yamlConfig, + bool allowUnknownFields = false, + bool bypassMetadataChecks = false) { + Record.SetYAML(yamlConfig); + Record.SetAllowUnknownFields(allowUnknownFields); + Record.SetBypassMetadataChecks(bypassMetadataChecks); } TString ToString() const override { @@ -76,8 +82,14 @@ namespace NKikimr { NKikimrBlobStorage::TEvControllerReplaceConfigRequest, EvControllerReplaceConfigRequest> { TEvControllerReplaceConfigRequest() = default; - TEvControllerReplaceConfigRequest(std::optional clusterYaml, std::optional storageYaml, - std::optional switchDedicatedStorageSection, bool dedicatedConfigMode) { + TEvControllerReplaceConfigRequest( + std::optional clusterYaml, + std::optional storageYaml, + std::optional switchDedicatedStorageSection, + bool dedicatedConfigMode, + bool allowUnknownFields, + bool bypassMetadataChecks) { + if (clusterYaml) { Record.SetClusterYaml(*clusterYaml); } @@ -88,6 +100,8 @@ namespace NKikimr { Record.SetSwitchDedicatedStorageSection(*switchDedicatedStorageSection); } Record.SetDedicatedConfigMode(dedicatedConfigMode); + Record.SetAllowUnknownFields(allowUnknownFields); + Record.SetBypassMetadataChecks(bypassMetadataChecks); } TString ToString() const override { diff --git a/ydb/core/blobstorage/nodewarden/distconf_console.cpp b/ydb/core/blobstorage/nodewarden/distconf_console.cpp index 3f6b712e913d..ee125a8a53bb 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_console.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_console.cpp @@ -103,7 +103,8 @@ namespace NKikimr::NStorage { } NTabletPipe::SendData(SelfId(), ConsolePipeId, new TEvBlobStorage::TEvControllerConsoleCommitRequest( - MainConfigYaml), ++CommitRequestCookie); + MainConfigYaml), // FIXME: probably should propagate force here + ++CommitRequestCookie); break; } diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index dab31011e1b1..67815c3b23c1 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -113,6 +113,8 @@ void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { } } catch (const yexception &e) { opCtx.Error = e.what(); + } catch (const std::exception& e) { + opCtx.Error = e.what(); } } diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index b2ec3f533b8e..ad89f4d8008a 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -16,17 +16,35 @@ namespace NKikimr::NConsole { class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped { public: - TConsoleCommitActor(TActorId senderId, const TString& mainYamlConfig, TActorId interconnectSession, ui64 cookie) + TConsoleCommitActor( + TActorId senderId, + const TString& mainYamlConfig, + bool allowUnknownFields, + bool bypassMetadataChecks, + TActorId interconnectSession, + ui64 cookie) : SenderId(senderId) , MainYamlConfig(mainYamlConfig) + , AllowUnknownFields(allowUnknownFields) + , BypassMetadataChecks(bypassMetadataChecks) , InterconnectSession(interconnectSession) , Cookie(cookie) {} void Bootstrap(const TActorId& consoleId) { - auto request = std::make_unique(); - request->Record.MutableRequest()->set_config(MainYamlConfig); - Send(consoleId, request.release()); + auto executeRequest = [&](auto& request) { + request->Record.MutableRequest()->set_config(MainYamlConfig); + request->Record.MutableRequest()->set_allow_unknown_fields(AllowUnknownFields); + Send(consoleId, request.release()); + }; + + if (BypassMetadataChecks) { + auto request = std::make_unique(); + executeRequest(request); + } else { + auto request = std::make_unique(); + executeRequest(request); + } Become(&TThis::StateWork); } @@ -38,6 +56,13 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped(); + response->Record.SetStatus(NKikimrBlobStorage::TEvControllerConsoleCommitResponse::Committed); + SendInReply(std::move(response)); + PassAway(); + } + void Handle(TEvConsole::TEvGenericError::TPtr& ev) { auto response = std::make_unique(); response->Record.SetStatus(NKikimrBlobStorage::TEvControllerConsoleCommitResponse::NotCommitted); @@ -48,6 +73,7 @@ class TConfigsManager::TConsoleCommitActor : public TActorBootstrapped(); - const auto& mainYamlConfig = ev->Get()->Record.GetYAML(); + auto& record = ev->Get()->Record; + const auto& mainYamlConfig = record.GetYAML(); + bool allowUnknownFields = record.GetAllowUnknownFields(); + bool bypassMetadataChecks = record.GetBypassMetadataChecks(); if (!CheckSession(*ev, response, NKikimrBlobStorage::TEvControllerConsoleCommitResponse::SessionMismatch)) { return; } - IActor* actor = new TConsoleCommitActor(ev->Sender, mainYamlConfig, ev->InterconnectSession, ev->Cookie); + IActor* actor = new TConsoleCommitActor( + ev->Sender, + mainYamlConfig, + allowUnknownFields, + bypassMetadataChecks, + ev->InterconnectSession, + ev->Cookie); CommitActor = Register(actor); } @@ -133,22 +170,26 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerValidateConfigRequest: } auto& record = response->Record; + + bool bypassMetadataChecks = ev->Get()->Record.GetBypassMetadataChecks(); + auto mainYamlConfig = ev->Get()->Record.GetYAML(); TUpdateConfigOpContext opCtx; - ReplaceMainConfigMetadata(mainYamlConfig, false, opCtx); + ReplaceMainConfigMetadata(mainYamlConfig, bypassMetadataChecks, opCtx); ValidateMainConfig(opCtx); + bool hasForbiddenUnknownFields = !opCtx.UnknownFields.empty() && !ev->Get()->Record.GetAllowUnknownFields(); - if (opCtx.Error || !opCtx.UnknownFields.empty()) { + if (opCtx.Error || hasForbiddenUnknownFields) { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigNotValid); TStringStream s; if (opCtx.Error) { s << *opCtx.Error << (opCtx.UnknownFields.empty() ? "" : " and "); } - if (!opCtx.UnknownFields.empty()) { + if (hasForbiddenUnknownFields) { s << "has forbidden unknown fields"; } - record.SetErrorReason(s.Str()); + record.SetErrorReason(s.Str()); // TODO get warnings back } else { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigIsValid); record.SetYAML(opCtx.UpdatedConfig); diff --git a/ydb/core/cms/ui/yaml_config.js b/ydb/core/cms/ui/yaml_config.js index 37f1625211cb..69a05ce616f0 100644 --- a/ydb/core/cms/ui/yaml_config.js +++ b/ydb/core/cms/ui/yaml_config.js @@ -235,16 +235,16 @@ class YamlConfigState { onYamlLoaded(success, data) { if (success) { - this.cluster = data.Response.identity.cluster; - $('#yaml-cluster').text(data.Response.identity.cluster); + this.cluster = data.Response.identity[0].cluster; + $('#yaml-cluster').text(data.Response.identity[0].cluster); - this.version = data.Response.identity.version; - $('#yaml-version').text(data.Response.identity.version); + this.version = data.Response.identity[0].version; + $('#yaml-version').text(data.Response.identity[0].version); - if (this.config !== data.Response.config) { - this.codeMirror.setValue((data.Response.config !== undefined) ? data.Response.config[0] : ""); + if (this.config !== data.Response.config[0]) { + this.codeMirror.setValue((data.Response.config[0] !== undefined) ? data.Response.config[0] : ""); this.codeMirror.trigger('fold', 'editor.foldLevel2'); - this.config = data.Response.config; + this.config = data.Response.config[0]; } if (data.Response.volatile_configs === undefined) { diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index fc111a854c1e..96e4a9d9ee28 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -99,7 +99,7 @@ #include #include -#include +#include #include #include #include @@ -620,8 +620,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["tablet_service"] = &hasTabletService; TServiceCfg hasView = services.empty(); names["view"] = &hasView; - TServiceCfg hasBSConfig = services.empty(); - names["bsconfig"] = &hasBSConfig; + TServiceCfg hasConfig = services.empty(); + names["config"] = &hasConfig; std::unordered_set enabled; for (const auto& name : services) { @@ -915,8 +915,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { grpcRequestProxies[0], hasView.IsRlAllowed())); } - if (hasBSConfig) { - server.AddService(new NGRpcService::TBSConfigGRpcService(ActorSystem.Get(), Counters, grpcRequestProxies[0])); + if (hasConfig) { + server.AddService(new NGRpcService::TConfigGRpcService(ActorSystem.Get(), Counters, grpcRequestProxies[0])); } if (ModuleFactories) { diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 421659f5d94c..eeba273c0964 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -36,17 +36,7 @@ SRCS( PEERDIR( contrib/libs/protobuf - ydb/library/actors/core - ydb/library/actors/dnsresolver - ydb/library/actors/interconnect - ydb/library/actors/memory_log - ydb/library/actors/prof - ydb/library/actors/protos - ydb/library/actors/util library/cpp/getopt/small - ydb/public/sdk/cpp/src/library/grpc/client - ydb/library/grpc/server - ydb/library/grpc/server/actors library/cpp/logger library/cpp/malloc/api library/cpp/messagebus @@ -57,8 +47,8 @@ PEERDIR( library/cpp/svnversion ydb/core/actorlib_impl ydb/core/audit - ydb/core/base ydb/core/backup/controller + ydb/core/base ydb/core/blob_depot ydb/core/blobstorage ydb/core/blobstorage/backpressure @@ -86,10 +76,10 @@ PEERDIR( ydb/core/health_check ydb/core/http_proxy ydb/core/jaeger_tracing + ydb/core/kafka_proxy ydb/core/kesus/proxy ydb/core/kesus/tablet ydb/core/keyvalue - ydb/core/kafka_proxy ydb/core/kqp ydb/core/kqp/federated_query ydb/core/kqp/finalize_script_service @@ -124,11 +114,11 @@ PEERDIR( ydb/core/tracing ydb/core/tx ydb/core/tx/columnshard - ydb/core/tx/coordinator ydb/core/tx/conveyor/service - ydb/core/tx/limiter/service - ydb/core/tx/limiter/grouped_memory/usage + ydb/core/tx/coordinator ydb/core/tx/datashard + ydb/core/tx/limiter/grouped_memory/usage + ydb/core/tx/limiter/service ydb/core/tx/long_tx_service ydb/core/tx/long_tx_service/public ydb/core/tx/mediator @@ -145,44 +135,54 @@ PEERDIR( ydb/core/viewer ydb/core/ymq/actor ydb/core/ymq/http + ydb/library/actors/core + ydb/library/actors/dnsresolver + ydb/library/actors/interconnect + ydb/library/actors/memory_log + ydb/library/actors/prof + ydb/library/actors/protos + ydb/library/actors/util ydb/library/folder_service ydb/library/folder_service/proto + ydb/library/grpc/server + ydb/library/grpc/server/actors ydb/library/pdisk_io ydb/library/security - yql/essentials/minikql/comp_nodes/llvm14 - yt/yql/providers/yt/codec/codegen - yt/yql/providers/yt/comp_nodes/llvm14 - yt/yql/providers/yt/comp_nodes/dq/llvm14 ydb/library/yql/providers/pq/cm_client ydb/library/yql/providers/s3/actors - yql/essentials/public/udf/service/exception_policy ydb/public/lib/base ydb/public/lib/deprecated/client + ydb/public/sdk/cpp/src/library/grpc/client ydb/services/auth ydb/services/backup - ydb/services/bsconfig ydb/services/cms - ydb/services/dynamic_config + ydb/services/config ydb/services/datastreams + ydb/services/deprecated/persqueue_v0 ydb/services/discovery + ydb/services/dynamic_config + ydb/services/ext_index/metadata + ydb/services/ext_index/service ydb/services/fq ydb/services/kesus ydb/services/keyvalue ydb/services/local_discovery ydb/services/maintenance - ydb/services/metadata/ds_table ydb/services/metadata - ydb/services/ext_index/service - ydb/services/ext_index/metadata + ydb/services/metadata/ds_table ydb/services/monitoring ydb/services/persqueue_cluster_discovery - ydb/services/deprecated/persqueue_v0 ydb/services/persqueue_v1 ydb/services/rate_limiter ydb/services/replication ydb/services/tablet ydb/services/view ydb/services/ydb + yql/essentials/minikql/comp_nodes/llvm14 + yql/essentials/public/udf/service/exception_policy + yt/yql/providers/yt/codec/codegen + yt/yql/providers/yt/comp_nodes/dq/llvm14 + yt/yql/providers/yt/comp_nodes/llvm14 ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/grpc_services/rpc_bsconfig.cpp b/ydb/core/grpc_services/rpc_config.cpp similarity index 52% rename from ydb/core/grpc_services/rpc_bsconfig.cpp rename to ydb/core/grpc_services/rpc_config.cpp index 7583763bd4c3..2f4f8ce81cea 100644 --- a/ydb/core/grpc_services/rpc_bsconfig.cpp +++ b/ydb/core/grpc_services/rpc_config.cpp @@ -1,7 +1,9 @@ #include "service_keyvalue.h" #include #include -#include "rpc_bsconfig_base.h" +#include +#include +#include "rpc_config_base.h" #include #include @@ -12,33 +14,79 @@ namespace NKikimr::NGRpcService { using TEvReplaceStorageConfigRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using TEvFetchStorageConfigRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using TEvBootstrapClusterRequest = - TGrpcRequestOperationCall; + TGrpcRequestOperationCall; using namespace NActors; using namespace Ydb; -bool CopyToConfigRequest(const Ydb::BSConfig::ReplaceStorageConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { - to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(from.yaml_config())); +struct BSConfigApiShim { + std::optional SwitchDedicatedStorageSection; + std::optional MainConfig; + std::optional StorageConfig; + bool DedicatedConfigMode = false; +}; + +BSConfigApiShim ConvertConfigReplaceRequest(const auto& request) { + BSConfigApiShim result; + + auto fillConfigs = [&](const auto& configBundle) { + if (configBundle.has_main_config()) { + result.MainConfig = configBundle.main_config(); + } + + if (configBundle.has_storage_config()) { + result.StorageConfig = configBundle.storage_config(); + } + }; + + switch (request.action_case()) { + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceEnableDedicatedStorageSection: + result.SwitchDedicatedStorageSection = true; + result.DedicatedConfigMode = true; + fillConfigs(request.replace_enable_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceDisableDedicatedStorageSection: + result.SwitchDedicatedStorageSection = false; + result.MainConfig = request.replace_disable_dedicated_storage_section(); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplaceWithDedicatedStorageSection: + result.DedicatedConfigMode = true; + fillConfigs(request.replace_with_dedicated_storage_section()); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::kReplace: + result.MainConfig = request.replace(); + break; + case Ydb::Config::ReplaceConfigRequest::ActionCase::ACTION_NOT_SET: + break; // TODO: handle as error? + } + + return result; +} + +bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to) { + auto shim = ConvertConfigReplaceRequest(from); + + to->CopyFrom(NKikimr::NYaml::BuildInitDistributedStorageCommand(shim.MainConfig.value_or(TString{}))); return true; } -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::BSConfig::ReplaceStorageConfigResult* /*to*/) { +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::Config::ReplaceConfigResult* /*to*/) { } -bool CopyToConfigRequest(const Ydb::BSConfig::FetchStorageConfigRequest &/*from*/, NKikimrBlobStorage::TConfigRequest *to) { +bool CopyToConfigRequest(const Ydb::Config::FetchConfigRequest &/*from*/, NKikimrBlobStorage::TConfigRequest *to) { to->AddCommand()->MutableReadHostConfig(); to->AddCommand()->MutableReadBox(); return true; } -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::BSConfig::FetchStorageConfigResult *to) { +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::Config::FetchConfigResult *to) { auto hostConfigStatus = from.GetStatus()[0]; auto boxStatus = from.GetStatus()[1]; NKikimrConfig::StorageConfig storageConfig; @@ -77,45 +125,61 @@ void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb } } storageConfig.set_item_config_generation(itemConfigGeneration); - to->set_yaml_config(NYaml::ParseProtoToYaml(storageConfig)); + auto& config = *to->add_config(); + auto& identity = *config.mutable_identity(); + identity.set_version(itemConfigGeneration); + identity.set_cluster(AppData()->ClusterName); + identity.mutable_main(); + config.set_config(NYaml::ParseProtoToYaml(storageConfig)); } class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc { + Ydb::Config::ReplaceConfigResult> { public: - using TBase = TBSConfigRequestGrpc; + using TBase = TBSConfigRequestGrpc; using TBase::TBase; - bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { + bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { + const auto& request = *GetProtoRequest(); + if (request.dry_run()) { + status = Ydb::StatusIds::BAD_REQUEST; + issues.AddIssue("DryRun is not supported yet."); + return false; + } return true; } + NACLib::EAccessRights GetRequiredAccessRights() const { return NACLib::GenericManage; } void FillDistconfQuery(NStorage::TEvNodeConfigInvokeOnRoot& ev) { auto *cmd = ev.Record.MutableReplaceStorageConfig(); - auto *request = GetProtoRequest(); - if (request->has_yaml_config()) { - cmd->SetYAML(request->yaml_config()); + + auto shim = ConvertConfigReplaceRequest(*GetProtoRequest()); + + if (shim.MainConfig) { + cmd->SetYAML(*shim.MainConfig); } - if (request->has_storage_yaml_config()) { - cmd->SetStorageYAML(request->storage_yaml_config()); + if (shim.StorageConfig) { + cmd->SetStorageYAML(*shim.StorageConfig); } - if (request->has_switch_dedicated_storage_section()) { - cmd->SetSwitchDedicatedStorageSection(request->switch_dedicated_storage_section()); + if (shim.SwitchDedicatedStorageSection) { + cmd->SetSwitchDedicatedStorageSection(*shim.SwitchDedicatedStorageSection); } - cmd->SetDedicatedStorageSectionConfigMode(request->dedicated_config_mode()); + cmd->SetDedicatedStorageSectionConfigMode(shim.DedicatedConfigMode); } void FillDistconfResult(NKikimrBlobStorage::TEvNodeConfigInvokeOnRootResult& /*record*/, - Ydb::BSConfig::ReplaceStorageConfigResult& /*result*/) + Ydb::Config::ReplaceConfigResult& /*result*/) {} bool IsDistconfEnableQuery() const { NKikimrConfig::TAppConfig newConfig; try { - newConfig = NYaml::Parse(GetProtoRequest()->yaml_config()); + auto shim = ConvertConfigReplaceRequest(*GetProtoRequest()); + auto config = NFyaml::TDocument::Parse(shim.MainConfig.value_or(TString{"{}"})); + newConfig = NYamlConfig::YamlToProto(config.Root(), true, true); } catch (const std::exception&) { return false; // assuming no distconf enabled in this config } @@ -124,27 +188,35 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc ProcessControllerQuery() override { auto *request = GetProtoRequest(); - auto opt = [&](auto&& has, auto&& get) { - return std::invoke(has, request) ? std::make_optional(std::invoke(get, request)) : std::nullopt; - }; - using T = std::decay_t; + + auto shim = ConvertConfigReplaceRequest(*request); + return std::make_unique( - opt(&T::has_yaml_config, &T::yaml_config), - opt(&T::has_storage_yaml_config, &T::storage_yaml_config), - opt(&T::has_switch_dedicated_storage_section, &T::switch_dedicated_storage_section), - request->dedicated_config_mode()); + shim.MainConfig, + shim.StorageConfig, + shim.SwitchDedicatedStorageSection, + shim.DedicatedConfigMode, + request->allow_unknown_fields() || request->bypass_checks(), + request->bypass_checks()); } }; class TFetchStorageConfigRequest : public TBSConfigRequestGrpc { + Ydb::Config::FetchConfigResult> { public: - using TBase = TBSConfigRequestGrpc; + using TBase = TBSConfigRequestGrpc; using TBase::TBase; - bool ValidateRequest(Ydb::StatusIds::StatusCode& /*status*/, NYql::TIssues& /*issues*/) override { + bool ValidateRequest(Ydb::StatusIds::StatusCode& status, NYql::TIssues& issues) override { + const auto& request = *GetProtoRequest(); + if (request.mode_case() != Ydb::Config::FetchConfigRequest::ModeCase::kAll) { + status = Ydb::StatusIds::BAD_REQUEST; + issues.AddIssue("Only fetch mode \"all\" is supported now."); + return false; + } return true; } + NACLib::EAccessRights GetRequiredAccessRights() const { return NACLib::GenericManage; } @@ -154,8 +226,16 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpcClusterName); + identity.mutable_main(); + config.set_config(conf); } bool IsDistconfEnableQuery() const { @@ -166,17 +246,30 @@ class TFetchStorageConfigRequest : public TBSConfigRequestGrpc(); auto& record = ev->Record; - record.SetDedicatedStorageSection(request.dedicated_storage_section()); - record.SetDedicatedClusterSection(request.dedicated_cluster_section()); + + switch (request.mode_case()) { + case Ydb::Config::FetchConfigRequest::ModeCase::kAll: + if (request.all().config_transform_case() == Ydb::Config::FetchConfigRequest::FetchModeAll::ConfigTransformCase::kDetachStorageConfigSection) { + record.SetDedicatedStorageSection(true); + record.SetDedicatedClusterSection(true); + } + break; + case Ydb::Config::FetchConfigRequest::ModeCase::kTarget: + // TODO: implement, currently impossible (see ValidateRequest) + break; + case Ydb::Config::FetchConfigRequest::ModeCase::MODE_NOT_SET: + break; // TODO: maybe error + } + return ev; } }; -void DoReplaceBSConfig(std::unique_ptr p, const IFacilityProvider&) { +void DoReplaceConfig(std::unique_ptr p, const IFacilityProvider&) { TActivationContext::AsActorContext().Register(new TReplaceStorageConfigRequest(p.release())); } -void DoFetchBSConfig(std::unique_ptr p, const IFacilityProvider&) { +void DoFetchConfig(std::unique_ptr p, const IFacilityProvider&) { TActivationContext::AsActorContext().Register(new TFetchStorageConfigRequest(p.release())); } diff --git a/ydb/core/grpc_services/rpc_bsconfig_base.h b/ydb/core/grpc_services/rpc_config_base.h similarity index 88% rename from ydb/core/grpc_services/rpc_bsconfig_base.h rename to ydb/core/grpc_services/rpc_config_base.h index 649df8511f88..67dee85fa793 100644 --- a/ydb/core/grpc_services/rpc_bsconfig_base.h +++ b/ydb/core/grpc_services/rpc_config_base.h @@ -3,7 +3,7 @@ #include "rpc_deferrable.h" -#include +#include #include #include #include @@ -103,10 +103,10 @@ class TDriveDeviceSet { }; -bool CopyToConfigRequest(const Ydb::BSConfig::ReplaceStorageConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to); -bool CopyToConfigRequest(const Ydb::BSConfig::FetchStorageConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to); -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::BSConfig::ReplaceStorageConfigResult* /*to*/); -void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::BSConfig::FetchStorageConfigResult *to); +bool CopyToConfigRequest(const Ydb::Config::ReplaceConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to); +bool CopyToConfigRequest(const Ydb::Config::FetchConfigRequest &from, NKikimrBlobStorage::TConfigRequest *to); +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &/*from*/, Ydb::Config::ReplaceConfigResult* /*to*/); +void CopyFromConfigResponse(const NKikimrBlobStorage::TConfigResponse &from, Ydb::Config::FetchConfigResult *to); template class TBaseBSConfigRequest { @@ -281,14 +281,36 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor) { + if constexpr (std::is_same_v) { TResultRecord result; const auto& record = ev->Get()->Record; if (record.HasClusterYaml()) { - result.set_yaml_config(ev->Get()->Record.GetClusterYaml()); + auto conf = ev->Get()->Record.GetClusterYaml(); + auto metadata = NYamlConfig::GetMainMetadata(conf); + if (metadata.Version && metadata.Cluster) { + auto& config = *result.add_config(); + auto& identity = *config.mutable_identity(); + identity.set_version(*metadata.Version); + identity.set_cluster(*metadata.Cluster); + identity.mutable_main(); + config.set_config(conf); + } else { + // impossible + } } if (record.HasStorageYaml()) { - result.set_storage_yaml_config(ev->Get()->Record.GetStorageYaml()); + auto conf = ev->Get()->Record.GetStorageYaml(); + auto metadata = NYamlConfig::GetStorageMetadata(conf); + if (metadata.Version && metadata.Cluster) { + auto& config = *result.add_config(); + auto& identity = *config.mutable_identity(); + identity.set_version(*metadata.Version); + identity.set_cluster(*metadata.Cluster); + identity.mutable_storage(); + config.set_config(conf); + } else { + // impossible + } } self->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, self->ActorContext()); } else { @@ -299,7 +321,7 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor) { + if constexpr (std::is_same_v) { const auto& record = ev->Get()->Record; if (record.GetStatus() == NKikimrBlobStorage::TEvControllerReplaceConfigResponse::Success) { TResultRecord result; diff --git a/ydb/core/grpc_services/service_bsconfig.h b/ydb/core/grpc_services/service_config.h similarity index 63% rename from ydb/core/grpc_services/service_bsconfig.h rename to ydb/core/grpc_services/service_config.h index c09d5b40db4a..5b665ed30b35 100644 --- a/ydb/core/grpc_services/service_bsconfig.h +++ b/ydb/core/grpc_services/service_config.h @@ -7,9 +7,9 @@ namespace NKikimr::NGRpcService { class IRequestOpCtx; class IFacilityProvider; - void DoReplaceBSConfig(std::unique_ptr p, const IFacilityProvider&); + void DoReplaceConfig(std::unique_ptr p, const IFacilityProvider&); - void DoFetchBSConfig(std::unique_ptr p, const IFacilityProvider&); + void DoFetchConfig(std::unique_ptr p, const IFacilityProvider&); void DoBootstrapCluster(std::unique_ptr p, const IFacilityProvider&); diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index be849cc6046b..aac04d13ae77 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -5,8 +5,8 @@ ADDINCL( ) SRCS( - audit_log.cpp audit_dml_operations.cpp + audit_log.cpp audit_logins.cpp db_metadata_cache.h grpc_endpoint_publish_actor.cpp @@ -22,34 +22,34 @@ SRCS( rpc_alter_table.cpp rpc_backup.cpp rpc_begin_transaction.cpp - rpc_bsconfig.cpp rpc_calls.cpp rpc_cancel_operation.cpp rpc_cms.cpp rpc_commit_transaction.cpp - rpc_dynamic_config.cpp + rpc_config.cpp rpc_copy_table.cpp rpc_copy_tables.cpp - rpc_export.cpp rpc_create_coordination_node.cpp rpc_create_table.cpp rpc_describe_coordination_node.cpp + rpc_describe_external_data_source.cpp + rpc_describe_external_table.cpp rpc_describe_path.cpp rpc_describe_table.cpp rpc_describe_table_options.cpp - rpc_describe_external_data_source.cpp - rpc_describe_external_table.cpp + rpc_discovery.cpp rpc_drop_coordination_node.cpp rpc_drop_table.cpp - rpc_discovery.cpp + rpc_dynamic_config.cpp rpc_execute_data_query.cpp rpc_execute_scheme_query.cpp rpc_execute_yql_script.cpp - rpc_explain_yql_script.cpp rpc_explain_data_query.cpp + rpc_explain_yql_script.cpp + rpc_export.cpp rpc_forget_operation.cpp - rpc_fq_internal.cpp rpc_fq.cpp + rpc_fq_internal.cpp rpc_get_operation.cpp rpc_get_scale_recommendation.cpp rpc_get_shard_locations.cpp @@ -61,20 +61,21 @@ SRCS( rpc_kh_snapshots.cpp rpc_kqp_base.cpp rpc_list_operations.cpp - rpc_login.cpp rpc_load_rows.cpp rpc_log_store.cpp - rpc_node_registration.cpp + rpc_login.cpp rpc_maintenance.cpp rpc_make_directory.cpp rpc_modify_permissions.cpp rpc_monitoring.cpp + rpc_node_registration.cpp + rpc_object_storage.cpp rpc_ping.cpp rpc_prepare_data_query.cpp rpc_rate_limiter_api.cpp rpc_read_columns.cpp - rpc_read_table.cpp rpc_read_rows.cpp + rpc_read_table.cpp rpc_remove_directory.cpp rpc_rename_tables.cpp rpc_replication.cpp @@ -82,9 +83,8 @@ SRCS( rpc_scheme_base.cpp rpc_stream_execute_scan_query.cpp rpc_stream_execute_yql_script.cpp - rpc_whoami.cpp - rpc_object_storage.cpp rpc_view.cpp + rpc_whoami.cpp table_settings.cpp rpc_common/rpc_common_kqp_session.cpp diff --git a/ydb/core/jaeger_tracing/request_discriminator.cpp b/ydb/core/jaeger_tracing/request_discriminator.cpp index 73f46bec1321..51e35f035f02 100644 --- a/ydb/core/jaeger_tracing/request_discriminator.cpp +++ b/ydb/core/jaeger_tracing/request_discriminator.cpp @@ -66,9 +66,9 @@ extern const THashMap NameToRequestType = { {"RateLimiter.DescribeResource", ERequestType::RATELIMITER_DESCRIBE_RESOURCE}, {"RateLimiter.AcquireResource", ERequestType::RATELIMITER_ACQUIRE_RESOURCE}, - {"BSConfig.ReplaceStorageConfig", ERequestType::BSCONFIG_REPLACESTORAGECONFIG}, - {"BSConfig.FetchStorageConfig", ERequestType::BSCONFIG_FETCHSTORAGECONFIG}, - {"BSConfig.Bootstrap", ERequestType::BSCONFIG_BOOTSTRAP}, + {"Config.ReplaceConfig", ERequestType::CONFIG_REPLACECONFIG}, + {"Config.FetchConfig", ERequestType::CONFIG_FETCHCONFIG}, + {"Config.Bootstrap", ERequestType::CONFIG_BOOTSTRAP}, {"Topic.StreamWrite", ERequestType::TOPIC_STREAMWRITE}, {"Topic.StreamWrite.Init", ERequestType::TOPIC_STREAMWRITE_INIT}, diff --git a/ydb/core/jaeger_tracing/request_discriminator.h b/ydb/core/jaeger_tracing/request_discriminator.h index c02f08281cc4..42f46a0110cd 100644 --- a/ydb/core/jaeger_tracing/request_discriminator.h +++ b/ydb/core/jaeger_tracing/request_discriminator.h @@ -71,9 +71,9 @@ enum class ERequestType: size_t { RATELIMITER_DESCRIBE_RESOURCE, RATELIMITER_ACQUIRE_RESOURCE, - BSCONFIG_REPLACESTORAGECONFIG, - BSCONFIG_FETCHSTORAGECONFIG, - BSCONFIG_BOOTSTRAP, + CONFIG_REPLACECONFIG, + CONFIG_FETCHCONFIG, + CONFIG_BOOTSTRAP, PING_GRPC, PING_PROXY, diff --git a/ydb/core/mind/bscontroller/console_interaction.cpp b/ydb/core/mind/bscontroller/console_interaction.cpp index caff978e2d8e..1a8d739cf327 100644 --- a/ydb/core/mind/bscontroller/console_interaction.cpp +++ b/ydb/core/mind/bscontroller/console_interaction.cpp @@ -89,7 +89,13 @@ namespace NKikimr::NBsController { if (ConsolePipe) { Y_ABORT_UNLESS(Self.YamlConfig); if (const auto& [yaml, configVersion, yamlReturnedByFetch] = *Self.YamlConfig; yaml) { - NTabletPipe::SendData(Self.SelfId(), ConsolePipe, new TEvBlobStorage::TEvControllerConsoleCommitRequest(yaml)); + NTabletPipe::SendData( + Self.SelfId(), + ConsolePipe, + new TEvBlobStorage::TEvControllerConsoleCommitRequest( + yaml, + AllowUnknownFields, + BypassMetadataChecks)); } } break; @@ -145,7 +151,13 @@ namespace NKikimr::NBsController { if (ConsolePipe) { Y_ABORT_UNLESS(Self.YamlConfig); if (const auto& [yaml, configVersion, yamlReturnedByFetch] = *Self.YamlConfig; yaml) { - NTabletPipe::SendData(Self.SelfId(), ConsolePipe, new TEvBlobStorage::TEvControllerConsoleCommitRequest(yaml)); + NTabletPipe::SendData( + Self.SelfId(), + ConsolePipe, + new TEvBlobStorage::TEvControllerConsoleCommitRequest( + yaml, + AllowUnknownFields, + BypassMetadataChecks)); } } else { Y_ABORT_UNLESS(!ClientId); @@ -173,7 +185,7 @@ namespace NKikimr::NBsController { case NKikimrBlobStorage::TEvControllerConsoleCommitResponse::NotCommitted: STLOG(PRI_CRIT, BS_CONTROLLER, BSC28, "Console config not committed"); - Y_DEBUG_ABORT(); + Y_DEBUG_ABORT_S(record.GetErrorReason()); // FIXME: fails here on force break; case NKikimrBlobStorage::TEvControllerConsoleCommitResponse::Committed: @@ -257,6 +269,10 @@ namespace NKikimr::NBsController { if (record.HasClusterYaml()) { PendingYamlConfig.emplace(record.GetClusterYaml()); + // don't need to reset them explicitly + // every time we get new request we just replace them + AllowUnknownFields = record.GetAllowUnknownFields(); + BypassMetadataChecks = record.GetBypassMetadataChecks(); } else { PendingYamlConfig.reset(); } @@ -288,6 +304,8 @@ namespace NKikimr::NBsController { auto validateConfigEv = std::make_unique(); validateConfigEv->Record.SetYAML(record.GetClusterYaml()); + validateConfigEv->Record.SetAllowUnknownFields(record.GetAllowUnknownFields()); + validateConfigEv->Record.SetBypassMetadataChecks(record.GetBypassMetadataChecks()); NTabletPipe::SendData(Self.SelfId(), ConsolePipe, validateConfigEv.release()); } @@ -358,14 +376,14 @@ namespace NKikimr::NBsController { auto parseConfig = [&](const TString& yaml, NKikimrConfig::TAppConfig& appConfig, ui64& version) { try { auto json = NYaml::Yaml2Json(YAML::Load(yaml), true); - NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true); + NYaml::Parse(json, NYaml::GetJsonToProtoConfig(AllowUnknownFields), appConfig, true); if (json.Has("metadata")) { if (auto& metadata = json["metadata"]; metadata.Has("version")) { version = metadata["version"].GetUIntegerRobust(); } } } catch (const std::exception& ex) { - throw TExError(TStringBuilder() << "failed to parse YAML config: " << ex.what()); + throw TExError(TStringBuilder() << "failed to parse YAML config: " << ex.what() << "\n" << yaml); } }; diff --git a/ydb/core/mind/bscontroller/console_interaction.h b/ydb/core/mind/bscontroller/console_interaction.h index 5a9d9f834caf..5002d5739fe4 100644 --- a/ydb/core/mind/bscontroller/console_interaction.h +++ b/ydb/core/mind/bscontroller/console_interaction.h @@ -47,6 +47,9 @@ namespace NKikimr::NBsController { bool CommitInProgress = false; std::optional PendingYamlConfig; + bool AllowUnknownFields = false; + bool BypassMetadataChecks = false; + std::optional> PendingStorageYamlConfig; void MakeCommitToConsole(TString& config, ui32 configVersion); diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index 4c7bc46d211d..87552ea39b1e 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1425,6 +1425,8 @@ message TEvControllerProposeConfigResponse { message TEvControllerConsoleCommitRequest { optional string YAML = 1; + optional bool AllowUnknownFields = 2; + optional bool BypassMetadataChecks = 3; } message TEvControllerConsoleCommitResponse { @@ -1445,6 +1447,9 @@ message TEvControllerReplaceConfigRequest { optional bool SkipConsoleValidation = 5; optional bool SwitchDedicatedStorageSection = 6; optional bool DedicatedConfigMode = 7; + // console flags + optional bool AllowUnknownFields = 8; + optional bool BypassMetadataChecks = 9; } message TEvControllerReplaceConfigResponse { @@ -1464,6 +1469,8 @@ message TEvControllerReplaceConfigResponse { message TEvControllerValidateConfigRequest { optional string YAML = 1; + optional bool AllowUnknownFields = 2; + optional bool BypassMetadataChecks = 3; } message TEvControllerValidateConfigResponse { diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index 20797bebbd38..253df965fd44 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -692,6 +692,19 @@ TDatabaseMetadata GetDatabaseMetadata(const TString& config) { return {}; } +TStorageMetadata GetStorageMetadata(const TString& config) { + if (auto doc = GetMetadataDoc(config); doc) { + auto versionNode = doc->Node["version"]; + auto clusterNode = doc->Node["cluster"]; + return TStorageMetadata{ + .Version = versionNode ? std::optional{FromString(versionNode.Scalar())} : std::nullopt, + .Cluster = clusterNode ? std::optional{clusterNode.Scalar()} : std::nullopt, + }; + } + + return {}; +} + TVolatileMetadata GetVolatileMetadata(const TString& config) { if (auto doc = GetMetadataDoc(config); doc) { auto versionNode = doc->Node.at("version"); diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index 81ec9d0d8f4e..c404783291d0 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -185,6 +185,14 @@ struct TMainMetadata { std::optional Cluster; }; +/** + * Represents config metadata + */ +struct TStorageMetadata { + std::optional Version; + std::optional Cluster; +}; + /** * Represents volatile config metadata */ @@ -222,6 +230,11 @@ TMainMetadata GetMainMetadata(const TString& config); */ TDatabaseMetadata GetDatabaseMetadata(const TString& config); +/** + * Parses storage config metadata + */ +TStorageMetadata GetStorageMetadata(const TString& config); + /** * Parses volatile config metadata */ diff --git a/ydb/library/yaml_config/yaml_config_parser.cpp b/ydb/library/yaml_config/yaml_config_parser.cpp index 3cb1f1469dfe..6cf63bab21c1 100644 --- a/ydb/library/yaml_config/yaml_config_parser.cpp +++ b/ydb/library/yaml_config/yaml_config_parser.cpp @@ -544,6 +544,7 @@ namespace NKikimr::NYaml { ui64 nextHostConfigID = 1; + // TODO: validate all host_configs exists (or better just drop this legacy and use yaml anchors) // Find the next available host_config_id if (ephemeralConfig.HostConfigsSize()) { for(const auto& hostConfig : ephemeralConfig.GetHostConfigs()) { @@ -699,8 +700,10 @@ namespace NKikimr::NYaml { auto& vdiskLoc = ctx.CombinedDiskInfo[TCombinedDiskInfoKey{}]; vdiskLoc.SetNodeID("1"); - vdiskLoc.SetPath(drivePath.value()); - vdiskLoc.SetPDiskCategory(diskType.value()); + if (drivePath && diskType) { + vdiskLoc.SetPath(drivePath.value()); + vdiskLoc.SetPDiskCategory(diskType.value()); + } } if (!config.HasChannelProfileConfig()) { @@ -710,7 +713,9 @@ namespace NKikimr::NYaml { auto& channel = *channelProfile.AddChannel(); channel.SetErasureSpecies(erasureName); channel.SetPDiskCategory(1); - channel.SetStoragePoolKind(diskTypeLower.value()); + if (diskTypeLower) { + channel.SetStoragePoolKind(diskTypeLower.value()); + } }; } } @@ -1478,9 +1483,9 @@ namespace NKikimr::NYaml { return result; } - Ydb::BSConfig::ReplaceStorageConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { - Ydb::BSConfig::ReplaceStorageConfigRequest replaceRequest; - replaceRequest.set_yaml_config(data); + Ydb::Config::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data) { + Ydb::Config::ReplaceConfigRequest replaceRequest; + replaceRequest.set_replace(data); return replaceRequest; } diff --git a/ydb/library/yaml_config/yaml_config_parser.h b/ydb/library/yaml_config/yaml_config_parser.h index de3a0b4221b7..0cc211043030 100644 --- a/ydb/library/yaml_config/yaml_config_parser.h +++ b/ydb/library/yaml_config/yaml_config_parser.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include @@ -56,7 +56,7 @@ namespace NKikimr::NYaml { TSimpleSharedPtr unknownFieldsCollector = nullptr); NKikimrBlobStorage::TConfigRequest BuildInitDistributedStorageCommand(const TString& data); - Ydb::BSConfig::ReplaceStorageConfigRequest BuildReplaceDistributedStorageCommand(const TString& data); + Ydb::Config::ReplaceConfigRequest BuildReplaceDistributedStorageCommand(const TString& data); TString ParseProtoToYaml(const NKikimrConfig::StorageConfig& protoConfig); void ExtractExtraFields(NJson::TJsonValue& json, TTransformContext& ctx); @@ -65,6 +65,8 @@ namespace NKikimr::NYaml { void TransformProtoConfig(TTransformContext& ctx, NKikimrConfig::TAppConfig& config, NKikimrConfig::TEphemeralInputFields& ephemeralConfig, bool relaxed = false); + // TODO: replace bools with something meaningful + void Parse(const NJson::TJsonValue& json, NProtobufJson::TJson2ProtoConfig convertConfig, NKikimrConfig::TAppConfig& config, bool transform, bool relaxed = false); NKikimrConfig::TAppConfig Parse(const TString& data, bool transform = true); diff --git a/ydb/public/api/grpc/ya.make b/ydb/public/api/grpc/ya.make index cfd27e416671..811a5ebe927f 100644 --- a/ydb/public/api/grpc/ya.make +++ b/ydb/public/api/grpc/ya.make @@ -5,15 +5,16 @@ MAVEN_GROUP_ID(com.yandex.ydb) GRPC() SRCS( - ydb_federation_discovery_v1.proto - ydb_bsconfig_v1.proto ydb_auth_v1.proto ydb_cms_v1.proto + ydb_config_v1.proto ydb_coordination_v1.proto ydb_debug_v1.proto ydb_discovery_v1.proto ydb_export_v1.proto + ydb_federation_discovery_v1.proto ydb_import_v1.proto + ydb_keyvalue_v1.proto ydb_monitoring_v1.proto ydb_operation_v1.proto ydb_query_v1.proto @@ -22,7 +23,6 @@ SRCS( ydb_scripting_v1.proto ydb_table_v1.proto ydb_topic_v1.proto - ydb_keyvalue_v1.proto ) PEERDIR( diff --git a/ydb/public/api/grpc/ydb_bsconfig_v1.proto b/ydb/public/api/grpc/ydb_bsconfig_v1.proto deleted file mode 100644 index ee880d0abd4d..000000000000 --- a/ydb/public/api/grpc/ydb_bsconfig_v1.proto +++ /dev/null @@ -1,22 +0,0 @@ -syntax = "proto3"; - -package Ydb.BSConfig.V1; - -option java_package = "com.yandex.ydb.bsconfig.v1"; -option java_outer_classname = "BSConfigGrpc"; -option java_multiple_files = true; - -import "ydb/public/api/protos/ydb_bsconfig.proto"; - -service BSConfigService { - - // Initialize Blobstorage/single config - rpc ReplaceStorageConfig(BSConfig.ReplaceStorageConfigRequest) returns (BSConfig.ReplaceStorageConfigResponse); - - // Fetch Blobstorage/single config - rpc FetchStorageConfig(BSConfig.FetchStorageConfigRequest) returns (BSConfig.FetchStorageConfigResponse); - - // Bootstrap automatically configured cluster - rpc BootstrapCluster(BSConfig.BootstrapClusterRequest) returns (BSConfig.BootstrapClusterResponse); - -} diff --git a/ydb/public/api/grpc/ydb_config_v1.proto b/ydb/public/api/grpc/ydb_config_v1.proto new file mode 100644 index 000000000000..2d6e9c503fb0 --- /dev/null +++ b/ydb/public/api/grpc/ydb_config_v1.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package Ydb.Config.V1; + +option java_package = "com.yandex.ydb.config.v1"; +option java_outer_classname = "ConfigGrpc"; +option java_multiple_files = true; + +import "ydb/public/api/protos/ydb_config.proto"; + +service ConfigService { + // Replace one or many configs. + rpc ReplaceConfig(Config.ReplaceConfigRequest) returns (Config.ReplaceConfigResponse); + + // Fetch configs from cluster. + rpc FetchConfig(Config.FetchConfigRequest) returns (Config.FetchConfigResponse); + + // Start cluster initialization process. + rpc BootstrapCluster(Config.BootstrapClusterRequest) returns (Config.BootstrapClusterResponse); +} diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index f6aebfb41e7b..5565a61892eb 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -8,6 +8,7 @@ PEERDIR( SRCS( draft/datastreams.proto + draft/field_transformation.proto draft/fq.proto draft/persqueue_common.proto draft/persqueue_error_codes.proto @@ -20,26 +21,27 @@ SRCS( draft/ydb_tablet.proto draft/ydb_view.proto draft/ymq.proto - draft/field_transformation.proto - ydb_federation_discovery.proto persqueue_error_codes_v1.proto ydb_auth.proto - ydb_debug.proto - ydb_persqueue_v1.proto - ydb_persqueue_cluster_discovery.proto ydb_clickhouse_internal.proto ydb_cms.proto ydb_common.proto + ydb_config.proto ydb_coordination.proto + ydb_debug.proto ydb_discovery.proto ydb_export.proto + ydb_federation_discovery.proto ydb_formats.proto ydb_import.proto ydb_issue_message.proto + ydb_keyvalue.proto ydb_monitoring.proto ydb_operation.proto - ydb_query_stats.proto + ydb_persqueue_cluster_discovery.proto + ydb_persqueue_v1.proto ydb_query.proto + ydb_query_stats.proto ydb_rate_limiter.proto ydb_scheme.proto ydb_scripting.proto @@ -47,8 +49,6 @@ SRCS( ydb_table.proto ydb_topic.proto ydb_value.proto - ydb_keyvalue.proto - ydb_bsconfig.proto ) CPP_PROTO_PLUGIN0(validation ydb/public/lib/validation) diff --git a/ydb/public/api/protos/ydb_bsconfig.proto b/ydb/public/api/protos/ydb_bsconfig.proto deleted file mode 100644 index d1c6c462e84c..000000000000 --- a/ydb/public/api/protos/ydb_bsconfig.proto +++ /dev/null @@ -1,53 +0,0 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package Ydb.BSConfig; - -option java_package = "com.yandex.ydb.bsconfig.proto"; -option java_outer_classname = "BSConfigProtos"; -option java_multiple_files = true; - -import "ydb/public/api/protos/ydb_operation.proto"; - -// -// BSConfig API. -// - -message ReplaceStorageConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - optional string yaml_config = 2; // cluster yaml config - optional string storage_yaml_config = 3; // dedicated storage yaml config (when dual-config mode is enabled) - optional bool switch_dedicated_storage_section = 4; // if filled, can turn on or off dedicated section of YAML config - bool dedicated_config_mode = 5; // if true, then user expects system to work in dual-config mode -} - -message ReplaceStorageConfigResponse { - Ydb.Operations.Operation operation = 1; -} - -message ReplaceStorageConfigResult { -} - -message FetchStorageConfigRequest { - Ydb.Operations.OperationParams operation_params = 1; - bool dedicated_storage_section = 2; - bool dedicated_cluster_section = 3; -} - -message FetchStorageConfigResponse { - Ydb.Operations.Operation operation = 1; -} - -message FetchStorageConfigResult { - optional string yaml_config = 1; - optional string storage_yaml_config = 2; -} - -message BootstrapClusterRequest { - Ydb.Operations.OperationParams operation_params = 1; - string self_assembly_uuid = 2; -} - -message BootstrapClusterResponse { - Ydb.Operations.Operation operation = 1; -} diff --git a/ydb/public/api/protos/ydb_config.proto b/ydb/public/api/protos/ydb_config.proto new file mode 100644 index 000000000000..3bef77fda109 --- /dev/null +++ b/ydb/public/api/protos/ydb_config.proto @@ -0,0 +1,183 @@ +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.Config; + +option java_package = "com.yandex.ydb.config.proto"; +option java_outer_classname = "ConfigProtos"; +option java_multiple_files = true; + +import "ydb/public/api/protos/ydb_operation.proto"; +import "google/protobuf/empty.proto"; + +// Main config is "the" main config used for configuring everyting +// For most installations this config is enough, because it +// allows database administrator to configure all entites: +// whole cluster, database, specific node(s), etc. +message MainConfigIdentity {} + +// Storage config allows administrator separate management of +// storage/infrastructure part from main/application part +// it is useful on large clusters, where enumeration of all +// physical machines takes too much space and make it inconvenient +// to store and manage this configuration data together with other +message StorageConfigIdentity {} + +// Database config allows to manage specific parts of configuration +// by database administrators themself. It mostly applies to +// feature flags enabled on particular database +// This type of config is disabled by default and can be enabled +// in configuration +message DatabaseConfigIdentity { + string database = 1; +} + +// Representation of config metadata. +// Uniquely identifies config. +// Used to describe what config user want to change or receive. +message ConfigIdentity { + // Current config version + uint64 version = 1; + + // Identifier of the cluster + string cluster = 2; + + // Config type with attached identity data specific for particular type + // List of config types may be extended in future + oneof type { + MainConfigIdentity main = 3; + StorageConfigIdentity storage = 4; + DatabaseConfigIdentity database = 5; + } +} + +message ReplaceConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + + message ConfigBundle { + optional string main_config = 1; + optional string storage_config = 2; + } + + oneof action { + // Replaces config on cluster. + // By default awaits config with matching metadata (kind MainConfig, right cluster and version + 1) + string replace = 2; + + // Replaces storage and/or main configs on cluster in dedicated storage config mode + // By default awaits either MainConfig or StorageConfig or both with matching metadata + ConfigBundle replace_with_dedicated_storage_section = 3; + + // Replaces storage and main configs on cluster in single config mode **AND** enables dedicated storage config mode + // By default awaits MainConfig and StorageConfig with matching metadata + ConfigBundle replace_enable_dedicated_storage_section = 4; + + // Replaces main config on cluster in single config mode **AND** disables dedicated storage config mode + // By default awaits MainConfig with matching metadata + string replace_disable_dedicated_storage_section = 5; + } + + // Try to apply config(s) without making real changes + // Exact data returned and checks done are implementation defined + // Now renders final configs for all labels and makes basic validity checks + // All errors returned as issues in response + bool dry_run = 6; + + // Allows to apply config with fields unknown to config parser + // It is useful when user need to try some new experimental flag, then rollback to previous version + // and want to update existing config keeping flag from newer version in config + // All unknown fields still reported in result as warnings + bool allow_unknown_fields = 7; + + // Allows to apply config with incorrect metadata + // Metadata of stored config after this call will be replaced with correct one + // Exact set of excluded checks is implementation defined, this flag + // may ignore additional checks in future + // BEWARE: with this flag set user can apply config on wrong cluster + // or overwrite other users changes in case of race + // Behaviour identical to deprecated DynConfig.SetConfig() call + bool bypass_checks = 8; +} + +message ReplaceConfigResponse { + Ydb.Operations.Operation operation = 1; +} + +message ReplaceConfigResult {} + +message FetchConfigRequest { + Ydb.Operations.OperationParams operation_params = 1; + + // Settings for mode "all" + // In this mode server will return all configs to user + // There are additional settings allowing to transform configs in server answer + message FetchModeAll { + // Use this option to somehow transform main config + // Currently used to automatically derive detached StorageSection from MainConfig + // or vice versa + oneof config_transform { + // Optionally may be set to explicitly tell "do not transform anything" + google.protobuf.Empty none = 1; + + // Fetch will return MainConfig and StorageConfig derived from MainConfig + // by detaching storage config + // (MainConfig) -> (MainConfig*, StorageConfig) + // MainConfig with asterisk means MainConfig with excluded storage-related sections + google.protobuf.Empty detach_storage_config_section = 2; + + // Fetch will return MainConfig derived from MainConfig and StorageConfig + // by attaching storage config to main config + // (MainConfig*, StorageConfig) -> (MainConfig) + // MainConfig with asterisk means MainConfig with excluded storage-related sections + google.protobuf.Empty attach_storage_config_section = 3; + } + } + + // Settings for mode "by identity" + // In this mode server will return all configs with identites which are exactly match to identites in request + // If there is no matching identity on server the identity will be skipped + message FetchModeByIdentity { + // List of configs to fetch + repeated ConfigIdentity identity = 1; + } + + // Depending on mode Fetch will return different set of configs + // List of modes may be extended in future + oneof mode { + // Fetch all configs from cluster + FetchModeAll all = 2; + + // Fetch all configs with matching identities + FetchModeByIdentity target = 3; + } +} + +message FetchConfigResponse { + Ydb.Operations.Operation operation = 1; +} + +// Config with attached identity +// Allows to operate on on configs without necessary parsing config metadta field +// For example filtering out fetch result based on some specific rules +message ConfigEntry { + ConfigIdentity identity = 1; + string config = 2; +} + +message FetchConfigResult { + repeated ConfigEntry config = 1; +} + +message BootstrapClusterRequest { + Ydb.Operations.OperationParams operation_params = 1; + + // This field used to make this request indempotent + // Must be set to arbitrary **unique** string + string self_assembly_uuid = 2; +} + +message BootstrapClusterResponse { + Ydb.Operations.Operation operation = 1; +} + +message BootstrapClusterResult {} diff --git a/ydb/public/lib/ydb_cli/commands/ya.make b/ydb/public/lib/ydb_cli/commands/ya.make index 556f632b32bd..7ded23503157 100644 --- a/ydb/public/lib/ydb_cli/commands/ya.make +++ b/ydb/public/lib/ydb_cli/commands/ya.make @@ -59,7 +59,7 @@ PEERDIR( ydb/public/lib/ydb_cli/dump/files ydb/public/lib/ydb_cli/import ydb/public/lib/ydb_cli/topic - ydb/public/sdk/cpp/src/client/bsconfig + ydb/public/sdk/cpp/src/client/config ydb/public/sdk/cpp/src/client/coordination ydb/public/sdk/cpp/src/client/debug ydb/public/sdk/cpp/src/client/draft diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index 49fc537402d0..88e3685f9645 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -2,7 +2,7 @@ #include "ydb_dynamic_config.h" -#include +#include #include #define INCLUDE_YDB_INTERNAL_H @@ -39,7 +39,7 @@ void TCommandClusterBootstrap::Parse(TConfig& config) { int TCommandClusterBootstrap::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); - NYdb::NStorageConfig::TStorageConfigClient client(*driver); + NYdb::NConfig::TConfigClient client(*driver); auto result = client.BootstrapCluster(SelfAssemblyUUID).GetValueSync(); NStatusHelpers::ThrowOnErrorOrPrintIssues(result); return EXIT_SUCCESS; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 67d60b4fa4eb..b3087bb1ad37 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -1,6 +1,7 @@ #include "ydb_dynamic_config.h" #include +#include #include #include @@ -196,15 +197,44 @@ void TCommandConfigReplace::Parse(TConfig& config) { int TCommandConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NDynamicConfig::TDynamicConfigClient(*driver); - auto exec = [&]() { - if (Force) { - return client.SetConfig(DynamicConfig, DryRun, AllowUnknownFields).GetValueSync(); - } + auto client = NYdb::NConfig::TConfigClient(*driver); + + NYdb::NConfig::TReplaceConfigSettings settings; + + if (Force) { + settings.BypassChecks(); + } + + if (DryRun) { + settings.DryRun(); + } + + if (AllowUnknownFields) { + settings.AllowUnknownFields(); + } + + auto status = client.ReplaceConfig(DynamicConfig, settings).GetValueSync(); + + if (status.IsUnimplementedError()) { + Cerr << "Warning: Fallback to DynamicConfig API" << Endl; + + auto client = NYdb::NDynamicConfig::TDynamicConfigClient(*driver); + + status = [&]() { + if (Force) { + return client.SetConfig( + DynamicConfig, + DryRun, + AllowUnknownFields).GetValueSync(); + } + + return client.ReplaceConfig( + DynamicConfig, + DryRun, + AllowUnknownFields).GetValueSync(); + }(); + } - return client.ReplaceConfig(DynamicConfig, DryRun, AllowUnknownFields).GetValueSync(); - }; - auto status = exec(); NStatusHelpers::ThrowOnErrorOrPrintIssues(status); if (!status.GetIssues()) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index fb074502ac57..c570ffd4f8df 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -1,6 +1,6 @@ #include "ydb_storage_config.h" -#include +#include #include #include @@ -62,23 +62,43 @@ void TCommandStorageConfigFetch::Parse(TConfig& config) { int TCommandStorageConfigFetch::Run(TConfig& config) { auto driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); - auto result = client.FetchStorageConfig(DedicatedStorageSection, DedicatedClusterSection).GetValueSync(); + auto client = NYdb::NConfig::TConfigClient(*driver); + + bool needDetach = DedicatedStorageSection || DedicatedClusterSection; + + NYdb::NConfig::TFetchAllConfigsSettings settings; + + if (needDetach) { + settings.Transform(NYdb::NConfig::EFetchAllConfigsTransform::DETACH_STORAGE_CONFIG_SECTION); + } + + auto result = client.FetchAllConfigs(settings).GetValueSync(); NStatusHelpers::ThrowOnError(result); - const auto& clusterConfig = result.GetConfig(); - const auto& storageConfig = result.GetStorageConfig(); + TString clusterConfig; + TString storageConfig; + + for (const auto& entry : result.GetConfigs()) { + std::visit([&](auto&& arg) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + clusterConfig = entry.Config; + } else if constexpr (std::is_same_v) { + storageConfig = entry.Config; + } + }, entry.Identity); + } if (!clusterConfig.empty()) { if (!storageConfig.empty() || DedicatedStorageSection) { - Cout << "cluster config: " << Endl; + Cerr << "cluster config: " << Endl; } Cout << WrapYaml(TString(clusterConfig)); } if (!storageConfig.empty()) { if (!clusterConfig.empty() || DedicatedClusterSection) { - Cout << "storage config:" << Endl; + Cerr << "storage config:" << Endl; } Cout << WrapYaml(TString(storageConfig)); } @@ -141,8 +161,20 @@ void TCommandStorageConfigReplace::Parse(TConfig& config) { int TCommandStorageConfigReplace::Run(TConfig& config) { std::unique_ptr driver = std::make_unique(CreateDriver(config)); - auto client = NYdb::NStorageConfig::TStorageConfigClient(*driver); - auto status = client.ReplaceStorageConfig(ClusterYaml, StorageYaml, SwitchDedicatedStorageSection, DedicatedConfigMode).GetValueSync(); + auto client = NYdb::NConfig::TConfigClient(*driver); + + auto status = [&]() { + if (SwitchDedicatedStorageSection && *SwitchDedicatedStorageSection) { + return client.ReplaceConfigDisableDedicatedStorageSection(ClusterYaml.value()).GetValueSync(); + } else if (SwitchDedicatedStorageSection && !*SwitchDedicatedStorageSection) { + return client.ReplaceConfigEnableDedicatedStorageSection(ClusterYaml.value(), StorageYaml.value()).GetValueSync(); + } else if (DedicatedConfigMode) { + return client.ReplaceConfig(ClusterYaml.value(), StorageYaml.value()).GetValueSync(); + } else { + return client.ReplaceConfig(ClusterYaml.value()).GetValueSync(); + } + }(); + NStatusHelpers::ThrowOnError(status); if (!status.GetIssues()) { diff --git a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp b/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp deleted file mode 100644 index c4d588aad92b..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "ydb_storage_config.h" - -#include -#include - -namespace NYdb::NStorageConfig { - -class TStorageConfigClient::TImpl : public TClientImplCommon { -public: - TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) - { - } - - TAsyncStatus ReplaceStorageConfig(const TString& config) { - auto request = MakeRequest(); - request.set_yaml_config(config); - - return RunSimple( - std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncReplaceStorageConfig); - } - - TAsyncFetchStorageConfigResult FetchStorageConfig(const TStorageConfigSettings& settings = {}) { - auto request = MakeOperationRequest(settings); - auto promise = NThreading::NewPromise(); - - auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - TString config; - if (Ydb::BSConfig::FetchStorageConfigResult result; any && any->UnpackTo(&result)) { - config = result.yaml_config(); - } - - TFetchStorageConfigResult val(TStatus(std::move(status)), std::move(config)); - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred( - std::move(request), - extractor, - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncFetchStorageConfig, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings)); - return promise.GetFuture(); - } - - TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID) { - auto request = MakeRequest(); - request.set_self_assembly_uuid(selfAssemblyUUID); - - return RunSimple(std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncBootstrapCluster); - } -}; - -TStorageConfigClient::TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TStorageConfigClient::TImpl(CreateInternalInterface(driver), settings)) -{} - -TStorageConfigClient::~TStorageConfigClient() = default; - -TAsyncStatus TStorageConfigClient::ReplaceStorageConfig(const TString& config) { - return Impl_->ReplaceStorageConfig(config); -} - -TAsyncFetchStorageConfigResult TStorageConfigClient::FetchStorageConfig(const TStorageConfigSettings& settings) { - return Impl_->FetchStorageConfig(settings); -} - -TAsyncStatus TStorageConfigClient::BootstrapCluster(const TString& selfAssemblyUUID) { - return Impl_->BootstrapCluster(selfAssemblyUUID); -} - -} diff --git a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.h b/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.h deleted file mode 100644 index a2178fc29ccc..000000000000 --- a/ydb/public/sdk/cpp/client/ydb_bsconfig/ydb_storage_config.h +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace NYdb::NStorageConfig { - -struct TFetchStorageConfigResult : public TStatus { - TFetchStorageConfigResult( - TStatus&& status, - TString&& config) - : TStatus(std::move(status)) - , Config_(std::move(config)) - {} - - const TString& GetConfig() const { - return Config_; - } - -private: - TString Config_; -}; - -using TAsyncFetchStorageConfigResult = NThreading::TFuture; - -struct TStorageConfigSettings : public NYdb::TOperationRequestSettings {}; - -class TStorageConfigClient { -public: - - explicit TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); - - ~TStorageConfigClient(); - - // Replace config - TAsyncStatus ReplaceStorageConfig(const TString& config); - - // Fetch current cluster storage config - TAsyncFetchStorageConfigResult FetchStorageConfig(const TStorageConfigSettings& settings = {}); - - // Bootstrap cluster with automatic configuration - TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID); - -private: - class TImpl; - - std::unique_ptr Impl_; -}; - -} diff --git a/ydb/public/sdk/cpp/client/ydb_bsconfig/ya.make b/ydb/public/sdk/cpp/client/ydb_config/ya.make similarity index 87% rename from ydb/public/sdk/cpp/client/ydb_bsconfig/ya.make rename to ydb/public/sdk/cpp/client/ydb_config/ya.make index 28a8afbabafe..578691b9d383 100644 --- a/ydb/public/sdk/cpp/client/ydb_bsconfig/ya.make +++ b/ydb/public/sdk/cpp/client/ydb_config/ya.make @@ -1,7 +1,7 @@ LIBRARY() SRCS( - ydb_storage_config.cpp + ydb_config.cpp ) PEERDIR( diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp new file mode 100644 index 000000000000..dc97681a453c --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.cpp @@ -0,0 +1,182 @@ +#include "ydb_storage_config.h" + +#include +#include + +namespace NYdb::NConfig { + +class TConfigClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + request.set_replace(mainConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_with_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + request.set_replace_disable_dedicated_storage_section(mainConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_enable_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::vector configs; + if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { + for (const auto& entry : result.config()) { + TIdentityTypes identity; + + switch (entry.identity().type_case()) { + case Ydb::Config::ConfigIdentity::TypeCase::kMain: + identity = TMainConfigIdentity { + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::kStorage: + identity = TStorageConfigIdentity { + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::kDatabase: + identity = TDatabaseConfigIdentity { + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + .Database = entry.identity().database().database(), + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::TYPE_NOT_SET: + break; // leave in monostate; uknown identity + } + + configs.push_back(TConfig{ + .Identity = identity, + .Config = entry.config(), + }); + } + } + + TFetchConfigResult val(TStatus(std::move(status)), std::move(configs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Config::V1::ConfigService::Stub::AsyncFetchConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + return promise.GetFuture(); + } + + TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID, const TBootstrapClusterSettings& settings = {}) { + Y_UNUSED(settings); + auto request = MakeRequest(); + request.set_self_assembly_uuid(selfAssemblyUUID); + + return RunSimple(std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncBootstrapCluster); + } +private: + static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) { + request.set_dry_run(settings.DryRun_); + request.set_allow_unknown_fields(settings.AllowUnknownFields_); + request.set_bypass_checks(settings.BypassChecks_); + } +}; + +TConfigClient::TConfigClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TConfigClient::TImpl(CreateInternalInterface(driver), settings)) +{} + +TConfigClient::~TConfigClient() = default; + +TAsyncStatus TConfigClient::ReplaceConfig( + const TString& mainConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfig(mainConfig, settings); +} + +TAsyncStatus TConfigClient::ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfig(mainConfig, storageConfig, settings); +} + +TAsyncStatus TConfigClient::ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfigDisableDedicatedStorageSection(mainConfig, settings); +} + +TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig, settings); +} + +TAsyncFetchConfigResult TConfigClient::FetchAllConfigs(const TFetchAllConfigsSettings& settings) { + return Impl_->FetchAllConfigs(settings); +} + +TAsyncStatus TConfigClient::BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings) +{ + return Impl_->BootstrapCluster(selfAssemblyUUID, settings); +} + +} diff --git a/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h new file mode 100644 index 000000000000..0ade45945aa8 --- /dev/null +++ b/ydb/public/sdk/cpp/client/ydb_config/ydb_config.h @@ -0,0 +1,124 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace NYdb::NConfig { + +struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_FLAG(DryRun); + FLUENT_SETTING_FLAG(AllowUnknownFields); + FLUENT_SETTING_FLAG(BypassChecks); +}; + +enum class EFetchAllConfigsTransform { + NONE, + DETACH_STORAGE_CONFIG_SECTION, + ATTACH_STORAGE_CONFIG_SECTION, +}; + +struct TFetchAllConfigsSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(EFetchAllConfigsTransform, Transform); +}; + +struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; + +struct TMainConfigIdentity { + ui64 Version; + TString Cluster; +}; + +struct TStorageConfigIdentity { + ui64 Version; + TString Cluster; +}; + +struct TDatabaseConfigIdentity { + ui64 Version; + TString Cluster; + TString Database; +}; + +using TIdentityTypes = std::variant< + std::monostate + , TMainConfigIdentity + , TStorageConfigIdentity + , TDatabaseConfigIdentity + >; + +struct TConfig { + TIdentityTypes Identity; + TString Config; +}; + +struct TFetchConfigResult : public TStatus { + TFetchConfigResult( + TStatus&& status, + std::vector&& configs) + : TStatus(std::move(status)) + , Configs_(std::move(configs)) + {} + + const std::vector& GetConfigs() const { + return Configs_; + } + +private: + std::vector Configs_; +}; + +using TAsyncFetchConfigResult = NThreading::TFuture; + +class TConfigClient { +public: + explicit TConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); + + ~TConfigClient(); + + // Replace config + TAsyncStatus ReplaceConfig( + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings = {}); + + // Fetch current cluster storage config + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}); + + // Bootstrap cluster with automatic configuration + TAsyncStatus BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings = {}); + +private: + class TImpl; + + std::unique_ptr Impl_; +}; + +} diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp index b06da2e902a3..4c58cde0b023 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.cpp @@ -56,6 +56,10 @@ void TStatus::CheckStatusOk(const TStringType& str) const { Impl_->CheckStatusOk(str); } +bool TStatus::IsUnimplementedError() const { + return Impl_->Status.Status == EStatus::CLIENT_CALL_UNIMPLEMENTED; +} + void TStatus::RaiseError(const TStringType& str) const { Impl_->RaiseError(str); } diff --git a/ydb/public/sdk/cpp/client/ydb_types/status/status.h b/ydb/public/sdk/cpp/client/ydb_types/status/status.h index 072c4067c935..f1a9333e79ac 100644 --- a/ydb/public/sdk/cpp/client/ydb_types/status/status.h +++ b/ydb/public/sdk/cpp/client/ydb_types/status/status.h @@ -26,6 +26,7 @@ class TStatus { const NYql::TIssues& GetIssues() const; bool IsSuccess() const; bool IsTransportError() const; + bool IsUnimplementedError() const; const TStringType& GetEndpoint() const; const std::multimap& GetResponseMetadata() const; float GetConsumedRu() const; diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h deleted file mode 100644 index a6196feded41..000000000000 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/bsconfig/storage_config.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include - -namespace NYdb::inline V3::NStorageConfig { - -struct TFetchStorageConfigResult : public TStatus { - TFetchStorageConfigResult( - TStatus&& status, - std::string&& config, - std::string&& storage_config) - : TStatus(std::move(status)) - , Config_(std::move(config)) - , StorageConfig_(std::move(storage_config)) - {} - - const std::string& GetConfig() const { - return Config_; - } - - const std::string& GetStorageConfig() const { - return StorageConfig_; - } - -private: - std::string Config_; - std::string StorageConfig_; -}; - -using TAsyncFetchStorageConfigResult = NThreading::TFuture; - -struct TReplaceStorageConfigSettings : public NYdb::TOperationRequestSettings {}; -struct TFetchStorageConfigSettings : public NYdb::TOperationRequestSettings {}; -struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; - -class TStorageConfigClient { -public: - explicit TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); - - ~TStorageConfigClient(); - - // Replace config - TAsyncStatus ReplaceStorageConfig(const std::optional& yaml_config, - const std::optional& storage_yaml_config, - std::optional switch_dedicated_storage_section, - bool dedicated_config_mode, - const TReplaceStorageConfigSettings& settings = {}); - - // Fetch current cluster storage config - TAsyncFetchStorageConfigResult FetchStorageConfig(bool dedicated_storage_section, bool dedicated_cluster_section, - const TFetchStorageConfigSettings& settings = {}); - - // Bootstrap cluster with automatic configuration - TAsyncStatus BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings = {}); - -private: - class TImpl; - - std::unique_ptr Impl_; -}; - -} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h new file mode 100644 index 000000000000..e4fdb38e5832 --- /dev/null +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/config/config.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace NYdb::inline V3::NConfig { + +struct TReplaceConfigSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING_FLAG(DryRun); + FLUENT_SETTING_FLAG(AllowUnknownFields); + FLUENT_SETTING_FLAG(BypassChecks); +}; + +enum class EFetchAllConfigsTransform { + NONE, + DETACH_STORAGE_CONFIG_SECTION, + ATTACH_STORAGE_CONFIG_SECTION, +}; + +struct TFetchAllConfigsSettings : public NYdb::TOperationRequestSettings { + FLUENT_SETTING(EFetchAllConfigsTransform, Transform); +}; + +struct TBootstrapClusterSettings : public NYdb::TOperationRequestSettings {}; + +struct TMainConfigIdentity { + ui64 Version; + TString Cluster; +}; + +struct TStorageConfigIdentity { + ui64 Version; + TString Cluster; +}; + +struct TDatabaseConfigIdentity { + ui64 Version; + TString Cluster; + TString Database; +}; + +using TIdentityTypes = std::variant< + std::monostate + , TMainConfigIdentity + , TStorageConfigIdentity + , TDatabaseConfigIdentity + >; + +struct TConfig { + TIdentityTypes Identity; + TString Config; +}; + +struct TFetchConfigResult : public TStatus { + TFetchConfigResult( + TStatus&& status, + std::vector&& configs) + : TStatus(std::move(status)) + , Configs_(std::move(configs)) + {} + + const std::vector& GetConfigs() const { + return Configs_; + } + +private: + std::vector Configs_; +}; + +using TAsyncFetchConfigResult = NThreading::TFuture; + +class TConfigClient { +public: + explicit TConfigClient(const TDriver& driver, const TCommonClientSettings& settings = {}); + + ~TConfigClient(); + + // Replace config + TAsyncStatus ReplaceConfig( + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings = {}); + + // Replace config + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings = {}); + + // Fetch current cluster storage config + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}); + + // Bootstrap cluster with automatic configuration + TAsyncStatus BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings = {}); + +private: + class TImpl; + + std::unique_ptr Impl_; +}; + +} diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h index 96314b6e2d47..76d2d70934e9 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status/status.h @@ -25,6 +25,7 @@ class TStatus { const NYdb::NIssue::TIssues& GetIssues() const; bool IsSuccess() const; bool IsTransportError() const; + bool IsUnimplementedError() const; const std::string& GetEndpoint() const; const std::multimap& GetResponseMetadata() const; float GetConsumedRu() const; diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp b/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp deleted file mode 100644 index d8db1f0bc711..000000000000 --- a/ydb/public/sdk/cpp/src/client/bsconfig/storage_config.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include - -#include -#include - -#include - -namespace NYdb::inline V3::NStorageConfig { - -class TStorageConfigClient::TImpl : public TClientImplCommon { -public: - TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) - : TClientImplCommon(std::move(connections), settings) - { - } - - TAsyncStatus ReplaceStorageConfig(const std::optional& yaml_config, - const std::optional& storage_yaml_config, - std::optional switch_dedicated_storage_section, - bool dedicated_config_mode, - const TReplaceStorageConfigSettings& settings) { - auto request = MakeRequest(); - - if (yaml_config) { - request.set_yaml_config(*yaml_config); - } - if (storage_yaml_config) { - request.set_storage_yaml_config(*storage_yaml_config); - } - if (switch_dedicated_storage_section) { - request.set_switch_dedicated_storage_section(*switch_dedicated_storage_section); - } - request.set_dedicated_config_mode(dedicated_config_mode); - - return RunSimple( - std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncReplaceStorageConfig, - TRpcRequestSettings::Make(settings)); - } - - TAsyncFetchStorageConfigResult FetchStorageConfig(bool dedicated_storage_section, bool dedicated_cluster_section, - const TFetchStorageConfigSettings& settings) { - auto request = MakeOperationRequest(settings); - if (dedicated_storage_section) { - request.set_dedicated_storage_section(true); - } - if (dedicated_cluster_section) { - request.set_dedicated_cluster_section(true); - } - auto promise = NThreading::NewPromise(); - - auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { - NYdb::TStringType config; - NYdb::TStringType storage_config; - if (Ydb::BSConfig::FetchStorageConfigResult result; any && any->UnpackTo(&result)) { - config = result.yaml_config(); - storage_config = result.storage_yaml_config(); - } - - TFetchStorageConfigResult val(TStatus(std::move(status)), std::string{std::move(config)}, - std::string{std::move(storage_config)}); - promise.SetValue(std::move(val)); - }; - - Connections_->RunDeferred( - std::move(request), - extractor, - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncFetchStorageConfig, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings)); - return promise.GetFuture(); - } - - TAsyncStatus BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings) { - auto request = MakeRequest(); - request.set_self_assembly_uuid(selfAssemblyUUID); - return RunSimple(std::move(request), - &Ydb::BSConfig::V1::BSConfigService::Stub::AsyncBootstrapCluster, - TRpcRequestSettings::Make(settings)); - } -}; - -TStorageConfigClient::TStorageConfigClient(const TDriver& driver, const TCommonClientSettings& settings) - : Impl_(new TStorageConfigClient::TImpl(CreateInternalInterface(driver), settings)) -{} - -TStorageConfigClient::~TStorageConfigClient() = default; - -TAsyncStatus TStorageConfigClient::ReplaceStorageConfig(const std::optional& yaml_config, - const std::optional& storage_yaml_config, std::optional switch_dedicated_storage_section, - bool dedicated_config_mode, const TReplaceStorageConfigSettings& settings) { - return Impl_->ReplaceStorageConfig(yaml_config, storage_yaml_config, switch_dedicated_storage_section, - dedicated_config_mode, settings); -} - -TAsyncFetchStorageConfigResult TStorageConfigClient::FetchStorageConfig(bool dedicated_storage_section, - bool dedicated_cluster_section, const TFetchStorageConfigSettings& settings) { - return Impl_->FetchStorageConfig(dedicated_storage_section, dedicated_cluster_section, settings); -} - -TAsyncStatus TStorageConfigClient::BootstrapCluster(const std::string& selfAssemblyUUID, const TBootstrapClusterSettings& settings) { - return Impl_->BootstrapCluster(selfAssemblyUUID, settings); -} - - -} diff --git a/ydb/public/sdk/cpp/src/client/config/config.cpp b/ydb/public/sdk/cpp/src/client/config/config.cpp new file mode 100644 index 000000000000..8c645c737778 --- /dev/null +++ b/ydb/public/sdk/cpp/src/client/config/config.cpp @@ -0,0 +1,184 @@ +#include + +#include +#include + +#include + +namespace NYdb::inline V3::NConfig { + +class TConfigClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + request.set_replace(mainConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfig(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_with_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfigDisableDedicatedStorageSection(const TString& mainConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + request.set_replace_disable_dedicated_storage_section(mainConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncStatus ReplaceConfigEnableDedicatedStorageSection(const TString& mainConfig, const TString& storageConfig, const TReplaceConfigSettings& settings = {}) { + auto request = MakeRequest(); + auto& replace = *request.mutable_replace_enable_dedicated_storage_section(); + replace.set_main_config(mainConfig); + replace.set_storage_config(storageConfig); + + ApplyReplaceSettings(request, settings); + + return RunSimple( + std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncReplaceConfig); + } + + TAsyncFetchConfigResult FetchAllConfigs(const TFetchAllConfigsSettings& settings = {}) { + auto request = MakeOperationRequest(settings); + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::vector configs; + if (Ydb::Config::FetchConfigResult result; any && any->UnpackTo(&result)) { + for (const auto& entry : result.config()) { + TIdentityTypes identity; + + switch (entry.identity().type_case()) { + case Ydb::Config::ConfigIdentity::TypeCase::kMain: + identity = TMainConfigIdentity { + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::kStorage: + identity = TStorageConfigIdentity { + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::kDatabase: + identity = TDatabaseConfigIdentity { + .Version = entry.identity().version(), + .Cluster = entry.identity().cluster(), + .Database = entry.identity().database().database(), + }; + break; + case Ydb::Config::ConfigIdentity::TypeCase::TYPE_NOT_SET: + break; // leave in monostate; uknown identity + } + + configs.push_back(TConfig{ + .Identity = identity, + .Config = entry.config(), + }); + } + } + + TFetchConfigResult val(TStatus(std::move(status)), std::move(configs)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::Config::V1::ConfigService::Stub::AsyncFetchConfig, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + return promise.GetFuture(); + } + + TAsyncStatus BootstrapCluster(const TString& selfAssemblyUUID, const TBootstrapClusterSettings& settings = {}) { + Y_UNUSED(settings); + auto request = MakeRequest(); + request.set_self_assembly_uuid(selfAssemblyUUID); + + return RunSimple(std::move(request), + &Ydb::Config::V1::ConfigService::Stub::AsyncBootstrapCluster); + } +private: + static void ApplyReplaceSettings(auto& request, const TReplaceConfigSettings& settings) { + request.set_dry_run(settings.DryRun_); + request.set_allow_unknown_fields(settings.AllowUnknownFields_); + request.set_bypass_checks(settings.BypassChecks_); + } +}; + +TConfigClient::TConfigClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TConfigClient::TImpl(CreateInternalInterface(driver), settings)) +{} + +TConfigClient::~TConfigClient() = default; + +TAsyncStatus TConfigClient::ReplaceConfig( + const TString& mainConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfig(mainConfig, settings); +} + +TAsyncStatus TConfigClient::ReplaceConfig( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfig(mainConfig, storageConfig, settings); +} + +TAsyncStatus TConfigClient::ReplaceConfigDisableDedicatedStorageSection( + const TString& mainConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfigDisableDedicatedStorageSection(mainConfig, settings); +} + +TAsyncStatus TConfigClient::ReplaceConfigEnableDedicatedStorageSection( + const TString& mainConfig, + const TString& storageConfig, + const TReplaceConfigSettings& settings) +{ + return Impl_->ReplaceConfigEnableDedicatedStorageSection(mainConfig, storageConfig, settings); +} + +TAsyncFetchConfigResult TConfigClient::FetchAllConfigs(const TFetchAllConfigsSettings& settings) { + return Impl_->FetchAllConfigs(settings); +} + +TAsyncStatus TConfigClient::BootstrapCluster( + const TString& selfAssemblyUUID, + const TBootstrapClusterSettings& settings) +{ + return Impl_->BootstrapCluster(selfAssemblyUUID, settings); +} + +} diff --git a/ydb/public/sdk/cpp/src/client/bsconfig/ya.make b/ydb/public/sdk/cpp/src/client/config/ya.make similarity index 91% rename from ydb/public/sdk/cpp/src/client/bsconfig/ya.make rename to ydb/public/sdk/cpp/src/client/config/ya.make index c329c69b0e18..d129149f9fb6 100644 --- a/ydb/public/sdk/cpp/src/client/bsconfig/ya.make +++ b/ydb/public/sdk/cpp/src/client/config/ya.make @@ -3,7 +3,7 @@ LIBRARY() INCLUDE(${ARCADIA_ROOT}/ydb/public/sdk/cpp/sdk_common.inc) SRCS( - storage_config.cpp + config.cpp ) PEERDIR( diff --git a/ydb/public/sdk/cpp/src/client/types/status/status.cpp b/ydb/public/sdk/cpp/src/client/types/status/status.cpp index 1764499e40dc..be7e44082a78 100644 --- a/ydb/public/sdk/cpp/src/client/types/status/status.cpp +++ b/ydb/public/sdk/cpp/src/client/types/status/status.cpp @@ -54,6 +54,10 @@ bool TStatus::IsTransportError() const { && static_cast(Impl_->Status.Status) <= TRANSPORT_STATUSES_LAST; } +bool TStatus::IsUnimplementedError() const { + return Impl_->Status.Status == EStatus::CLIENT_CALL_UNIMPLEMENTED; +} + void TStatus::CheckStatusOk(const std::string& str) const { Impl_->CheckStatusOk(str); } diff --git a/ydb/public/sdk/cpp/src/client/ya.make b/ydb/public/sdk/cpp/src/client/ya.make index 65e4578e55fa..89be03b388dc 100644 --- a/ydb/public/sdk/cpp/src/client/ya.make +++ b/ydb/public/sdk/cpp/src/client/ya.make @@ -1,7 +1,19 @@ RECURSE( - bsconfig + cms + common_client + common_client/impl + config + coordination + datastreams + debug + discovery draft + driver + export + extension_common extensions + federated_topic + federated_topic/impl helpers impl/ydb_endpoints impl/ydb_internal/common @@ -13,19 +25,6 @@ RECURSE( impl/ydb_internal/thread_pool impl/ydb_internal/value_helpers impl/ydb_stats - resources - cms - common_client - common_client/impl - coordination - datastreams - debug - discovery - driver - export - extension_common - federated_topic - federated_topic/impl import operation params @@ -35,6 +34,7 @@ RECURSE( query query/impl rate_limiter + resources result scheme table diff --git a/ydb/services/bsconfig/grpc_service.cpp b/ydb/services/bsconfig/grpc_service.cpp deleted file mode 100644 index 3853531e0d6c..000000000000 --- a/ydb/services/bsconfig/grpc_service.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "grpc_service.h" - -#include -#include -#include -#include -#include "ydb/library/grpc/server/grpc_method_setup.h" - -namespace NKikimr::NGRpcService { - -TBSConfigGRpcService::TBSConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, NActors::TActorId grpcRequestProxyId) \ - : ActorSystem(actorSystem) \ - , Counters(std::move(counters)) - , GRpcRequestProxyId(grpcRequestProxyId) -{ -} - -TBSConfigGRpcService::~TBSConfigGRpcService() = default; - -void TBSConfigGRpcService::InitService(grpc::ServerCompletionQueue* cq, NYdbGrpc::TLoggerPtr logger) { - CQ = cq; - SetupIncomingRequests(std::move(logger)); -} - -void TBSConfigGRpcService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { - auto getCounterBlock = NGRpcService::CreateCounterCb(Counters, ActorSystem); - - #define SETUP_BS_METHOD(methodName, method, rlMode, requestType) \ - SETUP_METHOD(methodName, method, rlMode, requestType, BSConfig, bsconfig) - - SETUP_BS_METHOD(ReplaceStorageConfig, DoReplaceBSConfig, Rps, BSCONFIG_REPLACESTORAGECONFIG); - SETUP_BS_METHOD(FetchStorageConfig, DoFetchBSConfig, Rps, BSCONFIG_FETCHSTORAGECONFIG); - SETUP_BS_METHOD(BootstrapCluster, DoBootstrapCluster, Rps, BSCONFIG_BOOTSTRAP); - - #undef SETUP_BS_METHOD -} - -} // namespace NKikimr::NGRpcService diff --git a/ydb/services/bsconfig/bsconfig_ut.cpp b/ydb/services/config/bsconfig_ut.cpp similarity index 69% rename from ydb/services/bsconfig/bsconfig_ut.cpp rename to ydb/services/config/bsconfig_ut.cpp index 1cb4aa3696dd..10f42e4d390b 100644 --- a/ydb/services/bsconfig/bsconfig_ut.cpp +++ b/ydb/services/config/bsconfig_ut.cpp @@ -73,7 +73,7 @@ class TBasicKikimrWithGrpcAndRootSchema { } ServerSettings->Formats = new NKikimr::TFormatFactory; ServerSettings->FeatureFlags = appConfig.GetFeatureFlags(); - ServerSettings->RegisterGrpcService("bsconfig"); + ServerSettings->RegisterGrpcService("bsconfig"); Server_.Reset(new NKikimr::Tests::TServer(*ServerSettings)); Tenants_.Reset(new NKikimr::Tests::TTenants(Server_)); @@ -158,73 +158,108 @@ TString NormalizeYaml(const TString& yaml) { return normalized.Str(); } -Y_UNIT_TEST_SUITE(BSConfigGRPCService) { +Y_UNIT_TEST_SUITE(ConfigGRPCService) { template void AdjustCtxForDB(TCtx &ctx) { ctx.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, "root@builtin"); } - void ReplaceStorageConfig(auto &channel, std::optional yamlConfig, std::optional storageYamlConfig, - std::optional switchDedicatedStorageSection, bool dedicatedConfigMode) { - std::unique_ptr stub; - stub = Ydb::BSConfig::V1::BSConfigService::NewStub(channel); - - Ydb::BSConfig::ReplaceStorageConfigRequest request; - if (yamlConfig) { - request.set_yaml_config(*yamlConfig); - } - if (storageYamlConfig) { - request.set_storage_yaml_config(*storageYamlConfig); - } - if (switchDedicatedStorageSection) { - request.set_switch_dedicated_storage_section(*switchDedicatedStorageSection); + void ReplaceConfig( + auto &channel, + std::optional mainConfig, + std::optional storageConfig, + std::optional switchDedicatedStorageSection, + bool dedicatedConfigMode) { + + std::unique_ptr stub; + stub = Ydb::Config::V1::ConfigService::NewStub(channel); + + Ydb::Config::ReplaceConfigRequest request; + + if (!dedicatedConfigMode && !switchDedicatedStorageSection) { + if (mainConfig) { + request.set_replace(*mainConfig); + } + } else if (dedicatedConfigMode && !switchDedicatedStorageSection) { + auto& replace = *request.mutable_replace_with_dedicated_storage_section(); + if (mainConfig) { + replace.set_main_config(*mainConfig); + } + if (storageConfig) { + replace.set_storage_config(*storageConfig); + } + } else if (switchDedicatedStorageSection && *switchDedicatedStorageSection) { + auto& replace = *request.mutable_replace_enable_dedicated_storage_section(); + if (mainConfig) { + replace.set_main_config(*mainConfig); + } + if (storageConfig) { + replace.set_storage_config(*storageConfig); + } + } else if (switchDedicatedStorageSection && !*switchDedicatedStorageSection) { + if (mainConfig) { + request.set_replace_disable_dedicated_storage_section(*mainConfig); + } + } else { + Y_ABORT("invariant violation"); } - request.set_dedicated_config_mode(dedicatedConfigMode); - Ydb::BSConfig::ReplaceStorageConfigResponse response; - Ydb::BSConfig::ReplaceStorageConfigResult result; + Ydb::Config::ReplaceConfigResponse response; + Ydb::Config::ReplaceConfigResult result; - grpc::ClientContext replaceStorageConfigCtx; - AdjustCtxForDB(replaceStorageConfigCtx); - stub->ReplaceStorageConfig(&replaceStorageConfigCtx, request, &response); + grpc::ClientContext replaceConfigCtx; + AdjustCtxForDB(replaceConfigCtx); + stub->ReplaceConfig(&replaceConfigCtx, request, &response); UNIT_ASSERT_CHECK_STATUS(response.operation(), Ydb::StatusIds::SUCCESS); Cerr << "response: " << response.operation().result().DebugString() << Endl; response.operation().result().UnpackTo(&result); } - void FetchStorageConfig(auto& channel, bool dedicatedStorageSection, bool dedicatedClusterSection, - std::optional& yamlConfig, std::optional& storageYamlConfig) { - std::unique_ptr stub; - stub = Ydb::BSConfig::V1::BSConfigService::NewStub(channel); + void FetchConfig( + auto& channel, + bool dedicatedStorageSection, + bool dedicatedClusterSection, + std::optional& mainConfig, + std::optional& storageConfig) { + std::unique_ptr stub; + stub = Ydb::Config::V1::ConfigService::NewStub(channel); + + Ydb::Config::FetchConfigRequest request; + + auto& all = *request.mutable_all(); - Ydb::BSConfig::FetchStorageConfigRequest request; - request.set_dedicated_storage_section(dedicatedStorageSection); - request.set_dedicated_cluster_section(dedicatedClusterSection); + if (dedicatedStorageSection || dedicatedClusterSection) { + all.mutable_detach_storage_config_section(); + } - Ydb::BSConfig::FetchStorageConfigResponse response; - Ydb::BSConfig::FetchStorageConfigResult result; + Ydb::Config::FetchConfigResponse response; + Ydb::Config::FetchConfigResult result; - grpc::ClientContext fetchStorageConfigCtx; - AdjustCtxForDB(fetchStorageConfigCtx); - stub->FetchStorageConfig(&fetchStorageConfigCtx, request, &response); + grpc::ClientContext fetchConfigCtx; + AdjustCtxForDB(fetchConfigCtx); + stub->FetchConfig(&fetchConfigCtx, request, &response); UNIT_ASSERT_CHECK_STATUS(response.operation(), Ydb::StatusIds::SUCCESS); response.operation().result().UnpackTo(&result); - if (result.has_yaml_config()) { - yamlConfig.emplace(result.yaml_config()); - } else { - yamlConfig.reset(); - } + std::optional rcvMainConfig; + std::optional rcvStorageConfig; - if (result.has_storage_yaml_config()) { - storageYamlConfig.emplace(result.storage_yaml_config()); - } else { - storageYamlConfig.reset(); + for (auto& entry : result.config()) { + if (entry.identity().type_case() == Ydb::Config::ConfigIdentity::TypeCase::kMain) { + rcvMainConfig = entry.config(); + } + + if (entry.identity().type_case() == Ydb::Config::ConfigIdentity::TypeCase::kStorage) { + rcvStorageConfig = entry.config(); + } } + + mainConfig = rcvMainConfig; + storageConfig = rcvStorageConfig; } - Y_UNIT_TEST(ReplaceStorageConfig) { + Y_UNIT_TEST(ReplaceConfig) { TKikimrWithGrpcAndRootSchema server; TString yamlConfig = R"( metadata: @@ -263,18 +298,18 @@ selector_config: [] host_config_id: 2 )"; TString yamlConfigExpected = SubstGlobalCopy(yamlConfig, "version: 0", "version: 1"); - ReplaceStorageConfig(server.GetChannel(), yamlConfig, std::nullopt, std::nullopt, false); + ReplaceConfig(server.GetChannel(), yamlConfig, std::nullopt, std::nullopt, false); std::optional yamlConfigFetched, storageYamlConfigFetched; - FetchStorageConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); + FetchConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); UNIT_ASSERT(yamlConfigFetched); UNIT_ASSERT(!storageYamlConfigFetched); UNIT_ASSERT_VALUES_EQUAL(yamlConfigExpected, *yamlConfigFetched); } - Y_UNIT_TEST(FetchStorageConfig) { + Y_UNIT_TEST(FetchConfig) { TKikimrWithGrpcAndRootSchema server; std::optional yamlConfigFetched, storageYamlConfigFetched; - FetchStorageConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); + FetchConfig(server.GetChannel(), false, false, yamlConfigFetched, storageYamlConfigFetched); UNIT_ASSERT(!yamlConfigFetched); UNIT_ASSERT(!storageYamlConfigFetched); } diff --git a/ydb/services/config/grpc_service.cpp b/ydb/services/config/grpc_service.cpp new file mode 100644 index 000000000000..5198fe9215cd --- /dev/null +++ b/ydb/services/config/grpc_service.cpp @@ -0,0 +1,38 @@ +#include "grpc_service.h" + +#include +#include +#include +#include +#include "ydb/library/grpc/server/grpc_method_setup.h" + +namespace NKikimr::NGRpcService { + +TConfigGRpcService::TConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, NActors::TActorId grpcRequestProxyId) \ + : ActorSystem(actorSystem) \ + , Counters(std::move(counters)) + , GRpcRequestProxyId(grpcRequestProxyId) +{ +} + +TConfigGRpcService::~TConfigGRpcService() = default; + +void TConfigGRpcService::InitService(grpc::ServerCompletionQueue* cq, NYdbGrpc::TLoggerPtr logger) { + CQ = cq; + SetupIncomingRequests(std::move(logger)); +} + +void TConfigGRpcService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { + auto getCounterBlock = NGRpcService::CreateCounterCb(Counters, ActorSystem); + + #define SETUP_BS_METHOD(methodName, method, rlMode, requestType) \ + SETUP_METHOD(methodName, method, rlMode, requestType, Config, config) + + SETUP_BS_METHOD(ReplaceConfig, DoReplaceConfig, Rps, CONFIG_REPLACECONFIG); + SETUP_BS_METHOD(FetchConfig, DoFetchConfig, Rps, CONFIG_FETCHCONFIG); + SETUP_BS_METHOD(BootstrapCluster, DoBootstrapCluster, Rps, CONFIG_BOOTSTRAP); + + #undef SETUP_BS_METHOD +} + +} // namespace NKikimr::NGRpcService diff --git a/ydb/services/bsconfig/grpc_service.h b/ydb/services/config/grpc_service.h similarity index 63% rename from ydb/services/bsconfig/grpc_service.h rename to ydb/services/config/grpc_service.h index 4506aa3f1334..a83424748451 100644 --- a/ydb/services/bsconfig/grpc_service.h +++ b/ydb/services/config/grpc_service.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -8,13 +8,13 @@ namespace NKikimr::NGRpcService { -class TBSConfigGRpcService - : public NYdbGrpc::TGrpcServiceBase +class TConfigGRpcService + : public NYdbGrpc::TGrpcServiceBase { public: - TBSConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, + TConfigGRpcService(NActors::TActorSystem* actorSystem, TIntrusivePtr counters, NActors::TActorId grpcRequestProxyId); - ~TBSConfigGRpcService(); + ~TConfigGRpcService(); void InitService(grpc::ServerCompletionQueue* cq, NYdbGrpc::TLoggerPtr logger) override; @@ -29,4 +29,4 @@ class TBSConfigGRpcService grpc::ServerCompletionQueue* CQ = nullptr; }; -} // namespace NKikimr::NGRpcService \ No newline at end of file +} // namespace NKikimr::NGRpcService diff --git a/ydb/services/bsconfig/ut/ya.make b/ydb/services/config/ut/ya.make similarity index 71% rename from ydb/services/bsconfig/ut/ya.make rename to ydb/services/config/ut/ya.make index 94e955e17300..1004716648e1 100644 --- a/ydb/services/bsconfig/ut/ya.make +++ b/ydb/services/config/ut/ya.make @@ -1,4 +1,4 @@ -UNITTEST_FOR(ydb/services/bsconfig) +UNITTEST_FOR(ydb/services/config) SIZE(MEDIUM) @@ -10,7 +10,7 @@ PEERDIR( library/cpp/logger ydb/core/protos ydb/core/testlib/default - ydb/services/bsconfig + ydb/services/config ) YQL_LAST_ABI_VERSION() diff --git a/ydb/services/bsconfig/ya.make b/ydb/services/config/ya.make similarity index 100% rename from ydb/services/bsconfig/ya.make rename to ydb/services/config/ya.make diff --git a/ydb/services/ya.make b/ydb/services/ya.make index cd73f2e86e6d..aa339d9de37b 100644 --- a/ydb/services/ya.make +++ b/ydb/services/ya.make @@ -2,8 +2,8 @@ RECURSE( auth backup bg_tasks - bsconfig cms + config datastreams deprecated/persqueue_v0 discovery diff --git a/ydb/tests/functional/config/test_config_with_metadata.py b/ydb/tests/functional/config/test_config_with_metadata.py index f4772b197510..0a0f7f975e29 100644 --- a/ydb/tests/functional/config/test_config_with_metadata.py +++ b/ydb/tests/functional/config/test_config_with_metadata.py @@ -9,12 +9,12 @@ from ydb.tests.library.harness.util import LogLevels from ydb.tests.library.clients.kikimr_http_client import SwaggerClient from ydb.tests.library.harness.kikimr_runner import KiKiMR -from ydb.tests.library.clients.kikimr_config_client import BSConfigClient +from ydb.tests.library.clients.kikimr_config_client import ConfigClient from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.kv.helpers import create_kv_tablets_and_wait_for_start from ydb.public.api.protos.ydb_status_codes_pb2 import StatusIds -import ydb.public.api.protos.ydb_bsconfig_pb2 as bsconfig +import ydb.public.api.protos.ydb_config_pb2 as config logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def setup_class(cls): host = cls.cluster.nodes[1].host grpc_port = cls.cluster.nodes[1].port cls.swagger_client = SwaggerClient(host, cls.cluster.nodes[1].mon_port) - cls.bsconfig_client = BSConfigClient(host, grpc_port) + cls.config_client = ConfigClient(host, grpc_port) @classmethod def teardown_class(cls): @@ -141,7 +141,7 @@ def setup_class(cls): use_in_memory_pdisks=False, use_config_store=True, separate_node_configs=True, - extra_grpc_services=['bsconfig'], + extra_grpc_services=['config'], metadata_section=cls.metadata_section, additional_log_configs={'BS_NODE': LogLevels.DEBUG}, ) @@ -151,7 +151,7 @@ def setup_class(cls): host = cls.cluster.nodes[1].host grpc_port = cls.cluster.nodes[1].port cls.swagger_client = SwaggerClient(host, cls.cluster.nodes[1].mon_port) - cls.bsconfig_client = BSConfigClient(host, grpc_port) + cls.config_client = ConfigClient(host, grpc_port) def test_cluster_works_with_auto_conf_dir(self): table_path = '/Root/mydb/mytable_auto_conf' @@ -174,16 +174,18 @@ def test_config_stored_in_config_store(self): initial_config['metadata']['version'] = initial_version config_yaml = yaml.dump(initial_config) - replace_storage_config_response = self.bsconfig_client.replace_storage_config(config_yaml) - assert_that(replace_storage_config_response.operation.status == StatusIds.SUCCESS) + replace_config_response = self.config_client.replace_config(config_yaml) + assert_that(replace_config_response.operation.status == StatusIds.SUCCESS) - fetch_storage_config_response = self.bsconfig_client.fetch_storage_config() - assert_that(fetch_storage_config_response.operation.status == StatusIds.SUCCESS) + fetch_config_response = self.config_client.fetch_all_configs() + assert_that(fetch_config_response.operation.status == StatusIds.SUCCESS) - result = bsconfig.FetchStorageConfigResult() - fetch_storage_config_response.operation.result.Unpack(result) + result = config.FetchConfigResult() + fetch_config_response.operation.result.Unpack(result) - fetched_config = result.yaml_config + assert_that(result.config is not None) + assert_that(len(result.config) == 1) + fetched_config = result.config[0].config parsed_fetched_config = yaml.safe_load(fetched_config) assert_that(parsed_fetched_config is not None) assert_that(parsed_fetched_config.get('metadata') is not None) diff --git a/ydb/tests/library/clients/kikimr_config_client.py b/ydb/tests/library/clients/kikimr_config_client.py index 6dbde343ca0a..44e657b47c8d 100644 --- a/ydb/tests/library/clients/kikimr_config_client.py +++ b/ydb/tests/library/clients/kikimr_config_client.py @@ -2,19 +2,20 @@ # -*- coding: utf-8 -*- import time import os +from enum import Enum import grpc import logging -from ydb.public.api.grpc import ydb_bsconfig_v1_pb2_grpc as grpc_server -from ydb.public.api.protos import ydb_bsconfig_pb2 as bsconfig_api +from ydb.public.api.grpc import ydb_config_v1_pb2_grpc as grpc_server +from ydb.public.api.protos import ydb_config_pb2 as config_api logger = logging.getLogger() -def bsconfig_client_factory(server, port, cluster=None, retry_count=1): - return BSConfigClient( +def config_client_factory(server, port, cluster=None, retry_count=1): + return ConfigClient( server, port, cluster=cluster, retry_count=retry_count ) @@ -24,7 +25,12 @@ def channels_list(): return os.getenv('CHANNELS_LIST', '') -class BSConfigClient(object): +class ConfigClient(object): + class FetchTransform(Enum): + NONE = 1 + DETACH_STORAGE_CONFIG_SECTION = 2 + ATTACH_STORAGE_CONFIG_SECTION = 3 + def __init__(self, server, port, cluster=None, retry_count=1): self.server = server self.port = port @@ -37,7 +43,7 @@ def __init__(self, server, port, cluster=None, retry_count=1): ('grpc.max_send_message_length', 64 * 10 ** 6) ] self._channel = grpc.insecure_channel("%s:%s" % (self.server, self.port), options=self._options) - self._stub = grpc_server.BSConfigServiceStub(self._channel) + self._stub = grpc_server.ConfigServiceStub(self._channel) def _get_invoke_callee(self, method): return getattr(self._stub, method) @@ -56,21 +62,26 @@ def invoke(self, request, method): time.sleep(self.__retry_sleep_seconds) - def replace_storage_config(self, yaml_config, storage_yaml_config=None): - request = bsconfig_api.ReplaceStorageConfigRequest() - request.yaml_config = yaml_config - if storage_yaml_config is not None: - request.storage_yaml_config = storage_yaml_config - return self.invoke(request, 'ReplaceStorageConfig') + def replace_config(self, main_config): + request = config_api.ReplaceConfigRequest() + request.replace = main_config + return self.invoke(request, 'ReplaceConfig') + + def fetch_all_configs(self, transform=None): + request = config_api.FetchConfigRequest() + settings = config_api.FetchConfigRequest.FetchModeAll() + + if transform == ConfigClient.FetchTransform.DETACH_STORAGE_CONFIG_SECTION: + settings.set_detach_storage_config_section() + elif transform == ConfigClient.FetchTransform.ATTACH_STORAGE_CONFIG_SECTION: + settings.set_attach_storage_config_section() + + request.all.CopyFrom(settings) - def fetch_storage_config(self, dedicated_storage_section=False, dedicated_cluster_section=False): - request = bsconfig_api.FetchStorageConfigRequest() - request.dedicated_storage_section = dedicated_storage_section - request.dedicated_cluster_section = dedicated_cluster_section - return self.invoke(request, 'FetchStorageConfig') + return self.invoke(request, 'FetchConfig') def bootstrap_cluster(self, self_assembly_uuid): - request = bsconfig_api.BootstrapClusterRequest() + request = config_api.BootstrapClusterRequest() request.self_assembly_uuid = self_assembly_uuid return self.invoke(request, 'BootstrapCluster')